diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b331b02..cace9460 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,11 @@ on: permissions: contents: read +env: + DO_NOT_TRACK: "1" + KTX_TELEMETRY_DISABLED: "1" + NEXT_TELEMETRY_DISABLED: "1" + concurrency: group: ktx-ci-${{ github.ref }} cancel-in-progress: true @@ -212,7 +217,7 @@ jobs: flags: typescript name: typescript disable_search: true - fail_ci_if_error: true + fail_ci_if_error: false - name: Warn when Codecov token is missing for TypeScript if: env.CODECOV_TOKEN_CONFIGURED != 'true' @@ -231,7 +236,7 @@ jobs: flags: python name: python disable_search: true - fail_ci_if_error: true + fail_ci_if_error: false - name: Warn when Codecov token is missing for Python if: env.CODECOV_TOKEN_CONFIGURED != 'true' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 108c1989..b1efa96e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,6 +26,11 @@ permissions: contents: write id-token: write +env: + DO_NOT_TRACK: "1" + KTX_TELEMETRY_DISABLED: "1" + NEXT_TELEMETRY_DISABLED: "1" + concurrency: group: ktx-release-${{ github.ref }} cancel-in-progress: false diff --git a/.github/workflows/star-history.yml b/.github/workflows/star-history.yml new file mode 100644 index 00000000..b7d90c43 --- /dev/null +++ b/.github/workflows/star-history.yml @@ -0,0 +1,72 @@ +name: Refresh star history chart + +on: + schedule: + # Twice daily at 06:00 and 18:00 UTC. + - cron: "0 6,18 * * *" + workflow_dispatch: + +permissions: + contents: write + +env: + DO_NOT_TRACK: "1" + KTX_TELEMETRY_DISABLED: "1" + NEXT_TELEMETRY_DISABLED: "1" + +concurrency: + group: star-history-refresh + cancel-in-progress: true + +jobs: + refresh: + name: Regenerate assets/star-history.svg + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # RELEASE_PAT can push to the protected main branch; the default + # GITHUB_TOKEN is rejected by the branch-protection hook (GH006). + token: ${{ secrets.RELEASE_PAT }} + + - name: Fetch fresh star-history SVG + run: | + set -euo pipefail + # cachebust forces star-history to regenerate instead of serving its + # own server-side cache; --location follows the slug-normalizing 301. + url="https://api.star-history.com/svg?repos=Kaelio/ktx&type=Date&cachebust=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + curl --fail --location --silent --show-error \ + --retry 3 --retry-delay 5 --max-time 60 \ + -o assets/star-history.svg.new "$url" + # Guard against error pages / truncated responses before overwriting. + if ! grep -q "" assets/star-history.svg.new; then + echo "Downloaded file is not a valid SVG; aborting." >&2 + exit 1 + fi + if [ "$(wc -c < assets/star-history.svg.new)" -lt 1000 ]; then + echo "Downloaded SVG is suspiciously small; aborting." >&2 + exit 1 + fi + # The star-history API returns the SVG without a trailing newline, + # which end-of-file-fixer rewrites whenever pre-commit runs + # --all-files on a PR. Because the refresh commit below uses [skip ci], + # the hook never runs against it here, so an un-normalized file + # silently breaks the pre-commit check on every open PR. Normalize to + # exactly one trailing newline before committing. + printf '%s\n' "$(cat assets/star-history.svg.new)" > assets/star-history.svg + rm -f assets/star-history.svg.new + + - name: Commit if changed + run: | + set -euo pipefail + if git diff --quiet -- assets/star-history.svg; then + echo "Star-history chart unchanged; nothing to commit." + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add assets/star-history.svg + # [skip ci] keeps this housekeeping commit from triggering KTX CI. + git commit -m "chore: refresh star history chart [skip ci]" + git push diff --git a/.github/workflows/triage-issues.yml b/.github/workflows/triage-issues.yml index 5a341013..e5817e2c 100644 --- a/.github/workflows/triage-issues.yml +++ b/.github/workflows/triage-issues.yml @@ -7,6 +7,11 @@ on: permissions: issues: write +env: + DO_NOT_TRACK: "1" + KTX_TELEMETRY_DISABLED: "1" + NEXT_TELEMETRY_DISABLED: "1" + jobs: label-external: name: Add needs-triage to external issues @@ -17,7 +22,7 @@ jobs: github.event.issue.author_association != 'COLLABORATOR' steps: - name: Apply needs-triage label - uses: actions/github-script@v7 + uses: actions/github-script@v9 with: script: | await github.rest.issues.addLabels({ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cc2f483d..681cf0ab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,18 @@ repos: - id: check-case-conflict - id: mixed-line-ending + - repo: https://github.com/tombi-toml/tombi-pre-commit + rev: v1.1.0 + hooks: + - id: tombi-format + args: ["--offline"] + # uv.lock is generated and owned by uv, which writes its own canonical + # TOML layout. tombi reformats that layout differently, so once uv + # regenerates the lock (e.g. after a dependency or version change) + # tombi rewrites it and the hook fails on the modified file. Keep uv + # authoritative for its lockfile; tombi still formats hand-edited TOML. + exclude: ^uv\.lock$ + - repo: https://github.com/asottile/pyupgrade rev: v3.21.2 hooks: diff --git a/AGENTS.md b/AGENTS.md index a8640c48..ec715364 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,6 +24,11 @@ database migrations, ORPC contracts, or `python-service/` layout exist here. - **MUST**: Keep package/public API changes intentional. Do not add compatibility wrappers for old **ktx** names unless the user explicitly asks for a migration bridge. +- **MUST**: Avoid compatibility shims for old **ktx** features, command shapes, + configuration formats, or internal APIs. This rule does not prohibit + compatibility support for third-party systems and libraries, such as + Metabase version differences. Keep the **ktx** codebase clean instead of + preserving stale **ktx** behavior. - **MUST**: Treat **ktx** as having no public users unless the user says otherwise. Legacy support is not necessary by default; prefer clean breaking changes over compatibility shims, migration bridges, or preserved stale behavior. @@ -154,6 +159,65 @@ and naming asymmetries are bugs in waiting — see [`docs/code-design.md`](docs/code-design.md). Treat the `MUST` / `MUST NOT` rules there with the same weight as the ones in this file. +## Design Reasoning Defaults + +When proposing a design, an approach, or any non-trivial change, apply these +defaults and run the self-check before presenting it. They encode the +corrections users most often have to make; reaching these conclusions +autonomously — without being asked the leading question — is the bar. + +- **MUST**: Optimize for the best outcome, not for an unstated constraint. Do not + silently adopt "smallest change", "least effort", "cheapest", or "least user + intervention" as the goal unless the user said so. Default to the most correct, + durable solution, and present cost / effort / scope as information for the user + to weigh — not as a ceiling you impose on their behalf. +- **MUST**: Separate one-time cost from recurring cost before discarding an + option. A fixed cost paid once (a setup-time computation, an extra LLM call + during setup, a contract change) to make every later run cheaper or more + correct is usually worth it. Do not reject it with recurring-cost reasoning; + quantify both sides. (Example smell: "don't add an LLM call to a cost-cutting + feature" — wrong when the call is one-time and the savings recur.) +- **MUST**: Treat a user's example as a representative of a class, not as the + spec. Design for the general population the example stands for, then stress-test + against deliberately different instances — another warehouse, dialect, stack + layout, or input shape — before committing. If a design only works because of an + incidental property of the example (e.g. "the noise happened to be in a separate + schema *on this demo*"), it is curve-fitting; generalize it or state the + assumption explicitly. +- **MUST**: Prefer deriving from the system's own state over enumerating cases. + Favor an allowlist computed from declared/observed state (config, scanned + catalog, query log, the user's own inputs) over a denylist of known-bad + specifics (particular tables, schemas, tools, or vendors). A hardcoded or + hand-maintained list of external specifics is a smell: it rots and fails on the + next stack. The only acceptable static patterns are genuinely universal + invariants (e.g. DB-engine system catalogs) and ktx's own self-emitted + signatures. +- **SHOULD**: Before inventing an abstraction or hand-rolling structural logic, + search for what already exists and reuse it — the codebase's canonical + representation (a structured ref/key type) instead of a parallel string scheme, + and a mandated/available tool (e.g. `sqlglot` for SQL structure; see + [SQL and Structured Parsing](#sql-and-structured-parsing)) instead of + hand-parsing. Normalize ambiguous input to the canonical form at the boundary; + do not carry the ambiguity downstream. This is the single-source-of-truth / DRY + item from the Priority Hierarchy applied at design time. + +Before presenting a design, answer these explicitly: + +1. Am I optimizing for a goal the user actually stated, or one I assumed? +2. Does this generalize beyond the example in front of me? Name a real case where + it would break. +3. Am I enumerating known-bad cases when I could derive scope from the system's + own declared/observed state? +4. Is there an existing canonical representation or mandated tool I should reuse + instead of building or parsing my own? +5. Am I discarding the better option on a weak or misapplied constraint + (one-time vs recurring cost, "more surface area", "more work now")? + +A user question that nudges toward an alternative ("would X help?", "should I +always do Y?", "will you hardcode Z?") is a signal that a better option exists. +Investigate the implied direction and reason it through *before* defending the +original proposal — and prefer to have asked yourself the question first. + ## TypeScript Standards - Use Node 22+ and pnpm workspace commands. @@ -273,7 +337,8 @@ use `PascalCase` without the suffix. ## Telemetry -**ktx** ships anonymous PostHog telemetry. When adding commands or events: +**ktx** ships PostHog usage telemetry. Catalog telemetry events use strict +schemas. When adding commands or events: - **MUST NOT**: Add fields that carry user data — file paths, hostnames, environment values, SQL text, schema/table/column names, error messages, @@ -290,6 +355,24 @@ use `PascalCase` without the suffix. of collected data changes. Adding another event with no new field types needs no docs change. +### Error reports + +**ktx** also sends PostHog Error Tracking `$exception` events when telemetry is +enabled. This channel is separate from the strict catalog event schema and is +used only for exception diagnostics. + +`$exception` events may include stack frames, error class names, raw error +messages, cause chains, `source`, `handled`, `fatal`, runtime version fields, +OS/runtime fields, and the hashed `projectId` when known. Stack frames may +include local file paths and the local username when those appear in paths. + +`$exception` events must never intentionally include secrets, credentials, +database URLs, auth headers, raw argv, raw environment values, SQL text, +schema/table/column names as explicit properties, customer row data, user prompt +text, or raw MCP arguments. Reporters must redact call-site-provided secret +snapshots and common static credential patterns before the SDK serializes the +exception. + ## Documentation and Specs - Keep public documentation in `README.md`, package READMEs, example READMEs, @@ -318,6 +401,26 @@ use `PascalCase` without the suffix. source-code identifier, package/API name, or other literal value that must match the implementation. +### Product Category Naming + +- **MUST**: Use **context layer** as the primary public category for **ktx**. + Preferred phrase: `context layer for data agents`. +- **MUST**: Use **context engine** only as the secondary mechanism term for the + active system that builds, reconciles, validates, searches, and serves the + context layer. +- **MUST**: Keep **semantic layer** as the narrower term for executable metric + definitions, semantic sources, joins, measures, and SQL compilation. +- **MUST NOT**: Replace every `semantic layer` occurrence with `context layer`; + the semantic layer is one pillar inside the broader context layer. + +Preferred pattern: + +```md +**ktx** is an open-source context layer for data agents. Its context engine +ingests warehouse metadata, BI definitions, query history, docs, and approved +metrics, then turns them into reviewable files agents can search and execute. +``` + ### Terminology For canonical vocabulary used across docs, code, comments, CLI strings, and @@ -325,8 +428,9 @@ error messages — including the disambiguation rule for the overloaded word `source` (semantic / primary / context / source of truth) — see [`docs/terminology.md`](docs/terminology.md). Follow that file when choosing between near-synonyms (e.g. `connector` vs `adapter`, `data agent` vs -`database agent`, `fast ingest` vs `schema ingest`). Product-name rules in -this section take precedence over anything in that file when they conflict. +`database agent`, `context-source ingest` vs `source ingest`). Product-name +rules in this section take precedence over anything in that file when they +conflict. ### Updating `docs-site/` After Code Changes @@ -350,6 +454,23 @@ that do not change user-facing behavior. When you do update docs, follow the warrants docs but you are out of scope, call it out in your final summary rather than silently skipping it. +#### Monospace ligatures in `docs-site/` + +- **MUST**: Disable monospace ligatures on every surface that uses the + `var(--font-mono)` family (Geist Mono). Geist Mono fuses `--` into an + em-dash glyph that visually eats the adjacent space, so prompts like + `npx skills add Kaelio/ktx --skill ktx` render as + `Kaelio/ktx--skill ktx`. +- **MUST**: When adding a new container that renders user-visible monospace + text outside `` / `
` (e.g. a styled `
` + for a copyable prompt), verify the global ligature-off rule in + `docs-site/app/global.css` covers its selector. Either use Tailwind's + `font-mono` utility (already covered) or extend the rule to match the new + class — do not silently rely on Geist Mono's defaults. +- **SHOULD**: Prefer `` / `
` (or a `font-mono` wrapper) for any
+  string that contains CLI flags, paths, or other tokens with `--`, `->`,
+  `>=`, `!=`, `==`, `//` so ligatures never alter intent.
+
 ## LLM and Prompt Development
 
 When creating or modifying agent prompts, system prompts, tool descriptions, or
diff --git a/README.md b/README.md
index 285f92a6..d286e3f1 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,18 @@
   Documentation
   Join the ktx Slack community
   License
-  Y Combinator P25
+  Y Combinator P25
+

+ +

+ Quickstart · + CLI Reference · + Agent Setup · + Slack +

+ +

+ Built and maintained by Kaelio

--- @@ -22,11 +33,25 @@ warehouse accurately - from approved metric definitions, joinable columns, and business knowledge it builds and maintains for you. -Works with PostgreSQL, Snowflake, BigQuery, ClickHouse, MySQL, SQL Server, and -SQLite. Integrates with dbt, MetricFlow, LookML, Looker, Metabase, and Notion. +> [!NOTE] +> Run **ktx** with your own LLM API keys or a local agent sign-in — a +> **Claude Pro/Max** subscription through Claude Code, or your local Codex +> authentication. No extra usage billing from **ktx**. + +

+ + Watch the ktx launch video (1:56) + +

+ +

+ Ingestion: ktx ingests databases, BI tools, modeling code, and docs through its context engine (source connectors, context builder, reconciliation, validation) into wiki Markdown and semantic-layer YAML +

+ +

+ Serving: an agent queries ktx through MCP, which searches the wiki and semantic layer, returns approved metrics, and compiles them into read-only SQL run against the warehouse +

-Runs with your own LLM API keys or a **Claude -Pro/Max subscription - no extra usage billing from** **ktx**. ## Why ktx @@ -51,23 +76,35 @@ upkeep and don't absorb the rest of your company's knowledge. - **Serves agents at execution.** Exposes CLI and MCP tools with combined full-text and semantic search across wiki and semantic-layer entities. -Agents can run raw SQL when they need it, or compose semantic-layer queries -when they want approved metrics with reliable joins. +## How ktx compares -

- ktx ingestion flow from source systems through validation to wiki and semantic-layer outputs -

+| | General-purpose agent | Traditional semantic layer | **ktx** | +| --- | :---: | :---: | :---: | +| Builds warehouse context automatically | — | — | ✓ | +| Detects joinable columns + resolves fan/chasm traps | — | Manual | ✓ | +| Approved, reusable metric definitions | — | ✓ | ✓ | +| Absorbs wiki / Notion / team knowledge | — | — | ✓ | +| Flags contradictions across sources | — | — | ✓ | +| Ships CLI + MCP for agent execution | Partial | — | ✓ | +| Read-only by design | n/a | n/a | ✓ | -## Agent Setup +## Who is ktx for -Ask an agent such as Claude Code, Codex, Cursor, or OpenCode to install and -configure **ktx** from your project directory: +**Use ktx if you:** -```text -Follow instructions from -https://docs.kaelio.com/ktx/docs/agents-setup.md -to install and configure ktx -``` +- Want agents like Claude Code, Codex, Cursor, or OpenCode to query your + warehouse with approved metric definitions +- Have business knowledge scattered across dbt, Looker, Metabase, Notion, and + team wikis +- Need agents to reuse canonical SQL instead of inventing it on every prompt + +**Skip ktx if you:** + +- You don't have a SQL warehouse - **ktx** sits on top of one +- You only need one ad-hoc query - `psql` or a notebook will do + +Works with PostgreSQL, Snowflake, BigQuery, ClickHouse, MySQL, SQL Server, and +SQLite. Integrates with dbt, MetricFlow, LookML, Looker, Metabase, and Notion. ## Quick Start @@ -77,10 +114,10 @@ ktx setup ktx status ``` -`ktx setup` creates or resumes a local **ktx** project, configures providers and -connections, builds context, and installs agent integration. +`ktx setup` creates or resumes a local **ktx** project, configures providers +and connections, builds context, and installs agent integration. -Example `ktx status` output after setup: +Example `ktx status` after setup: ```text ktx project: /home/user/analytics @@ -93,38 +130,32 @@ ktx context built: yes Agent integration ready: yes (codex:project) ``` -## Telemetry +> [!TIP] +> Already using an agent? Ask Claude Code, Codex, Cursor, or OpenCode from +> your project directory: +> +> ```text +> Run npx skills add Kaelio/ktx --skill ktx and use the ktx skill to install +> and configure ktx in this project. +> ``` -**ktx** collects anonymous usage telemetry from interactive CLI runs to improve -setup, command reliability, and data-agent workflows. See -[Telemetry](https://docs.kaelio.com/ktx/docs/community/telemetry) for the event -catalog, privacy details, and opt-out options. +> [!IMPORTANT] +> If `ktx status` prints `ktx mcp start --project-dir ...`, run it before +> opening your agent client. -## Common Commands +## First commands | Command | Purpose | -|---------|---------| +| --- | --- | | `ktx setup` | Create, resume, or update a **ktx** project | | `ktx status` | Check project readiness | -| `ktx connection` | List configured connections | -| `ktx connection test` | Test every configured connection | -| `ktx connection test ` | Test one connection | | `ktx ingest` | Build context for every configured connection | -| `ktx ingest ` | Build context for one connection | -| `ktx ingest --text "..."` | Capture free-form notes into memory | -| `ktx ingest --file notes.md --connection-id ` | Capture a text file into memory | -| `ktx sl` | List semantic sources | | `ktx sl "revenue"` | Search semantic sources | -| `ktx sl validate --connection-id ` | Validate a semantic source | -| `ktx sl query --measure --format sql` | Compile semantic-layer SQL | -| `ktx sql --connection "select 1"` | Execute read-only SQL | -| `ktx wiki` | List local wiki pages | -| `ktx wiki "revenue definition"` | Search local wiki pages | -| `ktx mcp` | Show MCP daemon status | -| `ktx mcp start` | Start the local MCP server for agent clients | +| `ktx wiki "refund policy"` | Search local wiki pages | +| `ktx mcp start` | Start the MCP server for agent clients | -Project resolution defaults to `KTX_PROJECT_DIR`, then the nearest `ktx.yaml`, -then the current directory. Pass `--project-dir ` when scripting. +See the [CLI Reference](https://docs.kaelio.com/ktx/docs/cli-reference/ktx) +for every command, flag, and option. ## Project Layout @@ -140,45 +171,44 @@ my-project/ Commit `ktx.yaml`, `semantic-layer/`, and `wiki/`. Keep `.ktx/` local. -## Agent Usage +Project resolution defaults to `KTX_PROJECT_DIR`, then the nearest `ktx.yaml`, +then the current directory. Pass `--project-dir ` when scripting. -Install **ktx** integration for Claude Code, Claude Desktop, Codex, Cursor, -OpenCode, and generic `.agents` clients: +## FAQ -```bash -ktx setup --agents -``` +- **Does ktx send my schema or query results to a hosted service?** + No. **ktx** runs locally. The only data leaving your machine is what you + send to the LLM provider you configured. +- **Which LLM backends are supported?** + Anthropic API, Google Vertex AI, AI Gateway, the local Claude Code session + through the Claude Agent SDK, and your local Codex authentication through the + Codex SDK. See + [LLM configuration](https://docs.kaelio.com/ktx/docs/guides/llm-configuration). +- **How is ktx different from a dbt or MetricFlow semantic layer?** + **ktx** *ingests* those layers and combines them with raw-table + introspection and wiki content. Agents get one searchable surface instead + of three disconnected ones - and **ktx** flags contradictions across + sources. +- **Does ktx need a running server?** + There is no hosted service. The local MCP daemon runs on demand via + `ktx mcp start` when an agent client needs it. +- **Is my warehouse safe?** + Yes. Connections are read-only - **ktx** never writes to your database. -Pass `--target ` to install or repair one specific integration. +## Docs -A typical agent workflow combines wiki and semantic-layer search before -querying: +- [Quickstart](https://docs.kaelio.com/ktx/docs/getting-started/quickstart) +- [The Context Layer](https://docs.kaelio.com/ktx/docs/concepts/the-context-layer) +- [Building Context](https://docs.kaelio.com/ktx/docs/guides/building-context) +- [CLI Reference](https://docs.kaelio.com/ktx/docs/cli-reference/ktx) +- [Agent Quickstart](https://docs.kaelio.com/ktx/docs/ai-resources/agent-quickstart) +- [Community & Support](https://docs.kaelio.com/ktx/docs/community/support) -```bash -ktx sl "revenue" --json -ktx wiki "refund policy" --json -ktx sl query --connection-id warehouse --measure orders.revenue --format sql -``` +## Community -During setup, choose **Ask data questions with ktx MCP** for agent clients. -Choose **Ask data questions + manage ktx with CLI commands** when an operator -agent also needs pinned `ktx` admin commands. - -After setup, **ktx** prints **Required before using agents** with the exact -commands to run. If the output includes `ktx mcp start --project-dir ...`, run -it before opening your agent. Claude Desktop uses its own launcher and prints -separate skill upload steps under `.ktx/agents/claude/`. - -## Workspace layout - -| Path | Purpose | -|------|---------| -| `packages/cli` | TypeScript CLI package and published npm package source | -| `packages/cli/src/context` | Core context engine | -| `packages/cli/src/llm` | LLM and embedding providers | -| `packages/cli/src/connectors` | Database scan connectors | -| `python/ktx-sl` | Semantic-layer query planning | -| `python/ktx-daemon` | Portable compute service | +- **[Slack](https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ)** — ask questions, share what you're building, and chat with maintainers. +- **[GitHub Issues](https://github.com/Kaelio/ktx/issues)** — report bugs and request features. +- **[Contributing](https://docs.kaelio.com/ktx/docs/community/contributing)** — set up the repo, run tests, and open a PR. ## Development @@ -191,7 +221,18 @@ pnpm run build pnpm run check ``` -Use the development CLI locally: +**ktx** is a pnpm + uv workspace: + +| Path | Purpose | +| --- | --- | +| `packages/cli` | TypeScript CLI and published npm package source | +| `packages/cli/src/context` | Core context engine | +| `packages/cli/src/llm` | LLM and embedding providers | +| `packages/cli/src/connectors` | Database scan connectors | +| `python/ktx-sl` | Semantic-layer query planning | +| `python/ktx-daemon` | Portable compute service | + +Local development CLI: ```bash pnpm run setup:dev @@ -199,13 +240,6 @@ pnpm run link:dev ktx-dev --help ``` -**ktx** is a pnpm + uv workspace: - -- TypeScript packages live in `packages/*` -- CLI source lives in `packages/cli` -- Python runtime source lives in `python/ktx-sl` and `python/ktx-daemon` -- Public docs live in `docs-site/content/docs` - Useful checks: ```bash @@ -215,23 +249,28 @@ pnpm run dead-code uv run pytest -q ``` -## Docs +## Telemetry -- [Quickstart](docs-site/content/docs/getting-started/quickstart.mdx) -- [CLI Reference](docs-site/content/docs/cli-reference/ktx.mdx) -- [Building Context](docs-site/content/docs/guides/building-context.mdx) -- [Community & Support](docs-site/content/docs/community/support.mdx) -- [Contributing](docs-site/content/docs/community/contributing.mdx) - -## Community - -- **[Slack](https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ)** — ask questions, share what you're building, and chat with maintainers and other users. -- **[GitHub Issues](https://github.com/Kaelio/ktx/issues)** — report bugs and request features. -- **[Contributing guide](docs-site/content/docs/community/contributing.mdx)** — set up the repo, run tests, and open a PR. - -See [Community & Support](docs-site/content/docs/community/support.mdx) for the -full guide on where to ask what. +**ktx** collects privacy-conscious usage telemetry to understand installs and +improve setup, command reliability, and data-agent workflows. Catalog telemetry +events do not record file paths, hostnames, SQL, schema names, table names, +column names, error messages, raw environment values, or argv. Error reports use +PostHog Error Tracking and can include stack frames and raw error messages, +which may contain local file paths or the local username in those paths. +**ktx** redacts secrets, credentials, database URLs, auth headers, argv, raw +environment values, SQL text, row data, and user-typed prompt or MCP argument +text from the explicit `$exception` payload. See +[Telemetry](https://docs.kaelio.com/ktx/docs/community/telemetry) for the event +catalog and opt-out options. ## License **ktx** is licensed under the Apache License, Version 2.0. See `LICENSE`. + +## Star History + +

+ + ktx Star History Chart + +

diff --git a/assets/ktx-lockup.svg b/assets/ktx-lockup.svg index f1bcd2dd..2e45f8a6 100644 --- a/assets/ktx-lockup.svg +++ b/assets/ktx-lockup.svg @@ -19,14 +19,9 @@ - - ktx + + + + diff --git a/assets/launch-video-thumb.png b/assets/launch-video-thumb.png new file mode 100644 index 00000000..c7505732 Binary files /dev/null and b/assets/launch-video-thumb.png differ diff --git a/assets/star-history.svg b/assets/star-history.svg new file mode 100644 index 00000000..b34947d2 --- /dev/null +++ b/assets/star-history.svg @@ -0,0 +1 @@ +star-history.comMay 17May 24May 31 200400600800kaelio/ktxStar HistoryDateGitHub Stars diff --git a/docs-site/app/diagram-studio/page.tsx b/docs-site/app/diagram-studio/page.tsx new file mode 100644 index 00000000..205ebd7a --- /dev/null +++ b/docs-site/app/diagram-studio/page.tsx @@ -0,0 +1,12 @@ +import type { Metadata } from "next"; + +import { DiagramStudio } from "@/components/diagram-studio/studio"; + +export const metadata: Metadata = { + title: "Diagram studio", + robots: { index: false, follow: false }, +}; + +export default function DiagramStudioPage() { + return ; +} diff --git a/docs-site/app/global.css b/docs-site/app/global.css index a4cebc55..929e06b4 100644 --- a/docs-site/app/global.css +++ b/docs-site/app/global.css @@ -166,12 +166,16 @@ pre { } /* Disable monospace ligatures so `--flag` keeps a visible space and double - dashes don't fuse into an em-dash glyph. */ + dashes don't fuse into an em-dash glyph. Covers every monospace surface: + raw /
, the ktx-code wrapper, Tailwind's `font-mono` utility,
+   and anything that opts in via the `var(--font-mono)` family directly. */
 code,
 pre,
 pre code,
 .ktx-code,
-.ktx-code code {
+.ktx-code code,
+.font-mono,
+[style*="--font-mono"] {
   font-variant-ligatures: none !important;
   font-feature-settings: "liga" 0, "calt" 0 !important;
 }
diff --git a/docs-site/app/layout.config.tsx b/docs-site/app/layout.config.tsx
index 3245ab09..28ba6b03 100644
--- a/docs-site/app/layout.config.tsx
+++ b/docs-site/app/layout.config.tsx
@@ -5,7 +5,7 @@ import { SlackIcon } from "@/components/slack-icon";
 
 export const baseOptions: BaseLayoutProps = {
   nav: {
-    title: ,
+    title: Logo,
     transparentMode: "top",
   },
   links: [
diff --git a/docs-site/app/llms.mdx/docs/[[...slug]]/route.ts b/docs-site/app/llms.mdx/docs/[[...slug]]/route.ts
index 87dcbd42..1372d556 100644
--- a/docs-site/app/llms.mdx/docs/[[...slug]]/route.ts
+++ b/docs-site/app/llms.mdx/docs/[[...slug]]/route.ts
@@ -3,11 +3,6 @@ import {
   getLlmDocsPages,
   getPageMarkdown,
 } from "@/lib/llm-docs";
-import {
-  agentSetupSlug,
-  isAgentSetupSlug,
-  readAgentSetupMarkdown,
-} from "@/lib/agent-setup-markdown";
 
 export const dynamic = "force-static";
 
@@ -16,14 +11,6 @@ export async function GET(
   props: { params: Promise<{ slug?: string[] }> },
 ) {
   const params = await props.params;
-  if (isAgentSetupSlug(params.slug)) {
-    return new Response(await readAgentSetupMarkdown(), {
-      headers: {
-        "Content-Type": "text/markdown; charset=utf-8",
-      },
-    });
-  }
-
   const page = getLlmDocsPage(params.slug);
   if (!page) {
     return new Response("Documentation page not found.\n", {
@@ -42,8 +29,5 @@ export async function GET(
 }
 
 export function generateStaticParams() {
-  return [
-    ...getLlmDocsPages().map((page) => ({ slug: page.slug })),
-    { slug: [...agentSetupSlug] },
-  ];
+  return getLlmDocsPages().map((page) => ({ slug: page.slug }));
 }
diff --git a/docs-site/components/diagram-studio/flows.ts b/docs-site/components/diagram-studio/flows.ts
new file mode 100644
index 00000000..e63cc512
--- /dev/null
+++ b/docs-site/components/diagram-studio/flows.ts
@@ -0,0 +1,328 @@
+import { type Edge, MarkerType, type Node } from "@xyflow/react";
+
+import { C } from "./nodes";
+
+const EDGE_COLOR = "#b3bcc4";
+const MARKER_COLOR = "#9aa6ad";
+
+const labelStyle = {
+  fontFamily: "var(--font-inter), system-ui, sans-serif",
+  fontSize: 15,
+  fontWeight: 600,
+  fill: C.inkMuted,
+};
+const labelBgStyle = { fill: "#ffffff", stroke: C.chipBorder, strokeWidth: 1 };
+const labelBg = {
+  labelBgPadding: [8, 4] as [number, number],
+  labelBgBorderRadius: 6,
+  labelStyle,
+  labelBgStyle,
+};
+
+const marker = { type: MarkerType.ArrowClosed, color: MARKER_COLOR, width: 16, height: 16 };
+const edgeStyle = { stroke: EDGE_COLOR, strokeWidth: 2 };
+
+/* ============================== INGESTION =============================== */
+
+const SRC_W = 300;
+const SRC_H = 138;
+const SRC_GAP = 24;
+const srcY = (i: number) => i * (SRC_H + SRC_GAP);
+
+export const ingestionNodes: Node[] = [
+  {
+    id: "title",
+    type: "title",
+    position: { x: 0, y: -96 },
+    data: {
+      width: 560,
+      eyebrow: "1 · Ingestion",
+      title: "ktx builds your context layer",
+    },
+  },
+  {
+    id: "db",
+    type: "card",
+    position: { x: 0, y: srcY(0) },
+    data: {
+      width: SRC_W,
+      height: SRC_H,
+      accent: C.teal,
+      rows: [
+        { kind: "title", text: "Databases" },
+        { kind: "desc", text: "Schemas, keys, query history" },
+        { kind: "muted", text: "Postgres · Snowflake · BigQuery · …" },
+      ],
+      handles: [{ side: "right", type: "source", id: "out" }],
+    },
+  },
+  {
+    id: "bi",
+    type: "card",
+    position: { x: 0, y: srcY(1) },
+    data: {
+      width: SRC_W,
+      height: SRC_H,
+      accent: C.orange,
+      rows: [
+        { kind: "title", text: "BI tools" },
+        { kind: "desc", text: "Dashboards, explores, usage" },
+        { kind: "muted", text: "Metabase · Looker · …" },
+      ],
+      handles: [{ side: "right", type: "source", id: "out" }],
+    },
+  },
+  {
+    id: "model",
+    type: "card",
+    position: { x: 0, y: srcY(2) },
+    data: {
+      width: SRC_W,
+      height: SRC_H,
+      accent: C.amber,
+      rows: [
+        { kind: "title", text: "Modeling code" },
+        { kind: "desc", text: "Metrics, models, joins, entities" },
+        { kind: "muted", text: "dbt · LookML · MetricFlow · …" },
+      ],
+      handles: [{ side: "right", type: "source", id: "out" }],
+    },
+  },
+  {
+    id: "docs",
+    type: "card",
+    position: { x: 0, y: srcY(3) },
+    data: {
+      width: SRC_W,
+      height: SRC_H,
+      accent: C.emerald,
+      rows: [
+        { kind: "title", text: "Docs & notes" },
+        { kind: "desc", text: "Policies, definitions, notes" },
+        { kind: "muted", text: "Notion · any text · …" },
+      ],
+      handles: [{ side: "right", type: "source", id: "out" }],
+    },
+  },
+  {
+    id: "engine",
+    type: "engine",
+    position: { x: 420, y: 52 },
+    data: {
+      width: 380,
+      height: 520,
+      steps: [
+        { n: 1, title: "Source connectors", desc: "Read each source in its shape" },
+        { n: 2, title: "Context builder", desc: "Evidence into proposed updates" },
+        { n: 3, title: "Reconciliation", desc: "Merge with existing context" },
+        { n: 4, title: "Validation", desc: "Check references & semantics" },
+      ],
+      handles: [
+        { side: "left", type: "target", id: "in" },
+        { side: "right", type: "source", id: "out" },
+      ],
+    },
+  },
+  {
+    id: "wiki",
+    type: "card",
+    position: { x: 900, y: 66 },
+    data: {
+      width: 320,
+      height: 220,
+      accent: C.emerald,
+      rows: [
+        { kind: "mono", text: "wiki/*.md", color: C.emerald },
+        { kind: "title", text: "Wiki" },
+        { kind: "chips", items: ["free-form", "auto-maintained"] },
+        { kind: "desc", text: "Definitions, caveats, policies," },
+        { kind: "desc", text: "and notes agents can search." },
+      ],
+      handles: [{ side: "left", type: "target", id: "in" }],
+    },
+  },
+  {
+    id: "sl",
+    type: "card",
+    position: { x: 900, y: 338 },
+    data: {
+      width: 320,
+      height: 220,
+      accent: C.teal,
+      rows: [
+        { kind: "mono", text: "semantic-layer/*.yaml", color: C.teal },
+        { kind: "title", text: "Semantic layer" },
+        { kind: "chips", items: ["executable", "auto-maintained"] },
+        { kind: "desc", text: "Metrics, joins, dimensions, and" },
+        { kind: "desc", text: "filters ktx compiles into SQL." },
+      ],
+      handles: [{ side: "left", type: "target", id: "in" }],
+    },
+  },
+];
+
+const ingestEdge = (source: string, target: string): Edge => ({
+  id: `${source}-${target}`,
+  source,
+  target,
+  sourceHandle: "out",
+  targetHandle: "in",
+  type: "default",
+  style: edgeStyle,
+  markerEnd: marker,
+});
+
+export const ingestionEdges: Edge[] = [
+  ingestEdge("db", "engine"),
+  ingestEdge("bi", "engine"),
+  ingestEdge("model", "engine"),
+  ingestEdge("docs", "engine"),
+  ingestEdge("engine", "wiki"),
+  ingestEdge("engine", "sl"),
+];
+
+/* =============================== RUNTIME ================================ */
+
+export const runtimeNodes: Node[] = [
+  {
+    id: "title",
+    type: "title",
+    position: { x: 0, y: -84 },
+    data: {
+      width: 560,
+      eyebrow: "2 · Serving",
+      title: "agents query it through MCP",
+    },
+  },
+  {
+    id: "agent",
+    type: "card",
+    position: { x: 0, y: 115 },
+    data: {
+      width: 280,
+      height: 190,
+      accent: C.neutral,
+      align: "center",
+      rows: [
+        { kind: "title", text: "Your agent" },
+        { kind: "muted", text: "Claude Code · Cursor" },
+        { kind: "muted", text: "Codex · OpenCode" },
+      ],
+      handles: [
+        { side: "right", type: "source", id: "ask", top: "42%" },
+        { side: "right", type: "target", id: "answer", top: "62%" },
+      ],
+    },
+  },
+  {
+    id: "hub",
+    type: "hub",
+    position: { x: 420, y: 85 },
+    data: {
+      width: 360,
+      height: 250,
+      rows: [
+        "Search wiki + semantic layer",
+        "Return approved metrics",
+        "Compile metrics → SQL",
+      ],
+      handles: [
+        { side: "left", type: "target", id: "ask", top: "42%" },
+        { side: "left", type: "source", id: "answer", top: "62%" },
+        { side: "right", type: "source", id: "to-context", top: "30%" },
+        { side: "right", type: "source", id: "to-warehouse", top: "72%" },
+      ],
+    },
+  },
+  {
+    id: "context",
+    type: "card",
+    position: { x: 920, y: 15 },
+    data: {
+      width: 300,
+      height: 150,
+      accent: C.teal,
+      rows: [
+        { kind: "title", text: "Context layer" },
+        { kind: "mono", text: "wiki/*.md", color: C.emerald },
+        { kind: "mono", text: "semantic-layer/*.yaml", color: C.teal },
+      ],
+      handles: [{ side: "left", type: "target", id: "in" }],
+    },
+  },
+  {
+    id: "warehouse",
+    type: "card",
+    position: { x: 920, y: 255 },
+    data: {
+      width: 300,
+      height: 150,
+      accent: C.slate,
+      rows: [
+        { kind: "title", text: "Warehouse" },
+        {
+          kind: "badge",
+          text: "read-only",
+          bg: "#ecf6f8",
+          border: "#bfe3ea",
+          color: C.teal,
+        },
+        { kind: "desc", text: "Runs the compiled SQL" },
+      ],
+      handles: [{ side: "left", type: "target", id: "in" }],
+    },
+  },
+];
+
+export const runtimeEdges: Edge[] = [
+  {
+    id: "ask",
+    source: "agent",
+    sourceHandle: "ask",
+    target: "hub",
+    targetHandle: "ask",
+    type: "default",
+    label: "ask",
+    ...labelBg,
+    style: edgeStyle,
+    markerEnd: marker,
+  },
+  {
+    id: "answer",
+    source: "hub",
+    sourceHandle: "answer",
+    target: "agent",
+    targetHandle: "answer",
+    type: "default",
+    label: "answer",
+    ...labelBg,
+    style: edgeStyle,
+    markerEnd: marker,
+  },
+  {
+    id: "search",
+    source: "hub",
+    sourceHandle: "to-context",
+    target: "context",
+    targetHandle: "in",
+    type: "smoothstep",
+    label: "search + read",
+    ...labelBg,
+    style: edgeStyle,
+    markerStart: marker,
+    markerEnd: marker,
+  },
+  {
+    id: "readonly",
+    source: "hub",
+    sourceHandle: "to-warehouse",
+    target: "warehouse",
+    targetHandle: "in",
+    type: "smoothstep",
+    label: "read-only",
+    ...labelBg,
+    style: edgeStyle,
+    markerStart: marker,
+    markerEnd: marker,
+  },
+];
diff --git a/docs-site/components/diagram-studio/mascot.tsx b/docs-site/components/diagram-studio/mascot.tsx
new file mode 100644
index 00000000..467f6ee5
--- /dev/null
+++ b/docs-site/components/diagram-studio/mascot.tsx
@@ -0,0 +1,57 @@
+/**
+ * Inlined ktx mascot, ported from assets/ktx-mascot.svg.
+ *
+ * - `light` renders the dark-bodied mascot for light surfaces.
+ * - `dark` renders the cream-bodied mascot for dark surfaces (e.g. the ktx
+ *   hub panel), mirroring brand/ktx-mascot-dark.svg.
+ */
+export function KtxMascot({
+  variant = "light",
+  size = 56,
+}: {
+  variant?: "light" | "dark";
+  size?: number;
+}) {
+  const body = variant === "dark" ? "#F5F1EA" : "#1B3139";
+  const eye = variant === "dark" ? "#1B3139" : "#F5F1EA";
+  return (
+    
+      
+        
+        
+        
+      
+      
+      
+      
+      
+    
+  );
+}
diff --git a/docs-site/components/diagram-studio/nodes.tsx b/docs-site/components/diagram-studio/nodes.tsx
new file mode 100644
index 00000000..f648a905
--- /dev/null
+++ b/docs-site/components/diagram-studio/nodes.tsx
@@ -0,0 +1,493 @@
+"use client";
+
+import { Handle, Position, type Node, type NodeProps } from "@xyflow/react";
+
+import { KtxMascot } from "./mascot";
+
+/** Fixed palette mirrored from the approved SVG diagrams so the exported PNG
+ *  is theme-independent (one image that reads on light and dark GitHub). */
+export const C = {
+  ink: "#1b1b18",
+  inkSoft: "#57534e",
+  inkMuted: "#8c857f",
+  cardBorder: "#e2dfd9",
+  engineBg: "#15323a",
+  engineBorder: "#23474f",
+  cyan: "#55dced",
+  stepNum: "#06262c",
+  stepTitle: "#f3f1ec",
+  stepDesc: "#9fb6bc",
+  hubRow: "#eef4f5",
+  chipBg: "#faf9f6",
+  chipBorder: "#e7e5e4",
+  teal: "#0e7490",
+  emerald: "#059669",
+  orange: "#f97316",
+  amber: "#d97706",
+  slate: "#334155",
+  neutral: "#94a3b8",
+} as const;
+
+const DISPLAY = "var(--font-display), system-ui, sans-serif";
+const BODY = "var(--font-inter), system-ui, sans-serif";
+const MONO = "var(--font-mono), ui-monospace, monospace";
+
+const CARD_SHADOW = "0 3px 12px rgba(27, 49, 57, 0.10)";
+const ENGINE_SHADOW = "0 6px 22px rgba(2, 12, 15, 0.30)";
+
+/** ktx logo mascot size, shared by the engine and hub headers. */
+const LOGO_SIZE = 56;
+
+type HandleSpec = {
+  side: "left" | "right";
+  type: "source" | "target";
+  id: string;
+  top?: string;
+};
+
+function Handles({ specs }: { specs?: HandleSpec[] }) {
+  if (!specs) return null;
+  return (
+    <>
+      {specs.map((h) => (
+        
+      ))}
+    
+  );
+}
+
+/* ------------------------------- Card node ------------------------------- */
+
+type CardRow =
+  | { kind: "title"; text: string }
+  | { kind: "mono"; text: string; color: string }
+  | { kind: "desc"; text: string }
+  | { kind: "muted"; text: string }
+  | { kind: "chips"; items: string[] }
+  | { kind: "badge"; text: string; bg: string; border: string; color: string };
+
+type CardData = {
+  width: number;
+  height: number;
+  accent: string;
+  align?: "center";
+  rows: CardRow[];
+  handles?: HandleSpec[];
+};
+
+function gapFor(kind: CardRow["kind"], prev?: CardRow["kind"]): number {
+  if (!prev) return 0;
+  if (kind === "desc" && prev === "desc") return 3;
+  if (kind === "mono" && prev === "mono") return 2;
+  if (kind === "title") return 6;
+  return 10;
+}
+
+function CardRowView({ row }: { row: CardRow }) {
+  switch (row.kind) {
+    case "title":
+      return (
+        
+          {row.text}
+        
+      );
+    case "mono":
+      return (
+        
+          {row.text}
+        
+      );
+    case "desc":
+      return (
+        
+          {row.text}
+        
+      );
+    case "muted":
+      return (
+        
+          {row.text}
+        
+      );
+    case "chips":
+      return (
+        
+ {row.items.map((c) => ( + + {c} + + ))} +
+ ); + case "badge": + return ( + + {row.text} + + ); + } +} + +function CardNode({ data }: NodeProps>) { + const center = data.align === "center"; + return ( +
+ + + {data.rows.map((row, i) => ( +
+ +
+ ))} +
+ ); +} + +/* ------------------------------ Engine node ------------------------------ */ + +type EngineStep = { n: number; title: string; desc: string }; + +type EngineData = { + width: number; + height: number; + steps: EngineStep[]; + handles?: HandleSpec[]; +}; + +function EngineNode({ data }: NodeProps>) { + return ( +
+ + +
+ + + ktx + +
+
+ {data.steps.map((s) => ( +
+ + {s.n} + +
+ + {s.title} + + + {s.desc} + +
+
+ ))} +
+
+ ); +} + +/* -------------------------------- Hub node ------------------------------- */ + +type HubData = { + width: number; + height: number; + rows: string[]; + handles?: HandleSpec[]; +}; + +function HubNode({ data }: NodeProps>) { + return ( +
+ + +
+ + + ktx + +
+
+ {data.rows.map((r) => ( +
+ + + {r} + +
+ ))} +
+
+ ); +} + +/* ------------------------------- Title node ------------------------------ */ + +type TitleData = { width: number; eyebrow: string; title: string }; + +function TitleNode({ data }: NodeProps>) { + return ( +
+ + {data.eyebrow} + + + {data.title} + +
+ ); +} + +export const nodeTypes = { + card: CardNode, + engine: EngineNode, + hub: HubNode, + title: TitleNode, +}; diff --git a/docs-site/components/diagram-studio/studio.tsx b/docs-site/components/diagram-studio/studio.tsx new file mode 100644 index 00000000..7b96ae7b --- /dev/null +++ b/docs-site/components/diagram-studio/studio.tsx @@ -0,0 +1,242 @@ +"use client"; + +import "@xyflow/react/dist/style.css"; + +import { useCallback, useRef, useState } from "react"; +import { + Background, + BackgroundVariant, + type Edge, + getNodesBounds, + type Node, + ReactFlow, + ReactFlowProvider, + useEdgesState, + useNodesState, + useReactFlow, +} from "@xyflow/react"; +import { toPng } from "html-to-image"; + +import { + ingestionEdges, + ingestionNodes, + runtimeEdges, + runtimeNodes, +} from "./flows"; +import { nodeTypes } from "./nodes"; + +const EXPORT_PADDING = 48; +const EXPORT_PIXEL_RATIO = 2; + +function DiagramCanvasInner({ + initialNodes, + initialEdges, + fileName, + height, + dark, +}: { + initialNodes: Node[]; + initialEdges: Edge[]; + fileName: string; + height: number; + dark: boolean; +}) { + const wrapperRef = useRef(null); + const [nodes, , onNodesChange] = useNodesState(initialNodes); + const [edges, , onEdgesChange] = useEdgesState(initialEdges); + const { getNodes } = useReactFlow(); + const [busy, setBusy] = useState(false); + + const download = useCallback(async () => { + const viewport = wrapperRef.current?.querySelector( + ".react-flow__viewport", + ); + if (!viewport) return; + setBusy(true); + try { + await document.fonts.ready; + const bounds = getNodesBounds(getNodes()); + const outW = Math.ceil(bounds.width + EXPORT_PADDING * 2); + const outH = Math.ceil(bounds.height + EXPORT_PADDING * 2); + const tx = EXPORT_PADDING - bounds.x; + const ty = EXPORT_PADDING - bounds.y; + const dataUrl = await toPng(viewport, { + width: outW, + height: outH, + pixelRatio: EXPORT_PIXEL_RATIO, + // transparent background so one PNG works on light and dark GitHub + style: { + width: `${outW}px`, + height: `${outH}px`, + transform: `translate(${tx}px, ${ty}px) scale(1)`, + }, + }); + const link = document.createElement("a"); + link.download = fileName; + link.href = dataUrl; + link.click(); + } finally { + setBusy(false); + } + }, [fileName, getNodes]); + + return ( +
+
+ +
+
+ + + +
+
+ ); +} + +function btnStyle(disabled: boolean): React.CSSProperties { + return { + fontFamily: "var(--font-inter), system-ui, sans-serif", + fontSize: 13, + fontWeight: 600, + padding: "7px 14px", + borderRadius: 8, + border: "1px solid #0e7490", + background: disabled ? "#9bbdc6" : "#0e7490", + color: "#ffffff", + cursor: disabled ? "default" : "pointer", + }; +} + +function DiagramCanvas(props: { + initialNodes: Node[]; + initialEdges: Edge[]; + fileName: string; + height: number; + dark: boolean; +}) { + return ( + + + + ); +} + +export function DiagramStudio() { + const [dark, setDark] = useState(false); + return ( +
+
+

+ ktx diagram studio +

+

+ Static diagrams. Export is a transparent 2× PNG framed to the node + bounds — the dark-background toggle is only for previewing. +

+ +
+ +
+

1 · Ingestion — building the context layer

+ +
+ +
+

2 · Serving — answering agents at runtime

+ +
+
+ ); +} + +const sectionTitle: React.CSSProperties = { + fontFamily: "var(--font-display), system-ui, sans-serif", + fontSize: 18, + fontWeight: 600, + color: "#1b1b18", + marginBottom: 12, +}; diff --git a/docs-site/components/logo.tsx b/docs-site/components/logo.tsx index afc926a8..77370280 100644 --- a/docs-site/components/logo.tsx +++ b/docs-site/components/logo.tsx @@ -1,40 +1,56 @@ -export function Logo() { +"use client"; + +import Link from "next/link"; + +const brandFont = { + fontFamily: "var(--font-display), var(--font-sans), sans-serif", +} as const; + +export function Logo({ href = "/", className }: { href?: string; className?: string }) { return ( -
-
- - -
-
+
+
+ + + + + + +
+ + ktx + + + by Kaelio + +
- ktx - - - by Kaelio + Docs
- - Docs -
); } diff --git a/docs-site/components/product-runtime.tsx b/docs-site/components/product-runtime.tsx new file mode 100644 index 00000000..bfe7d64a --- /dev/null +++ b/docs-site/components/product-runtime.tsx @@ -0,0 +1,576 @@ +"use client"; + +import { + type Edge, + type EdgeProps, + getSmoothStepPath, + Handle, + MarkerType, + type Node, + type NodeProps, + Position, +} from "@xyflow/react"; + +import { FlowCanvas } from "./flow-canvas"; + +type AgentNodeData = { + title: string; + items: string[]; +}; + +type HubNodeData = { + title: string; + badge: string; + rows: string[]; +}; + +type TargetNodeData = { + accent: string; + title: string; + body: string; + rows: { text: string; color?: string; mono?: boolean }[]; + badge?: string; +}; + +type AgentNode = Node; +type HubNode = Node; +type TargetNode = Node; +type FlowNode = AgentNode | HubNode | TargetNode; + +const AGENT_W = 252; +const AGENT_H = 96; +const HUB_W = 306; +const HUB_H = 190; +const TARGET_W = 268; +const TARGET_H = 148; + +const CENTER_X = 470; +const ROW_AGENT_Y = 0; +const ROW_HUB_Y = 196; +const ROW_TARGET_Y = 488; + +const AGENT_X = CENTER_X - AGENT_W / 2; +const HUB_X = CENTER_X - HUB_W / 2; + +const TARGET_GAP_X = 38; +const TARGETS_TOTAL = TARGET_W * 2 + TARGET_GAP_X; +const TARGETS_START_X = CENTER_X - TARGETS_TOTAL / 2; +const CONTEXT_X = TARGETS_START_X; +const WAREHOUSE_X = TARGETS_START_X + TARGET_W + TARGET_GAP_X; + +const EDGE_STROKE = "#94a3b8"; +const CYCLE_STROKE = "#0e7490"; +const EMERALD = "#059669"; +const TEAL = "#0e7490"; + +const nodes: FlowNode[] = [ + { + id: "agent", + type: "agent", + position: { x: AGENT_X, y: ROW_AGENT_Y }, + data: { + title: "Your agent", + items: ["Claude Code", "Cursor", "Codex"], + }, + draggable: false, + selectable: false, + }, + { + id: "hub", + type: "hub", + position: { x: HUB_X, y: ROW_HUB_Y }, + data: { + title: "ktx", + badge: "MCP + CLI", + rows: [ + "Search wiki + semantic layer", + "Return approved metrics", + "Compile metrics → SQL", + ], + }, + draggable: false, + selectable: false, + }, + { + id: "context", + type: "target", + position: { x: CONTEXT_X, y: ROW_TARGET_Y }, + data: { + accent: TEAL, + title: "Context layer", + body: "Approved definitions agents search before they answer.", + rows: [ + { text: "wiki/*.md", color: EMERALD, mono: true }, + { text: "semantic-layer/*.yaml", color: TEAL, mono: true }, + ], + }, + draggable: false, + selectable: false, + }, + { + id: "warehouse", + type: "target", + position: { x: WAREHOUSE_X, y: ROW_TARGET_Y }, + data: { + accent: "#334155", + title: "Database", + badge: "read-only", + body: "Runs the compiled SQL. ktx never writes to it.", + rows: [], + }, + draggable: false, + selectable: false, + }, +]; + +const labelBg = { + labelBgPadding: [6, 3] as [number, number], + labelBgBorderRadius: 4, + labelStyle: { + fontSize: 13, + fontWeight: 600, + fill: "var(--color-fd-muted-foreground)", + }, + labelBgStyle: { + fill: "var(--color-fd-background)", + stroke: "var(--color-fd-border)", + strokeWidth: 1, + }, +}; + +const requestMarker = { + type: MarkerType.ArrowClosed, + color: EDGE_STROKE, + width: 16, + height: 16, +}; + +const flowEdges: Edge[] = [ + { + id: "e-ask", + source: "agent", + sourceHandle: "ask", + target: "hub", + targetHandle: "ask", + type: "straight", + label: "ask", + ...labelBg, + style: { stroke: EDGE_STROKE, strokeWidth: 1.5 }, + markerEnd: requestMarker, + }, + { + id: "e-answer", + source: "hub", + sourceHandle: "answer", + target: "agent", + targetHandle: "answer", + type: "straight", + label: "answer", + ...labelBg, + style: { stroke: EDGE_STROKE, strokeWidth: 1.5 }, + markerEnd: requestMarker, + }, + { + id: "e-search", + source: "hub", + sourceHandle: "to-context", + target: "context", + targetHandle: "in", + type: "smoothstep", + label: "search + read", + ...labelBg, + style: { stroke: CYCLE_STROKE, strokeWidth: 1.5 }, + markerStart: { type: MarkerType.ArrowClosed, color: CYCLE_STROKE, width: 14, height: 14 }, + markerEnd: { type: MarkerType.ArrowClosed, color: CYCLE_STROKE, width: 14, height: 14 }, + }, + { + id: "e-readonly", + source: "hub", + sourceHandle: "to-warehouse", + target: "warehouse", + targetHandle: "in", + type: "smoothstep", + label: "read-only", + ...labelBg, + style: { stroke: CYCLE_STROKE, strokeWidth: 1.5 }, + markerStart: { type: MarkerType.ArrowClosed, color: CYCLE_STROKE, width: 14, height: 14 }, + markerEnd: { type: MarkerType.ArrowClosed, color: CYCLE_STROKE, width: 14, height: 14 }, + }, +]; + +function AgentNodeView({ data }: NodeProps) { + return ( +
+ + +
+ + + +

+ {data.title} +

+
+
+ {data.items.map((item) => ( + + {item} + + ))} +
+
+ ); +} + +function HubNodeView({ data }: NodeProps) { + return ( +
+ + + + +
+ + k + + + {data.title} + + + {data.badge} + +
+
+ {data.rows.map((row) => ( +
+ + + {row} + +
+ ))} +
+
+ ); +} + +function TargetNodeView({ data }: NodeProps) { + return ( +
+ +
+

+ {data.title} +

+ {data.badge ? ( + + {data.badge} + + ) : null} +
+ {data.rows.length > 0 ? ( +
+ {data.rows.map((row) => ( + + {row.text} + + ))} +
+ ) : null} +

+ {data.body} +

+
+ ); +} + +/* ------------------------------- Particles ------------------------------- */ + +const PARTICLE_SPEED_PX_PER_SEC = 150; +const PARTICLE_MIN_DURATION_SEC = 5; + +type Leg = { + sx: number; + sy: number; + sPos: Position; + tx: number; + ty: number; + tPos: Position; +}; + +const AGENT_ASK_X = AGENT_X + AGENT_W * 0.35; +const AGENT_ANSWER_X = AGENT_X + AGENT_W * 0.65; +const AGENT_BOTTOM_Y = ROW_AGENT_Y + AGENT_H; +const HUB_ASK_X = HUB_X + HUB_W * 0.375; +const HUB_ANSWER_X = HUB_X + HUB_W * 0.625; +const HUB_TO_CONTEXT_X = HUB_X + HUB_W * 0.44; +const HUB_TO_WAREHOUSE_X = HUB_X + HUB_W * 0.56; +const HUB_BOTTOM_Y = ROW_HUB_Y + HUB_H; +const CONTEXT_TOP_X = CONTEXT_X + TARGET_W / 2; +const WAREHOUSE_TOP_X = WAREHOUSE_X + TARGET_W / 2; + +function buildCyclePath(spokeX: number, targetX: number): { + d: string; + length: number; +} { + const legs: Leg[] = [ + // agent → hub (ask, down) + { sx: AGENT_ASK_X, sy: AGENT_BOTTOM_Y, sPos: Position.Bottom, tx: HUB_ASK_X, ty: ROW_HUB_Y, tPos: Position.Top }, + // through the hub to its spoke handle (down, drawn behind the hub) + { sx: HUB_ASK_X, sy: ROW_HUB_Y, sPos: Position.Bottom, tx: spokeX, ty: HUB_BOTTOM_Y, tPos: Position.Top }, + // hub → target (down) + { sx: spokeX, sy: HUB_BOTTOM_Y, sPos: Position.Bottom, tx: targetX, ty: ROW_TARGET_Y, tPos: Position.Top }, + // target → hub (up) + { sx: targetX, sy: ROW_TARGET_Y, sPos: Position.Top, tx: spokeX, ty: HUB_BOTTOM_Y, tPos: Position.Bottom }, + // through the hub to its answer handle (up, drawn behind the hub) + { sx: spokeX, sy: HUB_BOTTOM_Y, sPos: Position.Top, tx: HUB_ANSWER_X, ty: ROW_HUB_Y, tPos: Position.Bottom }, + // hub → agent (answer, up) + { sx: HUB_ANSWER_X, sy: ROW_HUB_Y, sPos: Position.Top, tx: AGENT_ANSWER_X, ty: AGENT_BOTTOM_Y, tPos: Position.Bottom }, + ]; + + const segments = legs.map((leg) => { + const [segment] = getSmoothStepPath({ + sourceX: leg.sx, + sourceY: leg.sy, + sourcePosition: leg.sPos, + targetX: leg.tx, + targetY: leg.ty, + targetPosition: leg.tPos, + }); + return segment; + }); + + let d = segments[0]; + for (let i = 1; i < segments.length; i += 1) { + d += ` ${segments[i].replace(/^M/, "L")}`; + } + + const length = legs.reduce( + (sum, leg) => sum + Math.abs(leg.tx - leg.sx) + Math.abs(leg.ty - leg.sy), + 0, + ); + + return { d, length }; +} + +type ParticleEdgeData = { + d: string; + duration: number; + beginOffset: number; + color: string; +}; + +type ParticleEdge = Edge; + +function ParticleEdgeView({ id, data }: EdgeProps) { + if (!data) return null; + const pathId = `runtime-particle-path-${id}`; + return ( + <> + + + + + + + + + + + ); +} + +function makeCycleEdge( + id: string, + source: string, + spokeX: number, + targetX: number, + beginFraction: number, +): ParticleEdge { + const { d, length } = buildCyclePath(spokeX, targetX); + const duration = Math.max( + PARTICLE_MIN_DURATION_SEC, + length / PARTICLE_SPEED_PX_PER_SEC, + ); + return { + id, + source, + target: source, + type: "particle", + data: { d, duration, beginOffset: duration * beginFraction, color: CYCLE_STROKE }, + }; +} + +const particleEdges: ParticleEdge[] = [ + makeCycleEdge("p-context", "context", HUB_TO_CONTEXT_X, CONTEXT_TOP_X, 0), + makeCycleEdge("p-warehouse", "warehouse", HUB_TO_WAREHOUSE_X, WAREHOUSE_TOP_X, 0.5), +]; + +const nodeTypes = { + agent: AgentNodeView, + hub: HubNodeView, + target: TargetNodeView, +}; + +const edgeTypes = { + particle: ParticleEdgeView, +}; + +const edges = [...flowEdges, ...particleEdges]; + +export function ProductRuntime() { + return ( +
+
+

+ How serving works +

+

+ At runtime, agents reach ktx through MCP. ktx searches the context + layer, returns approved metrics, and compiles them into read-only SQL + the warehouse runs. +

+
+ +
+
+

+ Serving flow +

+

+ From an agent request to a governed answer +

+

+ The agent asks in plain language. ktx is the only thing that touches + the context layer and the warehouse, and every database connection + is read-only. +

+
+ + +
+ +
+ ); +} diff --git a/docs-site/components/semantic-layer-flow.tsx b/docs-site/components/semantic-layer-flow.tsx index 4d1ecfbd..9c6095c4 100644 --- a/docs-site/components/semantic-layer-flow.tsx +++ b/docs-site/components/semantic-layer-flow.tsx @@ -253,7 +253,7 @@ const engine: EngineNode = { }, { index: 3, - title: "Detect fan-out", + title: "Detect fanout", detail: "group measures by source, flag chasm traps", }, { diff --git a/docs-site/content/agents-setup.md b/docs-site/content/agents-setup.md deleted file mode 100644 index 4933ff10..00000000 --- a/docs-site/content/agents-setup.md +++ /dev/null @@ -1,201 +0,0 @@ -# Goal - -Set up **ktx** from scratch end-to-end as a fully autonomous, agent-driven replacement for the interactive `ktx setup` wizard. Detect the environment, install missing prerequisites, ask the user only for information you genuinely need (which connections to add, credentials), write a valid configuration, verify it works, and run a fast ingest. Keep the user updated throughout. - -# Operating principles - -- **Be autonomous.** Detect, decide, and act. Only ask the user when you need information that only they can provide: project location, which databases/sources to connect, credentials, and similar choices. -- **Stream short status updates.** Before each major phase ("Checking prerequisites…", "Installing uv…", "Configuring warehouse connection…", "Running fast ingest…") print a one-line update. Not chatty - just enough that the user can see what's happening. -- **Verify against docs, never guess.** CLI flags, config keys, and command names must come from the docs or from `ktx --help`. If something looks wrong or missing, say so explicitly. -- **Print every command you run and its exit code.** Terse, not silent. -- **Fail loudly with cause + fix.** When a command fails: capture the exact error, identify the cause, change something, retry. Never retry an unchanged command. Exceptions for *known soft-failures* are listed in Phase 4 - handle those without retrying. -- **No LLM-based ingestion in this flow.** Only `--fast` ingest. The user can run `--deep` later. -- **Platform-agnostic.** Detect the host OS first and pick the right install commands / path syntax. Anything path- or shell-specific must branch on OS. - -# Authoritative docs - -**ktx** docs are served at `https://docs.kaelio.com/ktx/`. **Start by fetching `https://docs.kaelio.com/ktx/llms.txt`** to discover the docs map. Scan it for a "troubleshooting" entry - if one exists, read it **before** running install/setup so you can apply known fixes preemptively rather than after failing. If no troubleshooting page is listed (current state of the docs), proceed. Then fetch any other `.md` pages you need (setup, ingest, status, connection types). **Never invent CLI flags or config keys** - verify against the docs or `ktx --help` / `ktx --help`. - -> **Note on the `ktx status` JSON example in the docs.** The docs page for `ktx status` shows an example shaped like `{"title": "...", "checks": [...]}`. That example is outdated. The real CLI output uses a top-level `verdict` field plus a `connections[]` array - see Phase 5 for the canonical success criteria. Trust the shape in this prompt over the docs example. - -# Workflow - -## Phase 1 - Detect environment - -Determine the host OS (e.g. via `uname -s`, `process.platform`, or `$env:OS`). Use the right install commands per OS for the rest of this flow. - -| Tool | macOS / Linux | Windows (PowerShell) | -|------|---------------|----------------------| -| `uv` | `curl -LsSf https://astral.sh/uv/install.sh \| sh` then re-source shell env | `irm https://astral.sh/uv/install.ps1 \| iex` | -| Node.js | use system / fnm / nvm - **do not** auto-install | use system / nvm-windows - **do not** auto-install | -| **ktx** CLI | `npm install -g …` (see Phase 2) | `npm install -g …` (see Phase 2) | - -If Node.js is missing, **stop and ask the user** to install it (https://nodejs.org/). Do not attempt to auto-install Node. - -## Phase 2 - Verify and install prerequisites - -Check each tool in order; install only if missing. - -1. **Node.js** - run `node --version`. Require >= 22. If missing or older, stop and instruct the user. -2. **`uv`** - run `uv --version`. If missing, run the OS-appropriate install command, then re-source the shell environment (`export PATH="$HOME/.local/bin:$PATH"` on Linux/macOS) so `uv` is on `PATH`. -3. **ktx CLI** - - - Install ktx with `npm install -g @kaelio/ktx` - - Verify with `ktx --version`. - -Print one status line per tool ("✓ uv 0.11.15 found", "Installing uv…", "✓ ktx 0.x.y installed"). - -## Phase 3 - Gather user choices - -Ask the user (grouped if your harness supports it; otherwise sequentially): - -1. **Project directory.** Default: current working directory. Confirm before continuing. -2. **LLM provider.** Default: `claude-code` with model `sonnet` (the user is already inside Claude Code; no extra API key needed). Offer `anthropic` (paste API key, stored as `env:` or `file:` ref) and `vertex` (GCP project + location) as alternatives. Skip if defaults are accepted. -3. **Embeddings backend.** Default: `sentence-transformers` (local, no API key, managed Python runtime). Offer `openai` only if the user has a key. -4. **Database connections.** Ask how many to add, then loop. For each, collect: - - Connection name (e.g. `warehouse`, `analytics`). - - Driver: one of `sqlite`, `postgres`, `mysql`, `sqlserver`, `bigquery`, `snowflake`. - - Connection URL/DSN (or service-account file for BigQuery). Accept `env:VAR_NAME` or `file:/abs/path` to avoid pasting raw secrets. - - **Heads-up for the user**: even if they paste a literal URL, **ktx** will silently relocate it into `/.ktx/secrets/-url` and rewrite `ktx.yaml` to `url: file:…` - this is correct, secure behavior and not a bug. - - Schemas / datasets to include (postgres / sqlserver / snowflake / bigquery only). - - Optional `enabled_tables` allowlist if the user wants to scope ingest to specific tables. -5. **Context sources** (dbt, Metabase, Looker, LookML, MetricFlow, Notion). Default: none. Ask only if the user mentions them. - -## Phase 4 - Configure the project - -Drive the existing wizard non-interactively (verify exact flag names with `ktx setup --help` and the docs - the automation flags are hidden from help but accepted): - -``` -ktx setup \ - --project-dir \ - --no-input --yes \ - --llm-backend --llm-model \ - [--anthropic-api-key-env ANTHROPIC_API_KEY | --anthropic-api-key-file ] \ - [--vertex-project

--vertex-location ] \ - --embedding-backend \ - [--embedding-api-key-env OPENAI_API_KEY] \ - --skip-sources \ - --database --database-connection-id --database-url \ - [--database-schema …] -``` - -Notes on the flags above: -- **Project creation is automatic with `--no-input --yes`.** When - `ktx.yaml` exists, setup resumes it. When it doesn't exist, setup creates it - at `--project-dir`. -- **`--database-connection-id` is dual-purpose.** With `--database` or - `--database-url`, it names the new connection. Without those flags, it - selects an existing connection id. -- **Configure one new database connection per setup command.** If the user - wants multiple new connections, run setup again for each connection. -- **You don't need `--skip-agents` in this flow.** The agent integration step - is opt-in: setup leaves it alone unless you pass `--agents --target - `. -- **`--skip-sources`** is correct and is the documented way to leave context sources unconfigured. - -### Known soft-failure: `ktx setup` exits 1 after a successful fast build - -When you select a configuration that only does fast ingest, `ktx setup`'s final readiness verification fails with: - -``` -ktx context build did not pass agent-readiness verification. - : deep database context has not completed. -``` - -This is **expected** and **does not mean setup failed**. Treat the exit code as a soft-failure **only if all of the following hold**: - -- The build log shows the fast ingest reached `[100%] Scan completed` for every configured connection. -- `ktx connection test ` (run next) exits 0 for every connection. -- `ktx status --json --no-input` reports `verdict: "ready"`. - -If those three conditions hold, proceed to Phase 5 without retrying setup, and **do not** switch to `--deep` to "fix" the readiness gate - deep ingest is explicitly out of scope. Mention this in the final report under "Docs / CLI gaps" so the user is aware. - -If any of those three conditions do not hold, this is a real failure - capture the error, fetch the relevant docs page, fix the cause, retry. - -After `ktx setup` writes `ktx.yaml`, edit it directly for anything flags don't cover: -- Per-connection `enabled_tables` allowlist (snake_case, under `connections..enabled_tables`). -- Any advanced settings the user requested. - -Use a YAML-aware editor (e.g. `uv run python -c "import yaml; …"`) - do not hand-edit blindly. - -## Phase 5 - Verify - -`ktx setup` already runs a fast ingest of every database connection it configures, so you do not need to re-ingest by default. For each configured connection: - -``` -ktx connection test # must exit 0 -``` - -Only re-run ingest if setup's build log did **not** reach 100% for that connection: - -``` -ktx ingest --fast --no-input -``` - -**Mutex warning on `ktx ingest`**: passing both `--yes` and `--no-input` fails with `Choose only one runtime install mode: --yes or --no-input`. Setup already installed the managed Python runtime, so pass **only `--no-input`** to `ktx ingest`. (`--yes` is only needed when an ingest invocation has to install the runtime itself, which is not the case here.) - -Then run the global health check: - -``` -ktx status --json --no-input -``` - -Success requires (canonical shape - supersedes the example in the docs): -- `verdict: "ready"` at the top of the JSON. -- Every `connections[].status === "ok"`. -- `ktx connection test ` exited 0 for every connection. - -Do **not** run `--deep` ingest in this flow - that requires LLM time and is out of scope. - -### Optional: directly probe the ktx daemon - -If the user asks for stronger verification that `sentence-transformers` is actually serving (not just that setup said "ok"), do all of: - -1. `ktx admin runtime status --json` → expect `"kind": "ready"` and `"features": [..., "local-embeddings"]`. -2. `pgrep -fa ktx-daemon` → expect a process running `ktx-daemon serve-http`. -3. `curl -sS http://127.0.0.1:/health` → expect HTTP 200 with `{"status":"healthy",…}`. -4. `curl -sS -X POST http://127.0.0.1:/embeddings/compute -H 'content-type: application/json' -d '{"text":"hello"}'` → expect `{"embedding": [...384 floats...]}`. - -Discover the port from setup's log line `Started ktx daemon: http://127.0.0.1:` or from the daemon's OpenAPI at `GET /openapi.json`. Note: the routes are `/health` and `/embeddings/compute` - not `/healthz` or `/embeddings`. - -## Phase 6 - Final report - -Print a structured report: - -``` -ktx SETUP COMPLETE - -Project: -LLM: / -Embeddings: / -Runtime: managed Python ✓ (if the ktx daemon was started) - -Connections: - - () status=ok schemas=[…] tables= - - … - -Sources: -Verdict: ready -``` - -Then **Next steps** (copy-pasteable): -1. Enrich with AI descriptions and embeddings: `ktx ingest --deep` (several minutes per connection). -2. Add more connections later by rerunning this setup or via `ktx setup --database … --database-connection-id …`. -3. Configure context sources (dbt, Metabase, Looker, LookML, MetricFlow, Notion) - see `ktx setup --help` for `--source …` flags. -4. Install agent integration: `ktx setup --agents --target ` (with optional `--global` for `claude-code`/`codex`). -5. Connect the agent / MCP: see docs at `https://docs.kaelio.com/ktx/`. - -Under **Docs / CLI gaps to flag** include any of these that applied during your run: -- `ktx setup` exits non-zero after a successful fast build (deep-readiness gate); status reports ready. -- `ktx ingest` rejects `--yes` and `--no-input` together; docs don't note the conflict. -- `ktx status --json` real shape (`verdict`, `connections[]`) doesn't match the example in the docs page. -- The pasted DB URL was moved to `.ktx/secrets/-url` automatically. - -End with a single line: `RESULT: PASS` or `RESULT: FAIL - `. - -# Operating rules (recap) - -- Print every command you run and its exit code. Status updates may be terse, but never silent. -- On failure: capture the error, fetch the relevant docs page, fix the cause, retry. Never retry an unchanged command. -- Known soft-failures (listed in Phase 4 and Phase 5) are not real failures - handle them as documented; do not retry or escalate. -- If you find a docs/CLI gap ("docs say X but CLI does Y"), call it out in the final report. -- Never commit credentials - **ktx** accepts `env:` and `file:` references; prefer those. **ktx** will also auto-relocate literal URLs into `.ktx/secrets/`, but that does not protect anyone who pasted the URL into chat history. diff --git a/docs-site/content/docs/ai-resources/prompt-recipes.mdx b/docs-site/content/docs/ai-resources/prompt-recipes.mdx index c2a9f282..9ba8e3b8 100644 --- a/docs-site/content/docs/ai-resources/prompt-recipes.mdx +++ b/docs-site/content/docs/ai-resources/prompt-recipes.mdx @@ -14,7 +14,8 @@ Read https://docs.kaelio.com/ktx/llms.txt first. Then fetch only the ktx Markdow ## Set up a project ```text -Set up ktx in this repository. Start by reading /docs/ai-resources/agent-quickstart.md and /docs/getting-started/quickstart.md. Install the published CLI with npm; use pnpm only when working from a ktx source checkout. After setup, run ktx status and summarize which steps are complete, which files changed, and what still needs credentials or user input. +Run npx skills add Kaelio/ktx --skill ktx and use the ktx skill to install +and configure ktx in this project. ``` ## Find a command diff --git a/docs-site/content/docs/cli-reference/ktx-completion.mdx b/docs-site/content/docs/cli-reference/ktx-completion.mdx new file mode 100644 index 00000000..94f1c383 --- /dev/null +++ b/docs-site/content/docs/cli-reference/ktx-completion.mdx @@ -0,0 +1,86 @@ +--- +title: "ktx completion" +description: "Print a shell completion script for tab completion." +--- + +Print a shell completion script for **ktx**. Once installed, pressing Tab +completes commands, subcommands, and flags, and - inside a **ktx** project - the +names of things that already exist: semantic-layer source names for +`ktx sl read` and `ktx sl validate`, wiki page keys for `ktx wiki read`, and +configured connection ids for `ktx connection test`, `ktx ingest`, and +`ktx sql`. This saves you from remembering exact source, page, or connection +names. + +## Command signature + +```bash +ktx completion +``` + +`` must be `zsh` or `bash`. The command writes the script to stdout; it +does not modify any files. Enable completion by evaluating the script in your +shell startup file. + +## Installation + +Add the matching line to your shell startup file, then restart your shell (or +`source` the file). `ktx` must be on your `PATH`. + +```bash +# zsh — add to ~/.zshrc +eval "$(ktx completion zsh)" +``` + +```bash +# bash — add to ~/.bashrc +eval "$(ktx completion bash)" +``` + +To try it for the current session only, run the same `eval` line directly in +your terminal. + +## What gets completed + +| Position | Completions | +|----------|-------------| +| `ktx ` | Top-level commands (`setup`, `sl`, `wiki`, `ingest`, …) | +| `ktx sl ` | The `read` / `validate` / `query` subcommands | +| `ktx sl read ` | Existing semantic-layer source names | +| `ktx sl validate ` | Existing semantic-layer source names | +| `ktx wiki ` | The `read` subcommand | +| `ktx wiki read ` | Existing wiki page keys | +| `ktx connection test ` | Configured connection ids | +| `ktx ingest ` | Configured connection ids | +| `ktx sql --connection ` | Configured connection ids | +| `ktx completion ` | `zsh` or `bash` | +| `ktx --` | The command's flags and inherited global flags | +| `ktx sl --output ` | An option's allowed values (here `pretty`, `plain`, `json`) | +| `ktx sl --connection-id ` | Configured connection ids | + +Source names, wiki page keys, and connection ids are read from the **ktx** +project resolved from your current directory (or `--project-dir` / +`KTX_PROJECT_DIR`). Outside a **ktx** project, completion still suggests +commands and flags but no project entities. Bare `ktx sl ` and +`ktx wiki ` complete subcommands instead of entity names because their +positional arguments are free-text search queries. + +## Examples + +```bash +# Print the zsh completion script +ktx completion zsh + +# Print the bash completion script +ktx completion bash + +# Install for zsh +echo 'eval "$(ktx completion zsh)"' >> ~/.zshrc +``` + +## Common errors + +| Error | Cause | Recovery | +|-------|-------|----------| +| `error: command-argument value '' is invalid for argument 'shell'. Allowed choices are zsh, bash.` | A shell other than `zsh` or `bash` was requested | Re-run with `ktx completion zsh` or `ktx completion bash` | +| Tab completion does nothing | The script was not evaluated, or `ktx` is not on `PATH` | Confirm the `eval` line is in your startup file, restart the shell, and verify `ktx --version` runs | +| Source, page, or connection names are missing | The current directory is not inside a **ktx** project | Run from the project directory, or pass `--project-dir`, or set `KTX_PROJECT_DIR` | diff --git a/docs-site/content/docs/cli-reference/ktx-connection.mdx b/docs-site/content/docs/cli-reference/ktx-connection.mdx index 36185d68..9d78bdd8 100644 --- a/docs-site/content/docs/cli-reference/ktx-connection.mdx +++ b/docs-site/content/docs/cli-reference/ktx-connection.mdx @@ -104,6 +104,6 @@ configured connection and exit non-zero if any probe fails. | Error | Cause | Recovery | |-------|-------|----------| | No connections configured | The project has no entries under `connections` | Run `ktx setup` and add a database or context-source connection | -| Connection test fails | Credentials, network access, database, warehouse, or schema is invalid | Verify the same URL with the database's native client, then rerun `ktx setup` and reconfigure the connection | -| Mapping validation fails during setup | BI database mappings do not point at valid warehouse connections | Rerun `ktx setup` and update the context-source mapping selections | +| Connection test fails | Credentials, network access, database, warehouse, or schema is invalid | Use the setup recovery menu to retry or re-enter details; if it still fails, verify the same URL with the database's native client | +| Mapping validation fails during setup | BI database mappings do not point at valid warehouse connections | Use the setup recovery menu to retry validation or re-enter mapping selections; rerun `ktx setup` if you already exited | | Notion page picker cannot run | The terminal is non-interactive or Notion discovery failed | Rerun interactive `ktx setup`, or use non-interactive setup flags with explicit root page ids | diff --git a/docs-site/content/docs/cli-reference/ktx-ingest.mdx b/docs-site/content/docs/cli-reference/ktx-ingest.mdx index d4e06881..ab3d231d 100644 --- a/docs-site/content/docs/cli-reference/ktx-ingest.mdx +++ b/docs-site/content/docs/cli-reference/ktx-ingest.mdx @@ -5,9 +5,11 @@ description: "Build or refresh ktx context, or capture text into ktx memory." `ktx ingest` builds or refreshes **ktx** context from configured connections, and can also capture free-form text into **ktx** memory. Database connections build -schema context. Context-source connections ingest metadata from tools such as -dbt, Looker, Metabase, MetricFlow, LookML, and Notion. Pass `--text` or -`--file` to capture inline text or text files into memory instead. +enriched context — schema plus AI-generated descriptions, embeddings, and +relationship evidence — and require a configured model and embeddings. +Context-source connections ingest metadata from tools such as dbt, Looker, +Metabase, MetricFlow, LookML, and Notion. Pass `--text` or `--file` to capture +inline text or text files into memory instead. ## Command signature @@ -29,8 +31,6 @@ connection is selected. | Flag | Description | Default | |------|-------------|---------| | `--all` | Ingest all configured connections (same as bare invocation) | `false` | -| `--fast` | Use deterministic fast database ingest | Stored connection default, or `fast` | -| `--deep` | Use deep database ingest with AI-generated descriptions, embeddings, and relationship evidence | Stored connection default, or `fast` | | `--query-history` | Include database query-history usage patterns | Stored connection default | | `--no-query-history` | Skip database query-history usage patterns for this run | Stored connection default | | `--query-history-window-days ` | BigQuery/Snowflake query-history lookback window for this run | Stored connection default | @@ -44,12 +44,12 @@ connection is selected. | `--yes` | Install required managed runtime features without prompting | `false` | | `--no-input` | Disable interactive terminal input | - | -`--fast` and `--deep` are mutually exclusive. Depth flags apply only to -database connections. Query-history flags apply only to database connections +Database ingest always builds enriched context and requires a configured model +and embeddings (run `ktx setup`); connections without that configuration fail +before any work starts. Query-history flags apply only to database connections that support query history. The window flag applies to BigQuery and Snowflake; Postgres reads the current `pg_stat_statements` aggregate data instead of a -time-windowed history table. Query-history ingest runs after fast ingest and -requires deep ingest readiness. +time-windowed history table. Query-history ingest runs after the schema scan. When more than one connection is selected, database ingest runs first, then context-source ingest and memory updates run for context-source connections. @@ -72,14 +72,8 @@ ktx ingest # Build one database or context-source connection ktx ingest warehouse -# Force deterministic fast database ingest -ktx ingest warehouse --fast - -# Force deep database ingest with AI enrichment -ktx ingest warehouse --deep - # Include query-history usage patterns -ktx ingest warehouse --deep --query-history +ktx ingest warehouse --query-history # Set the lookback window for BigQuery or Snowflake query history ktx ingest warehouse --query-history-window-days 30 @@ -149,13 +143,51 @@ verbosity: KTX_INGEST_TRACE_LEVEL=trace ktx ingest metabase ``` +### Profiling a slow ingest + +Each timed phase and work unit records a `durationMs` in the trace, and each +agent loop records its step count and token usage. To see where wall-clock time +went, enable profiling and **ktx** prints a rolled-up breakdown to stderr at the +end of the run. There are two ways to turn it on, and two output formats. + +Turn it on per run with the `KTX_PROFILE_INGEST` environment variable, or +persistently with `ingest.profile` in `ktx.yaml` (useful for CI or while +iterating on a slow source): + +```bash +KTX_PROFILE_INGEST=1 ktx ingest metabase # human-readable table +KTX_PROFILE_INGEST=json ktx ingest metabase # raw JSON for coding agents +``` + +```yaml +ingest: + profile: true # human table; use "json" for the machine-readable form +``` + +Both formats report total wall time, time per phase, and the slowest work units, +splitting each work unit's agent-loop time into model time versus tool-execution +time. The `json` form emits the full structured profile (raw milliseconds and +token counts, stable keys) plus a `summary.headline` one-line diagnosis, so a +coding agent can parse it directly instead of scraping the table. If both the env +var and the config request profiling, `json` wins. Example headline: + +```text +Slowest phase: reconciliation (2m 05s, 48% of wall time). 2 work units (1 failed), ~88% model generation vs ~12% tools. +``` + +Work units run serially by default (`ingest.workUnits.maxConcurrency` is `1`); +raise it in `ktx.yaml` if the profile shows the run is bound by serialized +work-unit agent loops. If the provider reports an LLM rate limit, **ktx** shows +a transient wait message and temporarily reduces effective work-unit concurrency +according to `ingest.rateLimit`. + ## Common errors | Error | Cause | Recovery | |-------|-------|----------| | Connection not configured | The connection id is not present in `ktx.yaml` | Add the connection with `ktx setup` or update `ktx.yaml` | -| Deep readiness is missing | `--deep` or query history needs model, embedding, and scan-enrichment configuration | Run `ktx setup` or rerun with `--fast` | -| Query history is unsupported | The selected database driver does not support query history | Run fast ingest without query-history flags | +| Enrichment is not configured | Database ingest needs a model, embeddings, and scan-enrichment configuration | Run `ktx setup` to configure a model and embeddings | +| Query history is unsupported | The selected database driver does not support query history | Run ingest without query-history flags | | Python runtime is missing | The selected ingest target needs runtime-backed SQL analysis or source parsing | Accept the interactive prompt, rerun with `--yes`, or run the suggested `ktx admin runtime install` command | -| Context-source options were ignored | Depth and query-history flags were supplied for a context-source connection | Omit database-only flags when ingesting context-source connections | +| Context-source options were ignored | Query-history flags were supplied for a context-source connection | Omit database-only flags when ingesting context-source connections | | Text ingest stops early | `--fail-fast` was used and one item failed | Fix the failed item or rerun without `--fail-fast` to collect all failures | diff --git a/docs-site/content/docs/cli-reference/ktx-setup.mdx b/docs-site/content/docs/cli-reference/ktx-setup.mdx index a52a3eba..0e6cb57c 100644 --- a/docs-site/content/docs/cli-reference/ktx-setup.mdx +++ b/docs-site/content/docs/cli-reference/ktx-setup.mdx @@ -51,8 +51,9 @@ prompts. | Flag | Description | |------|-------------| -| `--llm-backend ` | LLM backend: `anthropic`, `vertex`, or `claude-code` | +| `--llm-backend ` | LLM backend: `anthropic`, `vertex`, `claude-code`, or `codex` | | `--llm-backend claude-code` | Use the local Claude Code session for **ktx** LLM calls | +| `--llm-backend codex` | Use local Codex authentication for **ktx** LLM calls | | `--llm-model ` | LLM model ID or backend model alias to validate and save | | `--anthropic-api-key-env ` | Environment variable containing the Anthropic API key | | `--anthropic-api-key-file ` | File containing the Anthropic API key | @@ -62,9 +63,14 @@ prompts. Choose only one Anthropic credential source. Anthropic credential flags are only valid with the Anthropic backend; Vertex flags are only valid with the Vertex -backend. The `claude-code` backend uses local Claude Code authentication instead +backend. The `claude-code` and `codex` backends use local authentication instead of Anthropic API key or Vertex flags. For Claude Code, `--llm-model` accepts -`sonnet`, `opus`, `haiku`, or a full Claude model ID. +`sonnet`, `opus`, `haiku`, or a full Claude model ID. For Codex, `--llm-model` +accepts `codex`, `default`, or a `gpt-*` / `codex-*` model ID such as +`gpt-5.5`; any other value is rejected before the auth probe. Run `codex` to +see the models available to your login, and pick a `gpt-*` / `codex-*` id from +that list. Note that `*-codex` API-billing model IDs (for example +`gpt-5.3-codex`) are not available to ChatGPT-subscription logins. ### Embeddings @@ -131,11 +137,34 @@ BigQuery; and `databases` for ClickHouse. Query history setup is supported for Postgres, BigQuery, and Snowflake. The window flag applies to BigQuery and Snowflake; Postgres reads the current `pg_stat_statements` aggregate data instead of a time-windowed history table. -Enabling query history makes deep ingest readiness matter for later -`ktx ingest` runs. +Later `ktx ingest` runs build enriched context and need a configured model and +embeddings, including when query history is enabled. + +When query history is enabled for PostgreSQL, Snowflake, or BigQuery, +`ktx setup` runs a non-blocking readiness probe after the connection test +passes. A failed probe still writes setup changes, prints the warehouse-specific +grant or extension remediation, and skips query-history processing until you +fix the prerequisite. If the later schema-context build also fails, interactive +setup offers **Disable query history and retry** so you can finish database +setup with `connections..context.queryHistory.enabled: false`. + +After the schema scan completes, setup can derive query-history service-account +filters from in-scope history. If **ktx** finds clear operational roles, it +prints each proposed exclusion with a reason and writes +`connections..context.queryHistory.filters.serviceAccounts` only when you +apply the proposal. In non-interactive setup with `--yes`, the proposal is +applied automatically. Existing `serviceAccounts` blocks are never overwritten. + +For BigQuery, the remediation tells you to grant `roles/bigquery.resourceViewer` +on the BigQuery project, or grant a custom role that contains +`bigquery.jobs.listAll`. ### Context Sources +In interactive setup, after you configure a database, choose +**Skip context sources** to leave optional context-source setup complete with no +sources. This is equivalent to passing `--skip-sources` in scripted setup. + | Flag | Description | |------|-------------| | `--source ` | Context-source connector type: `dbt`, `metricflow`, `metabase`, `looker`, `lookml`, or `notion` | @@ -144,9 +173,9 @@ Enabling query history makes deep ingest readiness matter for later | `--source-git-url ` | Git URL for dbt, MetricFlow, or LookML | | `--source-branch ` | Git branch for context-source setup | | `--source-subpath ` | Repo subpath for context-source setup | -| `--source-auth-token-ref ` | `env:` or `file:` credential reference for source repo auth | +| `--source-auth-token-ref ` | `env:` or `file:` credential reference for source repo auth or Notion integration token | | `--source-url ` | Source service URL for Metabase or Looker | -| `--source-api-key-ref ` | `env:` or `file:` API key reference for Metabase or Notion | +| `--source-api-key-ref ` | `env:` or `file:` API key reference for Metabase | | `--source-client-id ` | Looker client id | | `--source-client-secret-ref ` | `env:` or `file:` Looker client secret reference | | `--source-warehouse-connection-id ` | Warehouse connection id used for context-source mapping | @@ -175,6 +204,17 @@ ktx setup \ --llm-backend claude-code \ --llm-model opus +# Configure **ktx** to use local Codex authentication for LLM work +ktx setup --llm-backend codex --llm-model gpt-5.5 --no-input +``` + +When you choose `--llm-backend codex`, setup prints a warning if the public +Codex SDK and CLI surface cannot prove full Claude-Code-style isolation. The +backend restricts **ktx** runtime MCP tools to each run, but Codex may still +load user Codex config and built-in command execution or read-only file +capabilities. + +```bash # Script a Postgres connection that reads its URL from the environment ktx setup \ --project-dir ./analytics \ @@ -205,6 +245,14 @@ ktx setup \ --source-warehouse-connection-id warehouse \ --metabase-database-id 1 +# Add a Notion source that crawls selected root pages +ktx setup \ + --source notion \ + --source-connection-id notion-main \ + --source-auth-token-ref env:NOTION_TOKEN \ + --notion-crawl-mode selected_roots \ + --notion-root-page-id abc123def456 + # Install project-scoped agent integration for Codex ktx setup --agents --target codex ``` diff --git a/docs-site/content/docs/cli-reference/ktx-sl.mdx b/docs-site/content/docs/cli-reference/ktx-sl.mdx index 2dfba7ab..9e957d4e 100644 --- a/docs-site/content/docs/cli-reference/ktx-sl.mdx +++ b/docs-site/content/docs/cli-reference/ktx-sl.mdx @@ -11,13 +11,16 @@ the vocabulary agents use to generate correct SQL. ```bash ktx sl [options] [query...] # list (bare) or search (with query) -ktx sl validate [options] +ktx sl read +ktx sl validate ktx sl query [options] ``` - Bare `ktx sl` lists semantic sources. -- `ktx sl ` searches semantic sources (multi-word queries are - joined with a space). +- `ktx sl ` searches semantic sources. Multi-word queries are joined + with a space. +- `ktx sl read ` prints the YAML for one source. Add + `--connection-id` only when the source name exists in multiple connections. - `ktx sl validate` and `ktx sl query` remain as explicit subcommands. ## Subcommands @@ -26,6 +29,7 @@ ktx sl query [options] |-----------|-------------| | (none, no query) | List semantic sources | | (none, with query) | Search semantic sources | +| `read ` | Print the YAML for one semantic source | | `validate ` | Validate a semantic source against the database schema | | `query` | Compile or execute a semantic query | @@ -40,17 +44,23 @@ ktx sl query [options] | `--output ` | Output mode: `pretty` (default in TTY), `plain` (TSV), or `json` | `pretty` | | `--json` | Shortcut for `--output=json` (overrides `--output`) | `false` | +### `sl read` + +| Flag | Description | Default | +|------|-------------|---------| +| `--connection-id ` | Optional **ktx** connection id for disambiguation | - | + ### `sl validate` | Flag | Description | Default | |------|-------------|---------| -| `--connection-id ` | **ktx** connection id (required) | - | +| `--connection-id ` | Optional **ktx** connection id for disambiguation | - | ### `sl query` | Flag | Description | Default | |------|-------------|---------| -| `--connection-id ` | **ktx** connection id | - | +| `--connection-id ` | Required **ktx** connection id | - | | `--query-file ` | JSON semantic query file | - | | `--measure ` | Measure to query; repeatable (at least one required) | - | | `--dimension ` | Dimension to include; repeatable | - | @@ -65,8 +75,9 @@ ktx sl query [options] | `--no-input` | Disable interactive managed runtime installation | - | | `--max-rows ` | Maximum rows to return when executing | - | -`sl query` requires at least one `--measure` unless `--query-file` is set. -`--query-file` should point to a JSON semantic query object. +`sl query` requires `--connection-id` and at least one `--measure` unless +`--query-file` is set. `--query-file` must point to a JSON semantic query +object. ## Examples @@ -83,7 +94,16 @@ ktx sl --json # Search sources as JSON ktx sl "revenue" --json -# Validate a source against the live schema +# Print the YAML for a source name that is unique across connections +ktx sl read orders + +# Print the YAML for a source name that exists in multiple connections +ktx sl --connection-id my-warehouse read orders + +# Validate a source name that is unique across connections +ktx sl validate orders + +# Validate a source name that exists in multiple connections ktx sl validate orders --connection-id my-warehouse # Compile a query and view the generated SQL @@ -144,6 +164,12 @@ shows `#1`, `#2`, and later rank badges for the displayed results. Plain and JSON output keep the raw `score` value, which is a ranking score rather than a percentage. +`ktx sl read ` prints the source YAML directly to stdout when the +source name is unique across connections. If the name exists in multiple +connections, rerun the command with `--connection-id `. The command does +not wrap output in pretty, plain, or JSON formatting, so it can be piped to +other tools. + ```json { "sql": "SELECT orders.status, SUM(orders.total_amount) AS total_revenue FROM public.orders GROUP BY orders.status", @@ -160,7 +186,8 @@ percentage. | Error | Cause | Recovery | |-------|-------|----------| -| Source not found | Source name or connection id is wrong | Run `ktx sl --json` and retry with an exact source name and connection id | +| Source not found | Source name or connection id is wrong | Run `ktx sl ` or `ktx sl --connection-id ` to find the exact source name, then retry `ktx sl read ` or `ktx sl validate ` | +| Source name is ambiguous | The same source name exists in multiple connections | Rerun with `--connection-id ` from the error message | | Validation fails | YAML references missing columns, invalid joins, or invalid SQL expressions | Fix the source YAML and rerun `ktx sl validate` | | Query compile fails | Measure, dimension, filter, or segment name is invalid | Search sources with `ktx sl `, inspect the source YAML in your project files, then retry using declared fields | | Execution returns too many rows | `--max-rows` is missing or too high | Add `--max-rows` with a bounded value before executing | diff --git a/docs-site/content/docs/cli-reference/ktx-status.mdx b/docs-site/content/docs/cli-reference/ktx-status.mdx index c86c12e0..66e4964c 100644 --- a/docs-site/content/docs/cli-reference/ktx-status.mdx +++ b/docs-site/content/docs/cli-reference/ktx-status.mdx @@ -21,7 +21,7 @@ ktx status [options] | `--json` | Print JSON output | `false` | | `-v`, `--verbose` | Show every check, including passing ones | `false` | | `--validate` | Only validate the `ktx.yaml` schema; skip readiness checks | `false` | -| `--fast` | Skip checks that require external communication (Postgres query-history probe, Claude Code auth probe) | `false` | +| `--fast` | Skip checks that require external communication (query-history readiness probes, Claude Code auth probe, and Codex auth probe) | `false` | | `--no-input` | Disable interactive terminal input | - | ## Examples @@ -39,7 +39,7 @@ ktx status --verbose # Validate ktx.yaml without running readiness checks ktx status --validate -# Skip slow probes (Postgres pg_stat_statements, Claude Code auth) +# Skip slow probes (query-history readiness, Claude Code auth, Codex auth) ktx status --fast # Check a project from another directory @@ -57,6 +57,16 @@ flow, then rerun `ktx status`. Use `--fast` to skip this probe (useful in CI or offline contexts); skipped checks render as `-` and carry `"status": "skipped"` in JSON output. +For `llm.provider.backend: codex`, `ktx status` runs a minimal non-interactive +Codex request. If the probe fails, authenticate Codex locally with the Codex CLI +and verify the Codex CLI installation. + +When `llm.provider.backend: codex` is configured, `ktx status` also prints a +warning when the installed public Codex SDK and CLI surface cannot prove full +Claude-Code-style isolation. The warning does not block authenticated Codex +usage, but it marks the project status as partial so you can make an explicit +runtime-isolation decision. + A `Local data` section summarises what the project has accumulated locally: ingest run counts, last completed timestamp per connection, knowledge page counts by scope, semantic-layer source and dictionary value counts, and the diff --git a/docs-site/content/docs/cli-reference/ktx-wiki.mdx b/docs-site/content/docs/cli-reference/ktx-wiki.mdx index 2d52d5af..7887a463 100644 --- a/docs-site/content/docs/cli-reference/ktx-wiki.mdx +++ b/docs-site/content/docs/cli-reference/ktx-wiki.mdx @@ -1,21 +1,24 @@ --- title: "ktx wiki" -description: "List or search wiki pages." +description: "List, search, or read wiki pages." --- -List and search wiki pages in your **ktx** project. Wiki pages are Markdown -documents that capture business definitions, rules, and gotchas. Agents search -them for context when answering questions about your data. +List, search, and read wiki pages in your **ktx** project. Wiki pages are +Markdown documents that capture business definitions, rules, and gotchas. +Agents search them for context when answering questions about your data. ## Command signature ```bash -ktx wiki [options] [query...] +ktx wiki [options] [query...] # list (bare) or search (with query) +ktx wiki read ``` - Bare `ktx wiki` lists local wiki pages. -- `ktx wiki ` searches local wiki pages (multi-word queries are - joined with a space). +- `ktx wiki ` searches local wiki pages. Multi-word queries are + joined with a space. +- `ktx wiki read ` prints the whole Markdown file for one wiki page, + including YAML frontmatter. Edit the Markdown files under `wiki/` directly, or ingest source content with `ktx ingest`, when you need to add or update wiki knowledge. @@ -50,6 +53,9 @@ ktx wiki "monthly recurring revenue" # Search wiki pages as JSON ktx wiki "monthly recurring revenue" --json --limit 10 +# Print the exact Markdown file for a known page key +ktx wiki read revenue-definitions + # Print search results as TSV ktx wiki "monthly recurring revenue" --output plain @@ -62,8 +68,10 @@ ktx --debug wiki "monthly recurring revenue" --json Wiki commands print clack-style pretty output in a TTY and TSV-style plain output when requested. JSON output wraps the items with a command metadata envelope. Search results include `matchReasons` and `lanes` metadata so you can -see whether lexical, token, or semantic search contributed to the ranking. Open -the matching Markdown files directly when you need the full page contents. +see whether lexical, token, or semantic search contributed to the ranking. Use +`ktx wiki read ` when you need the full page contents. Read output is the +exact Markdown file stored on disk, including YAML frontmatter, and is not +wrapped in pretty, plain, or JSON formatting. Pretty search output shows `#1`, `#2`, and later rank badges for the displayed results. Plain and JSON output keep the raw `score` value, which is a ranking score rather than a percentage. @@ -121,4 +129,4 @@ stays machine-readable: | Error | Cause | Recovery | |-------|-------|----------| | Search returns no results | The query terms do not match summaries, tags, or content, and the semantic lane is unavailable or has no positive matches | Run with `--debug`, check the semantic lane status, retry with business synonyms, then create a page if the knowledge is missing | -| A page is missing | No Markdown file exists for that business context | Add a file under `wiki/` or run `ktx ingest ` | +| A page is missing | No Markdown file exists for that business context or `ktx wiki read ` used the wrong key | Run `ktx wiki ` to find the page key, then retry `ktx wiki read ` | diff --git a/docs-site/content/docs/cli-reference/ktx.mdx b/docs-site/content/docs/cli-reference/ktx.mdx index 010100d8..ebdeb1c6 100644 --- a/docs-site/content/docs/cli-reference/ktx.mdx +++ b/docs-site/content/docs/cli-reference/ktx.mdx @@ -36,9 +36,11 @@ ktx wiki list search + read sl list search + read validate query sql @@ -57,6 +59,7 @@ ktx stop status reindex + completion ``` The public context-build entrypoint is `ktx ingest [connectionId]` or @@ -71,6 +74,44 @@ The public context-build entrypoint is `ktx ingest [connectionId]` or | `-v`, `--version` | Show the CLI package name and version. | | `-h`, `--help` | Show help for the current command. | +## Update notices + +> **Note:** The update notifier writes only to stderr and keeps command stdout +> unchanged. + +When a newer package is available on your installed release channel, `ktx` +prints a short notice after the command finishes: + +```text +↑ Update available: ktx 0.9.0 → 0.10.0 + npm i -g @kaelio/ktx +``` + +Stable installs compare against the npm `latest` dist-tag. +Release-candidate installs compare against the `next` dist-tag and show: + +```text +npm i -g @kaelio/ktx@next +``` + +The check is skipped for JSON output, CI, non-TTY stdout, and hidden completion +commands. To opt out explicitly, set any of these environment variables: + +```bash +KTX_NO_UPDATE_CHECK=1 +NO_UPDATE_NOTIFIER=1 +DO_NOT_TRACK=1 +``` + +The `ktx` CLI prints one npm command because globally installed binaries don't +expose a reliable runtime package-manager signal. If you prefer another global +package manager, use the equivalent command: + +```bash +pnpm add -g @kaelio/ktx +yarn global add @kaelio/ktx +``` + ## Project resolution Most commands are project-aware. Pass `--project-dir ` when scripting or @@ -97,6 +138,10 @@ ktx ingest ktx sl "revenue" ktx wiki "revenue recognition" +# Print a known wiki page or semantic source +ktx wiki read revenue-definitions +ktx sl --connection-id warehouse read orders + # Execute read-only SQL ktx sql --connection warehouse "select count(*) from public.orders" diff --git a/docs-site/content/docs/cli-reference/meta.json b/docs-site/content/docs/cli-reference/meta.json index 49eb8ba7..2902f2c6 100644 --- a/docs-site/content/docs/cli-reference/meta.json +++ b/docs-site/content/docs/cli-reference/meta.json @@ -11,6 +11,7 @@ "ktx-wiki", "ktx-status", "ktx-mcp", - "ktx-admin" + "ktx-admin", + "ktx-completion" ] } diff --git a/docs-site/content/docs/community/telemetry.mdx b/docs-site/content/docs/community/telemetry.mdx index 9c22b432..78bdb3e5 100644 --- a/docs-site/content/docs/community/telemetry.mdx +++ b/docs-site/content/docs/community/telemetry.mdx @@ -1,12 +1,15 @@ --- title: Telemetry -description: Understand what anonymous usage telemetry ktx collects and how to opt out. +description: Understand what usage telemetry ktx collects and how to opt out. --- -**ktx** collects anonymous, aggregated usage telemetry from interactive CLI -runs so maintainers can see which commands work, where setup fails, and which -parts of the data-agent workflow need improvement. Telemetry is opt-out and -disabled automatically in CI and non-interactive runs. +**ktx** collects aggregated usage telemetry so maintainers can see +which commands work, where setup fails, and which parts of the data-agent +workflow need improvement. Telemetry is opt-out: it turns on the first time you +run **ktx** in any way — an interactive command, a script, or an +agent-launched MCP server — and prints a one-time notice (to the terminal when +there is one, otherwise to standard error). It stays disabled in CI and whenever +an opt-out is set. ## Opt out @@ -17,23 +20,58 @@ Use any of these mechanisms to disable telemetry: | `export KTX_TELEMETRY_DISABLED=1` | Disables telemetry for the shell and child processes | | `export DO_NOT_TRACK=1` | Standard do-not-track environment variable | | `CI=1` | Automatic in CI | -| Non-TTY output | Automatic for pipes and scripts | -| Edit `~/.ktx/telemetry.json` and set `"enabled": false` | Persistent for the machine | +| Edit `~/.ktx/telemetry.json` and set `"enabled": false` | Persistent for the machine, including the MCP server | ## What we collect -High-level signals only: which commands run, how long they take, whether they +High-level signals: which commands run, how long they take, whether they succeed or fail, and basic environment metadata (CLI version, Node version, OS -platform). For project-level analysis, **ktx** sends a salted hash of the -project directory — never the raw path. +platform). When an operation fails, we also include diagnostic detail about the +error so we can debug it. For project-level analysis, **ktx** sends a salted +hash of the project directory to group events. + +When an agent reaches **ktx** through MCP, we also record the connecting client +tool's self-reported name and version (for example Claude Desktop, Cursor, or +Cline) so we can see which agents people use **ktx** with. That describes the +tool, never you or your data. ## What we never collect -- File paths, hostnames, environment variable values, or command arguments -- `ktx.yaml` contents, connection passwords, API keys, or tokens -- Schema names, table names, column names, SQL text, or query results -- Error messages or stack traces -- Git remote URLs, Git user email, OS user, or hostname +We build telemetry around counts and coarse signals, not the contents of your +data or configuration. We don't deliberately collect your `ktx.yaml`, query +results, passwords, API keys, or access tokens. + +The one place environment-specific text can appear is failure diagnostics: when +an operation errors, the detail we record is the error as your tools reported +it, which can include identifiers from your setup. If you'd rather send nothing +at all, turn telemetry off using any of the options above. + +## Error reports + +When telemetry is enabled, **ktx** sends PostHog Error Tracking `$exception` +events for CLI and daemon exceptions. Error reports help group crashes and +handled failures into PostHog issues. + +Error reports can include: + +- Stack frames, including function names, local file paths, line numbers, and + SDK-provided source context. +- Error class names and raw error messages. +- Cause chains when the runtime exposes them. +- `source`, `handled`, and `fatal` diagnostic fields. +- Runtime version, OS, architecture, and CI fields. +- The hashed `projectId` when **ktx** knows the project. + +Error reports never intentionally include: + +- Secrets, credentials, API keys, tokens, cookies, signed URLs, or auth headers. +- Database URLs, connection strings, DSNs, raw argv, or raw environment values. +- SQL text, schema names, table names, or column names as explicit payload + properties. +- Customer row data. +- User prompt text or raw MCP arguments. + +The same opt-out controls listed above disable error reports. ## Storage and retention diff --git a/docs-site/content/docs/concepts/semantic-layer-internals.mdx b/docs-site/content/docs/concepts/semantic-layer-internals.mdx index 99410f4f..4788ef25 100644 --- a/docs-site/content/docs/concepts/semantic-layer-internals.mdx +++ b/docs-site/content/docs/concepts/semantic-layer-internals.mdx @@ -8,7 +8,7 @@ import { SemanticLayerFlow } from "@/components/semantic-layer-flow"; **ktx**'s semantic layer is a compiler that turns intent into SQL. The agent declares _what_ it wants - measures, dimensions, filters - in a small semantic query. **ktx** figures out the _how_: which tables to join, what -grain to aggregate at, how to keep fan-out from inflating measures, and +grain to aggregate at, how to keep fanout from inflating measures, and what dialect the warehouse speaks. This page covers four mechanics: @@ -16,7 +16,7 @@ This page covers four mechanics: - The semantic query contract agents send to the compiler. - The planner steps that turn a semantic query into SQL. - The join graph that backs those steps, and how it's built. -- The fan-out failure mode the compiler is designed to prevent. +- The fanout failure mode the compiler is designed to prevent. ## Imperative SQL vs declarative semantic querying @@ -84,14 +84,14 @@ same ordered steps before any SQL is emitted. 2. **Pick an anchor and build the join tree.** Choose the largest measure source as the root, then run a shortest-path search across the typed join graph to reach every required source. -3. **Detect fan-out.** Group measures by their owning source. If more +3. **Detect fanout.** Group measures by their owning source. If more than one group exists, the planner marks the query as a chasm trap and switches to aggregate-locality compilation. 4. **Classify filters.** Split predicates into row-level (`WHERE`) and aggregate-level (`HAVING`) based on whether they reference a measure. 5. **Generate SQL.** Emit Postgres-shaped SQL with the right shape: single-source aggregation when the query is safe, per-source CTEs - when fan-out is present. + when fanout is present. 6. **Transpile to the target dialect.** Run the result through `sqlglot` so the warehouse receives syntax it understands. @@ -107,7 +107,7 @@ inverted, so the planner can traverse from any anchor. | Relationship | Planning impact | |--------------|-----------------| | `many_to_one` | Safe direction for adding dimensions | -| `one_to_many` | Multiplies measures and triggers fan-out handling | +| `one_to_many` | Multiplies measures and triggers fanout handling | | `one_to_one` | Safe in either direction when keys match | | Equal-cost paths | Treated as ambiguous; aliases or explicit joins resolve them | @@ -286,9 +286,9 @@ inference. Each input contributes a different kind of authority.

-## Fan-out and aggregate locality +## Fanout and aggregate locality -Fan-out is the classic analytics failure mode. Two fact tables join to a +Fanout is the classic analytics failure mode. Two fact tables join to a shared dimension. A naive query joins them all together first, so each row from one fact is multiplied by the matching rows from the other. Measures duplicate, numbers go wrong, and the agent doesn't notice. @@ -336,5 +336,5 @@ different from what the agent first proposed. | Explain the semantic query shape | The semantic query contract | [ktx sl](/docs/cli-reference/ktx-sl) | | Describe what the planner does between query and SQL | What the planner does | [ktx sl](/docs/cli-reference/ktx-sl) | | Explain why **ktx** asks for grain and relationship types | The join graph | [Writing context](/docs/guides/writing-context) | -| Diagnose duplicated measures after a join | Fan-out and aggregate locality | [ktx sl](/docs/cli-reference/ktx-sl) | +| Diagnose duplicated measures after a join | Fanout and aggregate locality | [ktx sl](/docs/cli-reference/ktx-sl) | | Describe how semantic context stays current | Building and maintaining the graph | [Reviewing Context](/docs/guides/reviewing-context) | diff --git a/docs-site/content/docs/concepts/the-context-layer.mdx b/docs-site/content/docs/concepts/the-context-layer.mdx index 48be8c7e..a3ae6134 100644 --- a/docs-site/content/docs/concepts/the-context-layer.mdx +++ b/docs-site/content/docs/concepts/the-context-layer.mdx @@ -156,7 +156,7 @@ joins: relationship: many_to_one ``` -For how the compiler walks the join graph, handles fan-out, and transpiles +For how the compiler walks the join graph, handles fanout, and transpiles dialects, read [Semantic querying](/docs/concepts/semantic-layer-internals). ## Wiki pages @@ -240,7 +240,7 @@ models every time the warehouse changes. | **Surface** | Indexed docs and chats | Modeling language or runtime | YAML and Markdown files | | **Data-stack awareness** | None - treats data tools as text | High for declared metrics, none for the surrounding warehouse | Built in: scans schemas, dbt, BI tools, and query history | | **Maintenance** | Manual page authoring | Manual modeling, model-per-change | Auto-maintained: reconciles evidence with accepted files | -| **SQL safety** | None - generates plausible text | Compiled, dialect-correct | Compiled with join-graph and fan-out handling | +| **SQL safety** | None - generates plausible text | Compiled, dialect-correct | Compiled with join-graph and fanout handling | | **Agent edit loop** | Text-only | Tied to the modeling workflow | First-class: patch files, validate, review diffs | If you already use MetricFlow, LookML, dbt, or BI tools, **ktx** can ingest that diff --git a/docs-site/content/docs/configuration/ktx-yaml.mdx b/docs-site/content/docs/configuration/ktx-yaml.mdx index 2220814a..831e678a 100644 --- a/docs-site/content/docs/configuration/ktx-yaml.mdx +++ b/docs-site/content/docs/configuration/ktx-yaml.mdx @@ -66,8 +66,9 @@ read, how to think, and where to put the results. ## Minimal config A working `ktx.yaml` needs one entry in `connections`. Everything else accepts -defaults. The example below is enough for `ktx ingest warehouse` to run a fast -schema scan against a local Postgres. +defaults. The example below registers a local Postgres connection; building +context with `ktx ingest warehouse` also needs a model and embeddings, which +`ktx setup` configures. ```yaml connections: @@ -105,7 +106,7 @@ context-source drivers share the map. | Driver | Kind | Required fields | Common optional fields | |--------|------|-----------------|------------------------| -| `postgres` / `postgresql` | Warehouse | `driver` | `url`, `enabled_tables`, `historicSql`, `context.queryHistory` | +| `postgres` | Warehouse | `driver` | `url`, `enabled_tables`, `historicSql`, `context.queryHistory` | | `mysql` | Warehouse | `driver` | `url`, `enabled_tables` | | `sqlite` | Warehouse | `driver` | `url` or `path`, `enabled_tables` | | `sqlserver` | Warehouse | `driver` | `url`, `enabled_tables` | @@ -123,7 +124,7 @@ context-source drivers share the map. Warehouse connections are open objects: the listed fields are validated, and any other field is preserved and passed through to the connector. Use -`enabled_tables` to scope deep ingest to a specific list of +`enabled_tables` to scope ingest to a specific list of `schema.table` names - useful for smoke tests. ```yaml @@ -157,11 +158,14 @@ connections: dataset_ids: [analytics, mart] ``` -For Snowflake connections, set `maxSessions` when deep ingest needs more or -fewer concurrent warehouse sessions. The default is `4`. This caps all -concurrent Snowflake SQL work for that connector instance, including schema -introspection, table sampling, relationship profiling, relationship -validation, and read-only SQL execution. +For Postgres, MySQL, SQL Server, and Snowflake connections, set +`maxConnections` when scan or ingest work needs to stay below the target's +connection cap. Postgres, MySQL, and SQL Server default to `10`; Snowflake +defaults to `4`. This caps all concurrent SQL work for that connector instance, +including schema introspection, table sampling, relationship profiling, +relationship validation, and read-only SQL execution. BigQuery and ClickHouse +do not expose `maxConnections` because their connectors don't use client-side +connection pools. For Postgres, BigQuery, and Snowflake, `historicSql` and `context.queryHistory` toggle query-history ingest. The shape is connector-specific; the setup wizard @@ -175,9 +179,22 @@ connections: context: queryHistory: enabled: true + enabledSchemas: + - orbit_raw + - orbit_analytics minExecutions: 5 ``` +- `enabledSchemas`: Optional list of schema or dataset names that query-history + ingest may mine. Omit it to let **ktx** derive the modeled schema floor from + the connection and semantic-layer sources. Use `["*"]` to disable the floor + for discovery runs. +- `filters.serviceAccounts`: Optional service-account filter block. During + setup, when query history is enabled and no service-account block already + exists, **ktx** can propose exact role patterns such as `^svc_loader$` from + observed in-scope query history. The block uses `mode: exclude` and remains + hand-editable. + ### Metabase ```yaml @@ -372,13 +389,23 @@ llm: | Field | Type | Default | Purpose | |-------|------|---------|---------| -| `provider.backend` | `none` \| `anthropic` \| `vertex` \| `gateway` \| `claude-code` | `none` | Selected backend. `none` disables LLM features. `claude-code` uses the local Claude Code session and needs no API key. | +| `provider.backend` | `none` \| `anthropic` \| `vertex` \| `gateway` \| `claude-code` \| `codex` | `none` | Selected backend. `none` disables LLM features. `claude-code` uses the local Claude Code session and needs no API key. `codex` uses local Codex authentication and needs no API key. | | `provider.anthropic.api_key` | `string` | - | Anthropic API key. Required when `backend: anthropic`. Accepts `env:` or `file:` references. | | `provider.anthropic.base_url` | `string` | - | Override the Anthropic API base URL (proxy, self-hosted gateway). | | `provider.gateway.api_key` / `base_url` | `string` | - | Credentials for an AI Gateway provider. Required when `backend: gateway`. | | `provider.vertex.project` | `string` | - | Google Cloud project ID hosting the Vertex AI endpoint. | | `provider.vertex.location` | `string` | - | Vertex AI region (for example `us-east5`). Required when the `vertex` block is present. | +Use `codex` when local Codex authentication should power **ktx** LLM work: + +```yaml +llm: + provider: + backend: codex + models: + default: gpt-5.5 +``` + ### Model roles `models` overrides the per-role model. Keys are fixed; values are @@ -425,6 +452,16 @@ ingest: stepBudget: 40 maxConcurrency: 2 failureMode: continue + rateLimit: + enabled: true + throttleThreshold: 0.8 + minConcurrencyUnderPressure: 1 + maxWaitMs: 600000 + retry: + maxAttempts: 6 + baseDelayMs: 1000 + maxDelayMs: 60000 + jitter: true ``` ### Adapters @@ -471,6 +508,24 @@ handles failures. | `workUnits.maxConcurrency` | `int > 0` | `1` | How many work units run in parallel. | | `workUnits.failureMode` | `abort` \| `continue` | `continue` | `abort` stops the whole ingest run on the first failure; `continue` records it and keeps going. | +### Rate limits + +`rateLimit` controls provider-neutral pacing for LLM calls during ingest. When a +provider reports a subscription window, retry-after delay, or HTTP 429, +**ktx** pauses new work-unit model calls, shows a transient wait in the CLI, +and reduces work-unit concurrency while the provider is under pressure. + +| Field | Type | Default | Purpose | +|-------|------|---------|---------| +| `rateLimit.enabled` | `boolean` | `true` | Master switch for ingest LLM rate-limit pacing and visible waits. | +| `rateLimit.throttleThreshold` | `number between 0 and 1` | `0.8` | Fraction of a known provider window at which **ktx** starts reducing concurrency. | +| `rateLimit.minConcurrencyUnderPressure` | `int > 0` | `1` | Effective work-unit concurrency while a provider is under rate-limit pressure. | +| `rateLimit.maxWaitMs` | `int > 0` | unset | Caps how long a single provider-reset wait can last. This bounds each wait, not the whole run: after a capped wait elapses **ktx** retries and may pause again. Omit to wait until the provider's reset time. | +| `rateLimit.retry.maxAttempts` | `int > 0` | `6` | Maximum attempts for a single rate-limited LLM call before the failure surfaces (counts the first try). Also bounds how far opaque backoff grows for responses without a reset time or retry-after value. | +| `rateLimit.retry.baseDelayMs` | `int > 0` | `1000` | Initial opaque retry delay in milliseconds. | +| `rateLimit.retry.maxDelayMs` | `int > 0` | `60000` | Maximum opaque retry delay in milliseconds. | +| `rateLimit.retry.jitter` | `boolean` | `true` | Add jitter to opaque retry delays. | + ## `scan` `scan` configures how schema-level inputs become structured context: @@ -517,7 +572,7 @@ the manifest. | `relationships.maxLlmTablesPerBatch` | `int > 0` | `40` | Max tables included in a single LLM relationship-proposal batch. | | `relationships.maxCandidatesPerColumn` | `int > 0` | `25` | Max join partners considered per column. | | `relationships.profileSampleRows` | `int > 0` | `10000` | Rows sampled per table when profiling values for relationship inference. | -| `relationships.profileConcurrency` | `int > 0` | `4` | Parallel relationship-profile queries against the database. For Snowflake, effective database concurrency is also bounded by the connection's `maxSessions`. | +| `relationships.profileConcurrency` | `int > 0` | `4` | Parallel relationship-profile queries against the database. For pooled connectors, effective database concurrency is also bounded by the connection's `maxConnections`. | | `relationships.validationConcurrency` | `int > 0` | `4` | Parallel relationship validation queries against the database. | | `relationships.validationBudget` | `all` \| `int ≥ 0` | runtime default | Cap on validation queries per scan. `all` means unlimited. | diff --git a/docs-site/content/docs/getting-started/introduction.mdx b/docs-site/content/docs/getting-started/introduction.mdx index cc3b0ca8..50ffe20d 100644 --- a/docs-site/content/docs/getting-started/introduction.mdx +++ b/docs-site/content/docs/getting-started/introduction.mdx @@ -4,6 +4,7 @@ description: ktx is an open-source, self-improving context layer for data agents --- import { ProductMechanics } from "@/components/product-mechanics"; +import { ProductRuntime } from "@/components/product-runtime";
@@ -59,6 +60,8 @@ serves that context to agents at runtime. + + ## Use it for Use **ktx** when agents need more than raw database access. Agents can search wiki diff --git a/docs-site/content/docs/getting-started/quickstart.mdx b/docs-site/content/docs/getting-started/quickstart.mdx index f2ef3c7c..abd6044d 100644 --- a/docs-site/content/docs/getting-started/quickstart.mdx +++ b/docs-site/content/docs/getting-started/quickstart.mdx @@ -30,17 +30,18 @@ If you're a coding assistant choosing a docs route, start with the
Try **ktx** against a real data stack - Postgres, dbt, Metabase, and Notion - pre-loaded with the Orbit demo corpus. The page lists demo credentials - you can paste straight into `ktx setup`. + pre-loaded with the Orbit demo corpus. Hit **copy agent setup** on the page + for a one-shot prompt that has an agent install the full four-source demo, + or grab the raw credentials to paste into `ktx setup` yourself.
- Get demo credentials at kaelio.com/start + Get demo credentials at www.kaelio.com/start -
- Run setup from an agent -
-
- You can ask an agent such as Claude Code, Codex, Cursor, or OpenCode to - install and configure **ktx** for you. The{' '} - - agent setup Markdown prompt - {' '} - tells the agent how to check prerequisites, ask only for credentials or - connection choices, run ktx setup, verify connections, and - report the result. -
-
- Use a prompt like this from the project you want to configure: +
+
+ Or, ask an AI agent to install and configure **ktx** for you. +
+
+ + +
@@ -120,16 +170,15 @@ If you're a coding assistant choosing a docs route, start with the Prompt
-
-
Follow instructions from
-
https://docs.kaelio.com/ktx/docs/agents-setup.md
-
to install and configure ktx
+
+ Run {'`npx skills add Kaelio/ktx --skill ktx`'} and use the ktx skill to install and configure ktx
@@ -166,8 +215,8 @@ The wizard walks you through everything **ktx** needs in one pass: SQLite, PostgreSQL, MySQL, SQL Server, BigQuery, and Snowflake. 5. **Context sources** - optionally adds dbt, MetricFlow, LookML, Looker, Metabase, or Notion. You can skip and add them later. -6. **Build** - runs the first ingest so semantic sources and wiki pages - are ready for agents. +6. **Build** - offers to run the first ingest so semantic sources and wiki + pages are ready for agents. If you skip it, build later with `ktx ingest`. 7. **Agent integration** - installs project-local rules for Claude Code, Codex, Cursor, OpenCode, or universal `.agents`. @@ -187,7 +236,7 @@ Testing warehouse Connection test passed Building schema context for warehouse - Running fast database ingest + Running database scan ``` If setup exits early, rerun `ktx setup` in the same directory. **ktx** keeps @@ -198,6 +247,18 @@ progress under `.ktx/setup/` and resumes from the remaining work. > resuming setup, connecting an agent, checking status, or exploring a > pre-built demo project. +When the wizard finishes, it states where you stand and the single next action: + +- **Context built** - **ktx** confirms it is ready for agents and points you to + open your coding agent and ask a data question. +- **Build skipped** - **ktx** tells you setup is complete and that the only step + left is to build context with `ktx ingest`. + +Re-running `ktx setup` on an already-configured project goes straight to the +remaining step - building context or connecting an agent - instead of +re-asking every question. Once everything is ready, it confirms you are set +rather than reopening the configuration menu. + ## Verify When setup finishes, check readiness: @@ -219,18 +280,41 @@ Agent integration ready: yes (codex:project) For a structured check inside scripts, use `ktx status --json`. -When setup builds deep context, its final context check looks like: +If you skipped the build, `ktx context built` shows `no`. Build it with +`ktx ingest` - there is no need to re-run `ktx setup`. + +When setup finishes building context, its final context check looks like: ```text ktx context is ready for agents. Databases: - warehouse: deep context complete + warehouse: database context complete Context sources: dbt_main: memory update complete ``` +Before the build starts, **ktx** runs a live test for every connection the +build depends on. A context build can take several minutes, so if any required +connection is unreachable or misconfigured the build is blocked up front and +**ktx** names the failing connection by id and connector type: + +```text +KTX cannot build context: a required connection failed its live test. + +Failed connections: + warehouse (postgres) + +Each connection must be reachable before KTX builds context. +Run `ktx connection test ` to see the error, fix the connection, then retry. +``` + +Run `ktx connection test ` to see the underlying error, fix the +connection, then continue. In interactive setup you can retry without +restarting; with `--no-input` the build exits non-zero and names the failing +connection so scripts can stop early. + ## Connect a coding agent The setup wizard installs project-local agent rules in the last step. To @@ -277,7 +361,7 @@ ktx setup \ Then build context: ```bash -ktx ingest warehouse --fast +ktx ingest warehouse ``` See [ktx setup](/docs/cli-reference/ktx-setup) for the full automation flag @@ -290,7 +374,8 @@ surface. | `ktx: command not found` | Reinstall `@kaelio/ktx` and open a new shell | | Setup resumes the wrong project | Pass `--project-dir ` | | LLM or embeddings health check fails | Rerun setup and pick a different credential, model, or backend | -| Database test fails | Verify the same connection with the database's native client, then rerun setup | +| Database test fails | Use the setup recovery menu to retry or re-enter details; if it still fails, verify the same connection with the database's native client | +| Context build blocked: a connection failed its live test | Run `ktx connection test ` to see the error, fix the connection, then retry the build | | Agent integration is incomplete | Run `ktx setup --agents --target ` | ## Next steps diff --git a/docs-site/content/docs/guides/building-context.mdx b/docs-site/content/docs/guides/building-context.mdx index d6d58053..9bcf2659 100644 --- a/docs-site/content/docs/guides/building-context.mdx +++ b/docs-site/content/docs/guides/building-context.mdx @@ -24,7 +24,9 @@ external metadata can attach to known warehouse tables. ## Database ingest -Database ingest records table, column, type, constraint, and row-count context. +Database ingest always builds enriched context: tables, columns, types, +constraints, and row counts, plus AI-generated descriptions, embeddings, and +relationship evidence. ```bash # Build one configured database connection @@ -34,37 +36,37 @@ ktx ingest warehouse ktx ingest --all ``` -Depth controls how much context **ktx** builds: +Enriched ingest needs a configured model and embeddings. Run `ktx setup` first; +connections without that configuration fail before any work starts. -| Flag | Best for | What it does | -|------|----------|--------------| -| `--fast` | First setup, quick refreshes, CI smoke checks | Deterministic fast ingest with tables, columns, types, constraints, and row counts | -| `--deep` | Agent-ready context for real analysis | Fast ingest plus deep enrichment with descriptions, embeddings, relationship evidence, and optional query history | - -Examples: +Local-auth backends keep provider credentials out of `ktx.yaml`: ```bash -ktx ingest warehouse --fast -ktx ingest warehouse --deep -ktx ingest --all --deep +ktx setup --llm-backend claude-code --no-input +ktx setup --llm-backend codex --llm-model gpt-5.5 --no-input ``` -Deep ingest needs LLM and embedding readiness. Otherwise run `ktx setup` or use -`--fast`. - -With `claude-code`, **ktx** agent loops can invoke only the **ktx** MCP tools for the -current run. +With `claude-code`, **ktx** agent loops can invoke only the **ktx** MCP tools +for the current run. With `codex`, **ktx** restricts the temporary runtime MCP +server to the current run's tool set, disables Codex web search, requests a +read-only sandbox, and sets `approval_policy=never`. The public Codex SDK and +CLI surface may still load user Codex config and built-in command execution or +read-only file capabilities, so use `claude-code` for stricter runtime tool +isolation. ## Query history PostgreSQL, BigQuery, and Snowflake can add query-history context: common joins, -filters, service-account patterns, redaction rules, and high-usage templates. +filters, redaction rules, high-usage templates, and service-account exclusions. +When query history is enabled during setup, **ktx** reviews observed in-scope +roles and can write exact `filters.serviceAccounts` patterns for operational +traffic such as loader or refresh roles. Enable it during setup, store it under `connections..context.queryHistory`, or request it for one run: ```bash -ktx ingest warehouse --deep --query-history +ktx ingest warehouse --query-history # Set the lookback window for BigQuery or Snowflake query history ktx ingest warehouse --query-history-window-days 30 ``` @@ -74,8 +76,8 @@ for one run. ## Relationship evidence -**ktx** scores relationship candidates during supported deep database ingest. The -public CLI does not expose separate relationship review subcommands. +**ktx** scores relationship candidates during database ingest. The public CLI +does not expose separate relationship review subcommands. ## Context-source ingest @@ -159,7 +161,7 @@ After interactive setup: ```bash ktx status -ktx ingest --all --deep +ktx ingest --all ktx status ``` @@ -176,8 +178,8 @@ ktx wiki "revenue" --json --limit 10 | Symptom | Likely cause | Recovery | |---------|--------------|----------| | Connection not configured | The connection id is missing from `ktx.yaml` | Add it with `ktx setup` | -| Deep readiness is missing | LLM or embeddings are not setup-ready | Run `ktx setup`, or rerun with `--fast` | -| Query history is unsupported | The selected database driver does not expose query history | Run fast ingest without query-history flags | +| Enrichment is not configured | LLM or embeddings are not setup-ready | Run `ktx setup` to configure a model and embeddings | +| Query history is unsupported | The selected database driver does not expose query history | Run ingest without query-history flags | | No connections configured | The project has no entries under `connections` | Run `ktx setup` and add a database or context-source connection | -| Context-source flags have no effect | Depth and query-history flags were supplied for a context-source connector | Use those flags only for database connections | +| Context-source flags have no effect | Query-history flags were supplied for a context-source connector | Use query-history flags only for database connections | | Text ingest stops early | `--fail-fast` stopped on the first failed item | Fix the item or rerun without `--fail-fast` | diff --git a/docs-site/content/docs/guides/llm-configuration.mdx b/docs-site/content/docs/guides/llm-configuration.mdx index 880df24e..71ab9d80 100644 --- a/docs-site/content/docs/guides/llm-configuration.mdx +++ b/docs-site/content/docs/guides/llm-configuration.mdx @@ -16,6 +16,7 @@ Set `llm.provider.backend` to one of these values: - `gateway`: Use AI Gateway-compatible Anthropic model ids. - `claude-code`: Use your local Claude Code session through the Claude Agent SDK. **ktx** strips provider-routing environment variables from child processes. +- `codex`: Use your local Codex authentication through the Codex SDK. ## Claude Code @@ -47,6 +48,42 @@ model IDs are also accepted. metadata may still list host slash commands, skills, and subagents; **ktx** does not grant execution access to them. +## Codex backend + +Use `codex` when you want **ktx** to run LLM-backed workflows through your +local Codex authentication instead of a direct provider API key. + +```yaml +llm: + provider: + backend: codex + models: + default: gpt-5.5 +``` + +Configure it non-interactively: + +```bash +ktx setup --llm-backend codex --llm-model gpt-5.5 --no-input +``` + +This is separate from Codex agent-client setup. `ktx setup --agents --target +codex` installs instructions and MCP access for an end-user Codex session. +`ktx setup --llm-backend codex` makes **ktx** itself execute ingest, scan +enrichment, memory, and other LLM-backed work through Codex. + +During runtime loops, **ktx** starts a temporary loopback MCP server for the +current run, exposes only the tools passed to that run, asks Codex to use a +read-only sandbox, sets `approval_policy=never`, auto-approves only those +run-scoped MCP tools, and disables Codex web search. + +Codex backend isolation is currently limited by the public Codex SDK and CLI +surface. Codex may still load user Codex config and built-in command execution +or read-only file capabilities. Use `llm.provider.backend: claude-code` when +you need stricter Claude-Code-style runtime tool isolation, or remove host +Codex MCP and tool config before running untrusted prompts through the `codex` +backend. + ## Prompt caching `llm.promptCaching` has partial parity on `claude-code`. Status and doctor warn diff --git a/docs-site/content/docs/guides/serving-agents.mdx b/docs-site/content/docs/guides/serving-agents.mdx index 4c1ced4b..133739b7 100644 --- a/docs-site/content/docs/guides/serving-agents.mdx +++ b/docs-site/content/docs/guides/serving-agents.mdx @@ -111,12 +111,13 @@ non-obvious terms. Agents can refresh context when the user asks them to: ```bash -ktx ingest warehouse --fast +ktx ingest warehouse ktx ingest ktx ingest --file docs/revenue-notes.md --connection-id warehouse ``` -Use `--deep` only when LLM and embedding setup is ready. +Database ingest builds enriched context and requires a configured model and +embeddings; run `ktx setup` first if they are not ready. ## Good agent behavior diff --git a/docs-site/content/docs/integrations/agent-clients.mdx b/docs-site/content/docs/integrations/agent-clients.mdx index f7281dda..46a1ec8b 100644 --- a/docs-site/content/docs/integrations/agent-clients.mdx +++ b/docs-site/content/docs/integrations/agent-clients.mdx @@ -9,7 +9,9 @@ admin surface for setup, ingest, status, daemon lifecycle, and debugging. Run `ktx setup` and select your agent client targets, or configure manually using the snippets below. Choose **Ask data questions with ktx MCP** for agent clients. Choose **Ask data questions + manage ktx with CLI commands** only when -a developer or operator agent also needs pinned `ktx` admin commands. +a developer or operator agent also needs pinned `ktx` admin commands. Choose +**Skip agent setup for now** to leave agent integration incomplete and run +`ktx setup --agents` later. ## Install with setup @@ -43,14 +45,19 @@ ktx setup --agents --target codex --global manifest lets status checks report agent readiness and lets future cleanup remove only files **ktx** installed. -The interactive command asks two questions: +The interactive command asks what agents can do first: ```txt ◆ What should agents be allowed to do with this ktx project? │ ○ Ask data questions with ktx MCP │ ○ Ask data questions + manage ktx with CLI commands +│ ○ Skip agent setup for now └ +``` +If you choose an install mode, it then asks which targets to install: + +```txt ◆ Which agent targets should ktx install? │ ◻ Claude Code │ ◻ Claude Desktop @@ -183,10 +190,8 @@ Claude Desktop skill packages for the **ktx** workflows: - `~/Library/Application Support/Claude/claude_desktop_config.json` (macOS) or `%AppData%/Claude/claude_desktop_config.json` (Windows) gets an - `mcpServers.ktx` entry that runs the **ktx** MCP server over stdio via a local - launcher shim at `.ktx/agents/claude/ktx-plugin-runner.sh`. The shim locates - a usable Node.js (Volta, NVM, Homebrew, system) so Claude Desktop can spawn - the server without needing `node` in PATH. + `mcpServers.ktx` entry that runs the **ktx** MCP server over stdio with the + current Node.js executable and the installed `ktx` CLI entrypoint. - `.ktx/agents/claude/ktx-analytics.zip` contains the `ktx-analytics` skill. If you choose **Ask data questions + manage ktx with CLI commands**, **ktx** also generates `.ktx/agents/claude/ktx.zip` with the admin `ktx` skill. Claude diff --git a/docs-site/content/docs/integrations/primary-sources.mdx b/docs-site/content/docs/integrations/primary-sources.mdx index 81b8d400..6cb2d26f 100644 --- a/docs-site/content/docs/integrations/primary-sources.mdx +++ b/docs-site/content/docs/integrations/primary-sources.mdx @@ -517,5 +517,5 @@ No authentication required - SQLite is file-based. The file must be readable by | Connection URL appears in git diff | A literal credential URL was written to `ktx.yaml` | Replace it with `env:NAME` or `file:/path/to/secret` and rotate exposed credentials | | Database ingest returns no tables | Schema, database, or project filter is wrong, or the user lacks metadata permissions | Verify the schema list and grant metadata read permissions | | Query history is empty | Query history extension or warehouse history view is unavailable | Enable the warehouse-specific history feature, then rerun `ktx ingest --query-history` or `ktx setup` | -| Column statistics are missing | Connector cannot access stats tables or the warehouse does not expose them | Grant stats permissions where supported; otherwise rely on fast schema context | +| Column statistics are missing | Connector cannot access stats tables or the warehouse does not expose them | Grant stats permissions where supported; otherwise rely on schema-level context without column statistics | | Semantic query execution fails | Connection is missing, unreachable, or query execution is disabled | Run `ktx connection test ` and check the `ktx sl query` flags | diff --git a/docs-site/lib/agent-setup-markdown.ts b/docs-site/lib/agent-setup-markdown.ts deleted file mode 100644 index 5a42ea1f..00000000 --- a/docs-site/lib/agent-setup-markdown.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { readFile } from "node:fs/promises"; -import { join } from "node:path"; - -export const agentSetupSlug = ["agents-setup"] as const; - -export function isAgentSetupSlug(slug: string[] | undefined) { - return slug?.length === 1 && slug[0] === agentSetupSlug[0]; -} - -export function readAgentSetupMarkdown() { - return readFile(join(process.cwd(), "content/agents-setup.md"), "utf8"); -} diff --git a/docs-site/lib/llm-docs.ts b/docs-site/lib/llm-docs.ts index 7ed338a0..fd6c8dd1 100644 --- a/docs-site/lib/llm-docs.ts +++ b/docs-site/lib/llm-docs.ts @@ -52,8 +52,9 @@ ktx provides semantic-layer files, warehouse scans, wiki pages, provenance, and ## Agent Entry Points +- Installable setup skill: run \`npx skills add Kaelio/ktx --skill ktx\` from + the project you want to configure. ${link("/docs/ai-resources/agent-quickstart", "Agent Quickstart", "Task-first route for coding assistants using ktx")} -${link("/docs/agents-setup", "Agent Setup", "Copy-pasteable prompt for agents installing and configuring ktx")} ${link("/docs/ai-resources/markdown-access", "Markdown Access", "Fetch ktx docs as llms.txt, llms-full.txt, or per-page Markdown")} ${link("/docs/ai-resources/agent-instructions", "Agent Instructions", "Suggested instructions for coding assistants that need to read and cite ktx docs")} diff --git a/docs-site/next.config.mjs b/docs-site/next.config.mjs index 30a96741..e47a0cc7 100644 --- a/docs-site/next.config.mjs +++ b/docs-site/next.config.mjs @@ -6,15 +6,60 @@ const withMDX = createMDX(); const config = { basePath: "/ktx", async rewrites() { - return [ - { - source: "/docs/:path*.md", - destination: "/llms.mdx/docs/:path*", - }, - ]; + return { + beforeFiles: [ + { + source: "/stars", + has: [{ type: "host", value: "ktx.sh" }], + destination: "https://ktx-stars.vercel.app/stars", + basePath: false, + }, + { + source: "/stars/:path*", + has: [{ type: "host", value: "ktx.sh" }], + destination: "https://ktx-stars.vercel.app/stars/:path*", + basePath: false, + }, + ], + afterFiles: [ + { + source: "/docs/:path*.md", + destination: "/llms.mdx/docs/:path*", + }, + ], + }; }, async redirects() { + // Alias-host canonicalization MUST come before the generic root/docs + // redirects below. Those generic rules have no host guard, so if they ran + // first they would inject a "/ktx" basePath into the path on the alias + // hosts, which the alias catch-alls would then prepend a second time — + // producing https://docs.kaelio.com/ktx/ktx/docs/... Redirects also run + // before beforeFiles rewrites, so the ktx.sh catch-all must exclude + // /stars* to let the stars dashboard rewrite proxy through. return [ + { + source: "/slack", + has: [{ type: "host", value: "ktx.sh" }], + destination: + "https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ", + permanent: false, + basePath: false, + }, + { + source: "/:path*", + has: [{ type: "host", value: "docs.ktx.sh" }], + destination: "https://docs.kaelio.com/ktx/:path*", + permanent: true, + basePath: false, + }, + { + source: "/:path((?!stars(?:/|$)).*)", + has: [{ type: "host", value: "ktx.sh" }], + destination: "https://docs.kaelio.com/ktx/:path", + permanent: true, + basePath: false, + }, { source: "/", destination: "/ktx/docs/getting-started/introduction", @@ -27,20 +72,6 @@ const config = { permanent: false, basePath: false, }, - { - source: "/:path*", - has: [{ type: "host", value: "docs.ktx.sh" }], - destination: "https://docs.kaelio.com/ktx/:path*", - permanent: true, - basePath: false, - }, - { - source: "/:path*", - has: [{ type: "host", value: "ktx.sh" }], - destination: "https://docs.kaelio.com/ktx/:path*", - permanent: true, - basePath: false, - }, ]; }, }; diff --git a/docs-site/package.json b/docs-site/package.json index 4cf896ff..f418c0ee 100644 --- a/docs-site/package.json +++ b/docs-site/package.json @@ -12,15 +12,16 @@ "dependencies": { "@xyflow/react": "^12.10.2", "fumadocs-core": "16.8.10", - "fumadocs-mdx": "15.0.4", + "fumadocs-mdx": "15.0.7", "fumadocs-ui": "16.8.10", + "html-to-image": "1.11.11", "next": "^16", "react": "19.2.6", "react-dom": "19.2.6" }, "devDependencies": { "@tailwindcss/postcss": "^4", - "@types/node": "^25.7.0", + "@types/node": "^25.9.1", "@types/react": "^19", "@types/react-dom": "^19", "tailwindcss": "^4", diff --git a/docs-site/public/images/ingestion-flow-transparent.svg b/docs-site/public/images/ingestion-flow-transparent.svg deleted file mode 100644 index 86356d6b..00000000 --- a/docs-site/public/images/ingestion-flow-transparent.svg +++ /dev/null @@ -1,210 +0,0 @@ - - ktx ingestion flow - Source systems flow through source connectors, context builder, reconciliation, and validation to create wiki Markdown and semantic-layer YAML outputs. - - - - - - - - - - - - - - - - - - - - - - - - - Databases - Schemas, columns, keys, - row counts, and query - history. - - - PostgreSQL - - Snowflake - - BigQuery - - SQLite - - - - - - - BI tools - Dashboards, questions, - explores, usage, and trusted - examples. - - - Metabase - - Looker - - - - - - - Modeling code - Existing metrics, dimensions, - models, joins, and entities. - - - dbt - - LookML - - MetricFlow - - - - - - - Docs and notes - Policies, caveats, team - definitions, and analyst - context. - - - Notion - - Any text - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1 - Source connectors - Read each configured system in - its native shape. - - - - - - 2 - Context builder - Turn source evidence into - proposed context updates. - - - - - - 3 - Reconciliation - Merge new evidence with the - context that already exists. - - - - - - 4 - Validation - Check references and semantics - before agents rely on them. - - - - - - - - wiki/*.md - Wiki - - - free-form - - auto-maintained - - Definitions, caveats, policies, analyst notes, and - business language that agents can search. - - - - - - semantic-layer/*.yaml - Semantic layer - - - structured - - executable - - auto-maintained - - Metrics, joins, tables, dimensions, filters, and - segments that ktx can validate and compile into - SQL. - - - - - references - - - diff --git a/docs-site/public/images/ingestion-flow.png b/docs-site/public/images/ingestion-flow.png index 49bc544f..59f6ad17 100644 Binary files a/docs-site/public/images/ingestion-flow.png and b/docs-site/public/images/ingestion-flow.png differ diff --git a/docs-site/public/images/mcp-runtime-flow.png b/docs-site/public/images/mcp-runtime-flow.png new file mode 100644 index 00000000..ec56dff1 Binary files /dev/null and b/docs-site/public/images/mcp-runtime-flow.png differ diff --git a/docs-site/tests/docs-index-route.test.mjs b/docs-site/tests/docs-index-route.test.mjs index fdd8ec81..6fac0e3c 100644 --- a/docs-site/tests/docs-index-route.test.mjs +++ b/docs-site/tests/docs-index-route.test.mjs @@ -2,6 +2,8 @@ import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import { once } from "node:events"; import { readFile, writeFile } from "node:fs/promises"; +import http from "node:http"; +import https from "node:https"; import { dirname, join } from "node:path"; import { createServer } from "node:net"; import { after, before, test } from "node:test"; @@ -100,6 +102,37 @@ after(async () => { } }); +// Node's fetch (undici) overwrites the Host header with the connection host, +// so the alias-host redirect rules never match. The low-level http(s) client +// sends Host verbatim, which is what the alias canonicalization keys off of. +function requestWithHost(hostHeader, path) { + const target = new URL(docsSiteUrl); + const client = target.protocol === "https:" ? https : http; + const port = + target.port || (target.protocol === "https:" ? "443" : "80"); + + return new Promise((resolve, reject) => { + const request = client.request( + { + hostname: target.hostname, + port, + path, + method: "GET", + headers: { Host: hostHeader }, + }, + (response) => { + response.resume(); + resolve({ + status: response.statusCode, + location: response.headers.location, + }); + }, + ); + request.on("error", reject); + request.end(); + }); +} + test("/ktx/docs redirects to the docs introduction", async () => { const response = await fetch(`${docsSiteUrl}${docsBasePath}/docs`, { redirect: "manual", @@ -141,3 +174,51 @@ test("/ktx/api/search returns docs search results", async () => { "search should return at least one docs result", ); }); + +test("ktx.sh canonicalizes to a single /ktx basePath on the docs host", async () => { + const root = await requestWithHost("ktx.sh", "/"); + assert.equal(root.status, 308); + assert.equal(root.location, "https://docs.kaelio.com/ktx/"); + assert.ok( + !root.location.includes("/ktx/ktx"), + "the basePath must not be doubled", + ); + + const page = await requestWithHost( + "ktx.sh", + "/docs/getting-started/quickstart", + ); + assert.equal(page.status, 308); + assert.equal( + page.location, + "https://docs.kaelio.com/ktx/docs/getting-started/quickstart", + ); +}); + +test("docs.ktx.sh canonicalizes to a single /ktx basePath on the docs host", async () => { + const root = await requestWithHost("docs.ktx.sh", "/"); + assert.equal(root.status, 308); + assert.equal(root.location, "https://docs.kaelio.com/ktx"); + assert.ok( + !root.location.includes("/ktx/ktx"), + "the basePath must not be doubled", + ); + + const page = await requestWithHost("docs.ktx.sh", "/llms.txt"); + assert.equal(page.status, 308); + assert.equal(page.location, "https://docs.kaelio.com/ktx/llms.txt"); +}); + +test("ktx.sh keeps the /slack and /stars exceptions", async () => { + const slack = await requestWithHost("ktx.sh", "/slack"); + assert.equal(slack.status, 307); + assert.match(slack.location, /^https:\/\/join\.slack\.com\//); + + // /stars is proxied by a beforeFiles rewrite, so the apex catch-all must not + // canonicalize it to the docs host. + const stars = await requestWithHost("ktx.sh", "/stars"); + assert.ok( + !(stars.location ?? "").startsWith("https://docs.kaelio.com"), + "the stars dashboard must not be redirected to the docs host", + ); +}); diff --git a/docs-site/tests/product-mechanics-content.test.mjs b/docs-site/tests/product-mechanics-content.test.mjs index 5cce9001..d0c9471c 100644 --- a/docs-site/tests/product-mechanics-content.test.mjs +++ b/docs-site/tests/product-mechanics-content.test.mjs @@ -85,7 +85,7 @@ test("product mechanics component explains ingestion outputs", async () => { "compile into SQL", '"use client"', "@xyflow/react", - " { ); } - assert.match( - component, + // The ReactFlow canvas config lives in the shared FlowCanvas wrapper, which + // product-mechanics renders. Assert the static read-only behavior there. + const flowCanvas = await readDocsFile("components/flow-canvas.tsx"); + for (const guard of [ /nodesDraggable=\{false\}/, - "ReactFlow canvas should disable node dragging", - ); - assert.match( - component, - /panOnDrag=\{false\}/, - "ReactFlow canvas should disable panning", - ); - assert.match( - component, + /nodesConnectable=\{false\}/, /zoomOnScroll=\{false\}/, - "ReactFlow canvas should disable scroll zoom", - ); + /elementsSelectable=\{false\}/, + ]) { + assert.match( + flowCanvas, + guard, + `shared FlowCanvas should enforce static read-only behavior: ${guard}`, + ); + } assert.doesNotMatch(component, /raw-sources/); assert.doesNotMatch(component, /\.ktx/); diff --git a/docs-site/tests/product-runtime-content.test.mjs b/docs-site/tests/product-runtime-content.test.mjs new file mode 100644 index 00000000..ac643faa --- /dev/null +++ b/docs-site/tests/product-runtime-content.test.mjs @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; + +const docsSiteDir = join(dirname(fileURLToPath(import.meta.url)), ".."); + +async function readDocsFile(path) { + return readFile(join(docsSiteDir, path), "utf8"); +} + +test("docs introduction renders the serving phase after ingestion", async () => { + const introduction = await readDocsFile( + "content/docs/getting-started/introduction.mdx", + ); + + assert.match( + introduction, + /import\s+\{\s*ProductRuntime\s*\}\s+from\s+"@\/components\/product-runtime";/, + ); + assert.match(introduction, //); + + const mechanicsIndex = introduction.indexOf(""); + const runtimeIndex = introduction.indexOf(""); + const useCaseIndex = introduction.indexOf("## Use it for"); + + assert.ok( + runtimeIndex > mechanicsIndex, + "serving diagram should appear after the ingestion diagram", + ); + assert.ok( + runtimeIndex < useCaseIndex, + "serving diagram should appear before use-case sections", + ); +}); + +test("product runtime component explains the serving cycle", async () => { + const component = await readDocsFile("components/product-runtime.tsx"); + + for (const expectedText of [ + "How serving works", + "Serving flow", + "From an agent request to a governed answer", + "Your agent", + "Claude Code", + "Cursor", + "Codex", + "Search wiki + semantic layer", + "Return approved metrics", + "Compile metrics → SQL", + "Context layer", + "Database", + "search + read", + "read-only", + "wiki/*.md", + "semantic-layer/*.yaml", + '"use client"', + "@xyflow/react", + "FlowCanvas", + "getSmoothStepPath", + "animateMotion", + "runtime-particle", + "buildCyclePath", + ]) { + assert.ok( + component.includes(expectedText), + `component should include: ${expectedText}`, + ); + } + + assert.doesNotMatch(component, /raw-sources/); + assert.doesNotMatch(component, //` plus +`packages/cli/src/context/connections/{dialects,drivers}.ts` is the +canonical worked example: per-driver dialect classes carry +`/** @internal */`, `scripts/check-boundaries.mjs` enforces the import +boundary, and dispatch lives in the two registry files. Apply the +same shape to any other per-variant layer that grows beyond two +implementations. diff --git a/docs/terminology.md b/docs/terminology.md index fab5d290..4c9ec3cb 100644 --- a/docs/terminology.md +++ b/docs/terminology.md @@ -21,6 +21,41 @@ in prose when ambiguity is possible. Always qualify: Bare `source` is allowed only inside a section that has already established its referent (e.g., body of a `Semantic sources` page, or `sourceName` as a CLI arg). +## Context Layer and Context Engine + +Use **context layer** as the primary category term for what **ktx** provides to +data agents. + +Use **context engine** as the secondary mechanism term for how **ktx** builds, +maintains, validates, and serves that layer. + +| Concept | Use | Do not use | +|---|---|---| +| The whole **ktx** product category | **context layer** / **context layer for data agents** | knowledge layer, agent memory | +| The active system that builds and maintains context | **context engine** | context layer when describing ingest/reconciliation internals | +| The durable reviewed surface agents use | **context layer** | context engine | +| The compiler pillar for executable metrics and joins | **semantic layer** | context layer when specifically discussing SQL compilation | +| Prose/business knowledge files | **wiki** / **wiki pages** | wiki context | + +### Usage rules + +- Use **context layer** in taglines, page titles, meta descriptions, docs + introductions, comparison pages, and first-paragraph definitions. +- Use **context engine** when describing active behavior: ingesting evidence, + reconciling changes, validating references, maintaining files, search, CLI, + and MCP serving. +- Keep **semantic layer** for the narrower YAML/compiler surface: semantic + sources, measures, joins, dimensions, filters, SQL compilation, and semantic + queries. +- Do not use **context engine** as the primary replacement for the whole + product. It sounds like runtime infrastructure; **context layer** better + describes the durable YAML and Markdown surface users review in git. +- Do not use **context layer** when the sentence is specifically about the + compiler. Example: write "the semantic layer compiles semantic queries to + SQL," not "the context layer compiles semantic queries to SQL." +- Default lowercase in prose: `context layer`, `context engine`, `semantic + layer`. Title case only in page titles, headings, nav labels, and UI labels. + ## Canonical vocabulary | Concept | Use | Do not use | @@ -31,7 +66,8 @@ referent (e.g., body of a `Semantic sources` page, or `sourceName` as a CLI arg) | The connected database | **primary source** / **database connection** | data source | | Analytics-tooling integration | **context source** / **context-source connection** | BI source, BI model, metadata source, source tool | | YAML file describing a table | **semantic source** | semantic-layer source, model file, bare "source file" | -| The whole **ktx** surface | **context layer** (lowercase in prose) | "Context Layer" in prose | +| The whole **ktx** surface | **context layer** / **context layer for data agents** (lowercase in prose) | "Context Layer" in prose, knowledge layer, agent memory | +| The active system that builds and maintains context | **context engine** (lowercase in prose) | context layer when describing ingest/reconciliation internals | | The compiler pillar | **semantic layer** (lowercase in prose) | "Semantic Layer" in prose | | The query payload | **semantic query** (lowercase in prose) | "Semantic Query" | | The MCP layer | **MCP server** (the server), **MCP tools** (the functions) | "ktx MCP" as a standalone noun | @@ -41,8 +77,6 @@ referent (e.g., body of a `Semantic sources` page, or `sourceName` as a CLI arg) | Connection ref in prose | **connection id** (lowercase, two words) | "connection ID" | | CLI arg/flag literal | `connectionId` (code font) | — | | File path placeholder | `` (code font) | — | -| Fast schema mode | **fast ingest** | schema ingest, schema-only ingest | -| AI-enriched mode | **deep ingest** | AI-enriched ingest | | Ingest of a primary connection | **database ingest** | — | | Ingest of a context-source connection | **context-source ingest** | bare "source ingest" | | Wiki capture | **text ingest** | — | @@ -56,7 +90,7 @@ referent (e.g., body of a `Semantic sources` page, or `sourceName` as a CLI arg) | Wiki surface as a whole | **wiki** | "wiki context" | | A single Markdown file | **wiki page** | — | | YAML vs Markdown contrast | **wiki Markdown** (only when contrasting with **semantic source YAML**) | — | -| Joins multiplying rows (generic) | **fan-out** | — | +| Joins multiplying rows (generic) | **fanout** | — | | The two named patterns | **chasm trap** / **fan trap** | — | | Casual gloss in user prose | **double-count** | (avoid in technical/internals prose) | diff --git a/knip.json b/knip.json index 178ff87e..65b1a0a2 100644 --- a/knip.json +++ b/knip.json @@ -14,8 +14,8 @@ "src/telemetry/schema-writer.ts!", "src/telemetry/index.ts!", "scripts/**/*.mjs", - "src/**/*.test-utils.ts", - "src/**/acceptance-fixtures.ts", + "test/**/*.test-utils.ts", + "test/**/acceptance-fixtures.ts", "src/context/scan/relationship-benchmarks.ts!", "src/context/scan/relationship-benchmark-report.ts!" ] @@ -37,6 +37,9 @@ "@semantic-release/release-notes-generator", "conventional-changelog-conventionalcommits" ], + "ignore": [ + ".context/**" + ], "ignoreBinaries": [ "uv", "lsof" diff --git a/package.json b/package.json index 52776d50..e7714634 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,10 @@ { "name": "ktx-workspace", - "version": "0.5.0", + "version": "0.9.0", "description": "Workspace root for ktx packages", "private": true, "type": "module", - "packageManager": "pnpm@11.1.1", + "packageManager": "pnpm@11.4.0", "engines": { "node": ">=22.0.0", "pnpm": ">=10.20.0" @@ -24,6 +24,7 @@ "dead-code:fix": "biome check . --formatter-enabled=false --assist-enabled=false --write && knip --fix --format", "dead-code:knip": "knip --reporter compact", "dead-code:knip:production": "knip --production --reporter compact", + "deps:upgrade": "node scripts/upgrade-dependencies.mjs", "docs": "kill $(lsof -ti:3000) 2>/dev/null; pnpm --filter ktx-docs run dev", "ktx": "node scripts/run-ktx.mjs", "link:dev": "node scripts/link-dev-cli.mjs", @@ -31,6 +32,7 @@ "setup:dev": "node scripts/setup-dev.mjs", "release:published-smoke": "node scripts/published-package-smoke.mjs --require-config", "release:local-embeddings-smoke": "node scripts/local-embeddings-runtime-smoke.mjs --require-opt-in", + "release:codex-backend-smoke": "node scripts/codex-backend-live-smoke.mjs", "release:readiness": "node scripts/release-readiness.mjs", "release:update-version": "node scripts/update-public-release-version.mjs", "relationships:acquire-public-fixtures": "node scripts/acquire-public-benchmark-fixtures.mjs", @@ -58,11 +60,11 @@ "@semantic-release/github": "^12.0.8", "@semantic-release/npm": "^13.1.5", "@semantic-release/release-notes-generator": "^14.1.1", - "@types/node": "^25.7.0", + "@types/node": "^25.9.1", "better-sqlite3": "^12.10.0", "conventional-changelog-conventionalcommits": "^9.3.1", - "knip": "^6.12.2", - "pg": "^8.20.0", + "knip": "^6.14.1", + "pg": "^8.21.0", "semantic-release": "^25.0.3", "typescript": "^6.0.3", "yaml": "^2.9.0" diff --git a/packages/cli/package.json b/packages/cli/package.json index d10c1b2b..c08d26f2 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,11 @@ { "name": "@kaelio/ktx", - "version": "0.5.0", + "version": "0.9.0", "description": "Standalone ktx context layer for data agents", + "author": { + "name": "Kaelio", + "url": "https://www.kaelio.com" + }, "type": "module", "engines": { "node": ">=22.0.0" @@ -32,47 +36,50 @@ "build": "tsc -p tsconfig.json && node dist/telemetry/schema-writer.js src/telemetry/events.schema.json ../../python/ktx-daemon/src/ktx_daemon/telemetry/events.schema.json && node scripts/copy-runtime-assets.mjs && node ../../scripts/prepare-cli-bin.mjs", "clean": "node -e \"fs.rmSync('dist', { recursive: true, force: true })\"", "docs:commands": "pnpm run build && node dist/print-command-tree.js", - "smoke": "vitest run src/standalone-smoke.test.ts src/example-smoke.test.ts --testTimeout 30000", - "test": "vitest run --exclude src/standalone-smoke.test.ts --exclude src/example-smoke.test.ts --exclude src/setup-databases.test.ts --exclude src/scan.test.ts --exclude src/commands/connection-metabase-setup.test.ts --exclude src/setup-models.test.ts --exclude src/setup-sources.test.ts --exclude src/setup.test.ts --exclude src/connection.test.ts --exclude src/setup-embeddings.test.ts --exclude src/ingest.test.ts --exclude src/commands/connection-mapping.test.ts --exclude src/ingest-viz.test.ts --exclude src/demo.test.ts --exclude src/setup-project.test.ts --exclude src/sl.test.ts --exclude src/local-scan-connectors.test.ts --exclude src/commands/connection-notion.test.ts --exclude src/context/scan/local-scan.test.ts --exclude src/context/mcp/local-project-ports.test.ts --exclude src/context/ingest/local-stage-ingest.test.ts --exclude src/context/sl/pglite-sl-search-prototype.test.ts --exclude src/context/core/git.service.test.ts --exclude src/context/ingest/local-adapters.test.ts --exclude src/context/ingest/local-bundle-ingest.test.ts --exclude src/context/ingest/local-metabase-ingest.test.ts --exclude src/context/sl/local-sl.test.ts --exclude src/context/search/pglite-owner-process.test.ts --exclude src/context/scan/local-enrichment-artifacts.test.ts --exclude src/context/search/pglite-spike.test.ts --exclude src/context/wiki/local-knowledge.test.ts --exclude src/context/sl/local-query.test.ts --exclude src/context/scan/relationship-review-decisions.test.ts --exclude src/context/scan/relationship-profiling.test.ts", - "test:slow": "vitest run src/setup-databases.test.ts src/scan.test.ts src/commands/connection-metabase-setup.test.ts src/setup-models.test.ts src/setup-sources.test.ts src/setup.test.ts src/connection.test.ts src/setup-embeddings.test.ts src/ingest.test.ts src/commands/connection-mapping.test.ts src/ingest-viz.test.ts src/demo.test.ts src/setup-project.test.ts src/sl.test.ts src/local-scan-connectors.test.ts src/commands/connection-notion.test.ts src/context/scan/local-scan.test.ts src/context/mcp/local-project-ports.test.ts src/context/ingest/local-stage-ingest.test.ts src/context/sl/pglite-sl-search-prototype.test.ts src/context/core/git.service.test.ts src/context/ingest/local-adapters.test.ts src/context/ingest/local-bundle-ingest.test.ts src/context/ingest/local-metabase-ingest.test.ts src/context/sl/local-sl.test.ts src/context/search/pglite-owner-process.test.ts src/context/scan/local-enrichment-artifacts.test.ts src/context/search/pglite-spike.test.ts src/context/wiki/local-knowledge.test.ts src/context/sl/local-query.test.ts src/context/scan/relationship-review-decisions.test.ts src/context/scan/relationship-profiling.test.ts --testTimeout 30000", - "type-check": "tsc -p tsconfig.json --noEmit", + "smoke": "vitest run test/standalone-smoke.test.ts test/example-smoke.test.ts --testTimeout 30000", + "test": "vitest run --exclude test/standalone-smoke.test.ts --exclude test/example-smoke.test.ts --exclude test/setup-databases.test.ts --exclude test/scan.test.ts --exclude test/commands/connection-metabase-setup.test.ts --exclude test/setup-models.test.ts --exclude test/setup-sources.test.ts --exclude test/setup.test.ts --exclude test/connection.test.ts --exclude test/setup-embeddings.test.ts --exclude test/ingest.test.ts --exclude test/commands/connection-mapping.test.ts --exclude test/ingest-viz.test.ts --exclude test/demo.test.ts --exclude test/setup-project.test.ts --exclude test/sl.test.ts --exclude test/local-scan-connectors.test.ts --exclude test/commands/connection-notion.test.ts --exclude test/context/scan/local-scan.test.ts --exclude test/context/mcp/local-project-ports.test.ts --exclude test/context/ingest/local-stage-ingest.test.ts --exclude test/context/sl/pglite-sl-search-prototype.test.ts --exclude test/context/core/git.service.test.ts --exclude test/context/ingest/local-adapters.test.ts --exclude test/context/ingest/local-bundle-ingest.test.ts --exclude test/context/ingest/local-metabase-ingest.test.ts --exclude test/context/sl/local-sl.test.ts --exclude test/context/search/pglite-owner-process.test.ts --exclude test/context/scan/local-enrichment-artifacts.test.ts --exclude test/context/search/pglite-spike.test.ts --exclude test/context/wiki/local-knowledge.test.ts --exclude test/context/sl/local-query.test.ts --exclude test/context/scan/relationship-review-decisions.test.ts --exclude test/context/scan/relationship-profiling.test.ts", + "test:slow": "vitest run test/setup-databases.test.ts test/scan.test.ts test/commands/connection-metabase-setup.test.ts test/setup-models.test.ts test/setup-sources.test.ts test/setup.test.ts test/connection.test.ts test/setup-embeddings.test.ts test/ingest.test.ts test/commands/connection-mapping.test.ts test/ingest-viz.test.ts test/demo.test.ts test/setup-project.test.ts test/sl.test.ts test/local-scan-connectors.test.ts test/commands/connection-notion.test.ts test/context/scan/local-scan.test.ts test/context/mcp/local-project-ports.test.ts test/context/ingest/local-stage-ingest.test.ts test/context/sl/pglite-sl-search-prototype.test.ts test/context/core/git.service.test.ts test/context/ingest/local-adapters.test.ts test/context/ingest/local-bundle-ingest.test.ts test/context/ingest/local-metabase-ingest.test.ts test/context/sl/local-sl.test.ts test/context/search/pglite-owner-process.test.ts test/context/scan/local-enrichment-artifacts.test.ts test/context/search/pglite-spike.test.ts test/context/wiki/local-knowledge.test.ts test/context/sl/local-query.test.ts test/context/scan/relationship-review-decisions.test.ts test/context/scan/relationship-profiling.test.ts --testTimeout 30000", + "type-check": "tsc -p tsconfig.json --noEmit && tsc -p tsconfig.test.json --noEmit", "relationships:benchmarks": "pnpm --silent run build && node ../../scripts/relationship-benchmark-report.mjs", - "relationships:benchmarks:test": "KTX_RUN_RELATIONSHIP_BENCHMARKS=1 vitest run src/context/scan/relationship-benchmarks.test.ts", + "relationships:benchmarks:test": "KTX_RUN_RELATIONSHIP_BENCHMARKS=1 vitest run test/context/scan/relationship-benchmarks.test.ts", "search:pglite-spike": "node ../../scripts/pglite-hybrid-search-spike.mjs", "search:pglite-owner-prototype": "node ../../scripts/pglite-owner-process-prototype.mjs", "search:pglite-sl-prototype": "node ../../scripts/pglite-sl-search-prototype.mjs" }, "dependencies": { - "@ai-sdk/anthropic": "3.0.77", - "@ai-sdk/devtools": "0.0.17", - "@ai-sdk/google-vertex": "^4.0.128", - "@anthropic-ai/claude-agent-sdk": "0.3.142", + "@ai-sdk/anthropic": "3.0.78", + "@ai-sdk/devtools": "0.0.18", + "@ai-sdk/google-vertex": "^4.0.134", + "@anthropic-ai/claude-agent-sdk": "0.3.146", + "@clack/core": "1.3.1", "@clack/prompts": "1.4.0", - "@clickhouse/client": "^1.18.4", + "@clickhouse/client": "^1.18.5", "@commander-js/extra-typings": "14.0.0", "@google-cloud/bigquery": "^8.3.1", "@looker/sdk": "^26.8.0", "@looker/sdk-node": "^26.8.0", "@looker/sdk-rtl": "^21.6.5", "@modelcontextprotocol/sdk": "^1.29.0", - "@notionhq/client": "^5.21.0", - "ai": "^6.0.180", + "@notionhq/client": "^5.22.0", + "@openai/codex-sdk": "^0.133.0", + "ai": "^6.0.188", "better-sqlite3": "^12.10.0", "commander": "14.0.3", - "fflate": "^0.8.2", + "fflate": "^0.8.3", "handlebars": "^4.7.9", - "ink": "^7.0.2", + "ink": "^7.0.3", "lookml-parser": "7.1.0", "minimatch": "^10.2.5", - "mssql": "^12.5.2", + "mssql": "^12.5.4", "mysql2": "^3.22.3", - "openai": "^6.37.0", + "openai": "^6.38.0", "p-limit": "^7.3.0", - "pg": "^8.20.0", - "posthog-node": "^5.0.0", + "pg": "^8.21.0", + "posthog-node": "^5.34.9", "react": "^19.2.6", + "semver": "^7.8.1", "simple-git": "3.36.0", - "snowflake-sdk": "^2.4.1", + "snowflake-sdk": "^2.4.2", "yaml": "^2.9.0", "zod": "^4.4.3" }, @@ -81,14 +88,15 @@ "@electric-sql/pglite-socket": "^0.1.5", "@types/better-sqlite3": "^7.6.13", "@types/mssql": "^12.3.0", - "@types/node": "^25.7.0", + "@types/node": "^25.9.1", "@types/pg": "^8.20.0", - "@types/react": "^19.2.14", - "@vitest/coverage-v8": "^4.1.6", + "@types/react": "^19.2.15", + "@types/semver": "^7.7.1", + "@vitest/coverage-v8": "^4.1.7", "ajv": "8.20.0", "ink-testing-library": "^4.0.0", "typescript": "^6.0.3", - "vitest": "^4.1.6" + "vitest": "^4.1.7" }, "license": "Apache-2.0", "repository": { diff --git a/packages/cli/src/clack.ts b/packages/cli/src/clack.ts index 55d3e802..31be2e1b 100644 --- a/packages/cli/src/clack.ts +++ b/packages/cli/src/clack.ts @@ -1,7 +1,54 @@ import { cancel, confirm, isCancel, log, spinner } from '@clack/prompts'; +import type { KtxCliIo } from './cli-runtime.js'; const ESC = String.fromCharCode(0x1b); +export interface CliStyleEnv { + NO_COLOR?: string; + TERM?: string; +} + +function ansiEnabled(env: CliStyleEnv = process.env): boolean { + return !env.NO_COLOR && env.TERM !== 'dumb'; +} + +function ansiColor(text: string, open: number, close: number, env?: CliStyleEnv): string { + if (!ansiEnabled(env)) { + return text; + } + return `${ESC}[${open}m${text}${ESC}[${close}m`; +} + +export function dim(text: string, env?: CliStyleEnv): string { + return ansiColor(text, 2, 22, env); +} + +export function cyan(text: string, env?: CliStyleEnv): string { + return ansiColor(text, 36, 39, env); +} + +export interface RailBufferedSource { + stdoutText(): string; + stderrText(): string; +} + +export function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export function writePrefixedLines(write: (chunk: string) => void, output: string): void { + for (const line of output.split(/\r?\n/)) { + if (line.length > 0) { + write(`│ ${line}\n`); + } + } +} + +export function flushPrefixedBufferedCommandOutput(io: KtxCliIo, buffered: RailBufferedSource): void { + writePrefixedLines((chunk) => io.stdout.write(chunk), buffered.stdoutText()); + writePrefixedLines((chunk) => io.stderr.write(chunk), buffered.stderrText()); +} + export interface KtxCliSpinner { start(message: string): void; message(message: string): void; @@ -38,11 +85,11 @@ export function createClackSpinner(): KtxCliSpinner { } function magenta(text: string): string { - return `${ESC}[35m${text}${ESC}[39m`; + return ansiColor(text, 35, 39); } function red(text: string): string { - return `${ESC}[31m${text}${ESC}[39m`; + return ansiColor(text, 31, 39); } export function createStaticCliSpinner(io: KtxCliSpinnerIo): KtxCliSpinner { diff --git a/packages/cli/src/cli-program.ts b/packages/cli/src/cli-program.ts index a3c27375..6359d897 100644 --- a/packages/cli/src/cli-program.ts +++ b/packages/cli/src/cli-program.ts @@ -2,6 +2,7 @@ import { existsSync } from 'node:fs'; import { join } from 'node:path'; import { Command, type CommandUnknownOpts, InvalidArgumentError } from '@commander-js/extra-typings'; import type { KtxCliDeps, KtxCliIo, KtxCliPackageInfo } from './cli-runtime.js'; +import { registerCompletionCommands } from './commands/completion-commands.js'; import { registerConnectionCommands } from './commands/connection-commands.js'; import { registerIngestCommands } from './commands/ingest-commands.js'; import { registerWikiCommands } from './commands/knowledge-commands.js'; @@ -15,6 +16,7 @@ import { renderMissingProjectMessage } from './doctor.js'; import { findNearestKtxProjectDir, resolveKtxProjectDir } from './project-resolver.js'; import { profileMark, profileSpan } from './startup-profile.js'; import type { CommandOutcome } from './telemetry/index.js'; +import { prepareUpdateCheckNotice, type PrepareUpdateCheckNoticeOptions } from './update-check/update-check.js'; profileMark('module:cli-program'); @@ -38,6 +40,8 @@ interface KtxCommanderProgramOptions { runInit: (args: { projectDir: string; force: boolean }, io: KtxCliIo) => Promise; } +type KtxCliUpdateCheckOptions = Pick; + export interface BuildKtxProgramOptions { io: KtxCliIo; deps: KtxCliDeps; @@ -46,6 +50,7 @@ export interface BuildKtxProgramOptions { setExitCode?: (code: number) => void; argv?: string[]; setTelemetryModule?: (telemetry: typeof import('./telemetry/index.js')) => void; + updateCheck?: KtxCliUpdateCheckOptions; } type CommanderExitLike = { exitCode: number; code: string; message: string }; @@ -430,11 +435,29 @@ export function collectCommandFlagsPresent(command: CommandUnknownOpts): Record< export function buildKtxProgram(options: BuildKtxProgramOptions): Command { const program = createBaseProgram(options.packageInfo, options.io); + let pendingUpdateNotice: string | null = null; + program.hook('preAction', async (_thisCommand, actionCommand) => { + // The hidden completion command must stay silent and side-effect free: skip + // the telemetry notice, command span, project checks, and update checks entirely. + if (commandPath(actionCommand as CommandPathNode).includes('__complete')) { + return; + } + const commandNode = actionCommand as CommandPathNode; + const updateCheck = await prepareUpdateCheckNotice({ + io: options.io, + env: options.updateCheck?.env, + fetchDistTags: options.updateCheck?.fetchDistTags, + homeDir: options.updateCheck?.homeDir, + installedVersion: options.packageInfo.version, + now: options.updateCheck?.now, + commandOptions: commandOptions(commandNode), + }); + pendingUpdateNotice = updateCheck.notice; + const telemetry = await import('./telemetry/index.js'); options.setTelemetryModule?.(telemetry); await telemetry.showTelemetryNoticeIfNeeded(options.io, options.packageInfo); - const commandNode = actionCommand as CommandPathNode; const path = commandPath(commandNode); const projectDir = resolveCommandProjectDir(commandNode); const hasProject = ktxYamlExists(projectDir); @@ -451,6 +474,13 @@ export function buildKtxProgram(options: BuildKtxProgramOptions): Command { ensureProjectAvailable(options.io, commandNode); }); + program.hook('postAction', () => { + if (pendingUpdateNotice) { + options.io.stderr.write(pendingUpdateNotice); + pendingUpdateNotice = null; + } + }); + const context: KtxCliCommandContext = { io: options.io, deps: options.deps, @@ -476,6 +506,7 @@ export function buildKtxProgram(options: BuildKtxProgramOptions): Command { registerStatusCommands(program, context); registerMcpCommands(program, context); registerAdminCommands(program, context); + registerCompletionCommands(program, context); return program; } @@ -522,6 +553,13 @@ export async function runCommanderKtxCli( try { return await runBareInteractiveCommand(program, io, context); } catch (error) { + const telemetry = await import('./telemetry/index.js'); + await telemetry.reportException({ + error, + context: { source: 'bare-interactive', handled: true, fatal: false }, + packageInfo: info, + io, + }); io.stderr.write(`${formatCliError(error)}\n`); return 1; } @@ -556,6 +594,23 @@ export async function runCommanderKtxCli( outcome: commandOutcomeForParseResult(parseError, exitCode), error: parseError, }); + if ( + parseError && + !isCommanderExit(parseError) && + !isKtxProjectMissingAbortError(parseError) + ) { + await telemetryModule.reportException({ + error: parseError, + context: { + source: completed?.commandPath.join(' ') ?? 'commander parseAsync', + handled: true, + fatal: false, + }, + projectDir: completed?.projectGroupAttached ? completed.projectDir : undefined, + packageInfo: info, + io, + }); + } await telemetryModule.emitCompletedCommand({ completed, packageInfo: info, io }); await telemetryModule.shutdownTelemetryEmitter(); } diff --git a/packages/cli/src/cli-runtime.ts b/packages/cli/src/cli-runtime.ts index 68089720..4e13b472 100644 --- a/packages/cli/src/cli-runtime.ts +++ b/packages/cli/src/cli-runtime.ts @@ -89,6 +89,88 @@ export async function runInitForCommander( return await runInit(args, io); } +function signalExitCode(signal: NodeJS.Signals): number { + // 128 + signal number: SIGINT (2) -> 130, SIGTERM (15) -> 143. + return signal === 'SIGTERM' ? 143 : 130; +} + +/** + * Flush telemetry on interrupt for the real CLI process. `capture()` is + * fire-and-forget and the only flush guarantee lives in a `finally` a signal + * skips, so Ctrl-C / `kill` of a long-running command (ingest, `mcp stdio`) + * would otherwise drop its `command` event and queued events. Installed only + * when driving the actual process; programmatic/test callers pass their own + * `io` and never reach here. Returns a disposer that removes the listeners. + */ +function installTelemetrySignalFlush(io: KtxCliIo, info: KtxCliPackageInfo): () => void { + let handling = false; + const handle = (signal: NodeJS.Signals): void => { + if (handling) { + process.exit(signalExitCode(signal)); + } + handling = true; + void (async () => { + try { + const { emitAbortedCommandAndShutdown } = await import('./telemetry/index.js'); + await emitAbortedCommandAndShutdown({ packageInfo: info, io }); + } catch { + // Best-effort: never let a telemetry hiccup block the interrupt exit. + } + process.exit(signalExitCode(signal)); + })(); + }; + const onSigint = (): void => handle('SIGINT'); + const onSigterm = (): void => handle('SIGTERM'); + process.on('SIGINT', onSigint); + process.on('SIGTERM', onSigterm); + return () => { + process.off('SIGINT', onSigint); + process.off('SIGTERM', onSigterm); + }; +} + +/** @internal */ +export function createGlobalExceptionReporter(io: KtxCliIo, info: KtxCliPackageInfo) { + return async (source: 'uncaughtException' | 'unhandledRejection', error: unknown): Promise => { + const { reportException, shutdownTelemetryEmitter } = await import('./telemetry/index.js'); + await reportException({ + error, + context: { source, handled: false, fatal: true }, + io, + packageInfo: info, + immediate: true, + }); + await shutdownTelemetryEmitter(); + }; +} + +export function installGlobalExceptionHandlers(io: KtxCliIo, info: KtxCliPackageInfo): () => void { + const report = createGlobalExceptionReporter(io, info); + const handle = (source: 'uncaughtException' | 'unhandledRejection', error: unknown): void => { + void (async () => { + try { + await report(source, error); + } catch { + // Best-effort: preserve Node's process termination behavior. + } + if (error instanceof Error && error.stack) { + io.stderr.write(`${error.stack}\n`); + } else { + io.stderr.write(`${String(error)}\n`); + } + process.exit(1); + })(); + }; + const onUncaught = (error: Error): void => handle('uncaughtException', error); + const onUnhandled = (reason: unknown): void => handle('unhandledRejection', reason); + process.on('uncaughtException', onUncaught); + process.on('unhandledRejection', onUnhandled); + return () => { + process.off('uncaughtException', onUncaught); + process.off('unhandledRejection', onUnhandled); + }; +} + export async function runKtxCli( argv = process.argv.slice(2), io: KtxCliIo = process, @@ -98,7 +180,17 @@ export async function runKtxCli( profileMark('runtime:runKtxCli'); const { runCommanderKtxCli } = await profileSpan('import ./cli-program.js', () => import('./cli-program.js')); - return await runCommanderKtxCli(argv, io, deps, info, { - runInit: runInitForCommander, - }); + // Real-process entry only: flush telemetry if interrupted. Test/programmatic + // callers pass their own `io`, so they never install process-level handlers. + const removeSignalFlush = (io as unknown) === process ? installTelemetrySignalFlush(io, info) : undefined; + const removeGlobalExceptionHandlers = + (io as unknown) === process ? installGlobalExceptionHandlers(io, info) : undefined; + try { + return await runCommanderKtxCli(argv, io, deps, info, { + runInit: runInitForCommander, + }); + } finally { + removeGlobalExceptionHandlers?.(); + removeSignalFlush?.(); + } } diff --git a/packages/cli/src/command-tree.ts b/packages/cli/src/command-tree.ts index 2eeb24e8..9b5bf729 100644 --- a/packages/cli/src/command-tree.ts +++ b/packages/cli/src/command-tree.ts @@ -16,7 +16,11 @@ export function walkCommandTree(command: CommandUnknownOpts): CommandTreeNode { description: command.description(), aliases: command.aliases(), arguments: command.registeredArguments.map(formatArgumentDeclaration), - children: command.commands.map((child) => walkCommandTree(child)), + // Internal commands (e.g. the shell-completion helper `__complete`) use a + // `__` prefix and are omitted from the human-facing command tree. + children: command.commands + .filter((child) => !child.name().startsWith('__')) + .map((child) => walkCommandTree(child)), }; } diff --git a/packages/cli/src/commands/completion-commands.ts b/packages/cli/src/commands/completion-commands.ts new file mode 100644 index 00000000..332f103b --- /dev/null +++ b/packages/cli/src/commands/completion-commands.ts @@ -0,0 +1,44 @@ +import { Argument, type Command } from '@commander-js/extra-typings'; +import type { KtxCliCommandContext } from '../cli-program.js'; +import { computeCompletions } from '../completion/complete-engine.js'; +import { completionScript } from '../completion/completion-scripts.js'; +import { createProjectCompletionProviders } from '../completion/dynamic-candidates.js'; +import { profileMark } from '../startup-profile.js'; + +profileMark('module:commands/completion-commands'); + +export function registerCompletionCommands(program: Command, context: KtxCliCommandContext): void { + program + .command('completion') + .description('Print a shell completion script for ktx') + .addArgument(new Argument('', 'Target shell').choices(['zsh', 'bash'])) + .addHelpText( + 'after', + '\nEnable completion by adding the matching line to your shell startup file:\n' + + ' zsh: eval "$(ktx completion zsh)"\n' + + ' bash: eval "$(ktx completion bash)"\n', + ) + .action((shell) => { + context.io.stdout.write(completionScript(shell)); + }); + + // Hidden command invoked by the generated shell scripts. It must only ever + // print newline-separated candidates to stdout and exit 0, so a TAB press is + // never disrupted by an error, a telemetry notice, or a parse failure. + program + .command('__complete', { hidden: true }) + .argument('[words...]') + .allowUnknownOption(true) + .helpOption(false) + .action(async (words: string[]) => { + try { + const candidates = await computeCompletions(program, words, createProjectCompletionProviders()); + if (candidates.length > 0) { + context.io.stdout.write(`${candidates.join('\n')}\n`); + } + } catch { + // Swallow: completion must never break the shell. + } + context.setExitCode(0); + }); +} diff --git a/packages/cli/src/commands/ingest-commands.ts b/packages/cli/src/commands/ingest-commands.ts index 9ffd2562..b5efe443 100644 --- a/packages/cli/src/commands/ingest-commands.ts +++ b/packages/cli/src/commands/ingest-commands.ts @@ -29,8 +29,6 @@ export function registerIngestCommands( .usage('[options] [connectionId]') .argument('[connectionId]', 'Configured connection id to ingest (omit to ingest all)') .option('--all', 'Ingest all configured connections', false) - .addOption(new Option('--fast', 'Use deterministic database schema ingest').conflicts('deep')) - .addOption(new Option('--deep', 'Use AI-enriched database ingest').conflicts('fast')) .addOption(new Option('--query-history', 'Include database query-history usage patterns').conflicts('noQueryHistory')) .addOption(new Option('--no-query-history', 'Skip database query-history usage patterns')) .option('--query-history-window-days ', 'Query-history lookback window for this run', parsePositiveIntegerOption) @@ -87,8 +85,6 @@ export function registerIngestCommands( all: selection.kind === 'all', json: options.json === true, inputMode: options.input === false ? 'disabled' : 'auto', - ...(options.fast === true ? { depth: 'fast' as const } : {}), - ...(options.deep === true ? { depth: 'deep' as const } : {}), queryHistory, ...(options.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: options.queryHistoryWindowDays } : {}), cliVersion: context.packageInfo.version, diff --git a/packages/cli/src/commands/knowledge-commands.ts b/packages/cli/src/commands/knowledge-commands.ts index c7b7c8d7..b601b688 100644 --- a/packages/cli/src/commands/knowledge-commands.ts +++ b/packages/cli/src/commands/knowledge-commands.ts @@ -21,9 +21,9 @@ function isDebugEnabled(command: CommandWithGlobalOptions): boolean { } export function registerWikiCommands(program: Command, context: KtxCliCommandContext): void { - program + const wiki = program .command('wiki') - .description('List or search local wiki pages') + .description('List, search, or read local wiki pages') .usage('[options] [query...]') .argument('[query...]', 'Search query; omit to list all pages') .option('--user-id ', 'Local user id', 'local') @@ -76,4 +76,18 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon }); }, ); + + wiki + .command('read') + .description('Read a wiki page file by key') + .argument('', 'Wiki page key') + .action(async (key: string, _options, command) => { + const parentOpts = command.parent?.opts() as { userId?: string } | undefined; + await runKnowledgeArgs(context, { + command: 'read', + projectDir: resolveCommandProjectDir(command), + key, + userId: parentOpts?.userId ?? 'local', + }); + }); } diff --git a/packages/cli/src/commands/setup-commands.ts b/packages/cli/src/commands/setup-commands.ts index 54628346..0302e9ed 100644 --- a/packages/cli/src/commands/setup-commands.ts +++ b/packages/cli/src/commands/setup-commands.ts @@ -29,7 +29,7 @@ function embeddingBackend(value: string): 'openai' | 'sentence-transformers' { } function llmBackend(value: string): KtxSetupLlmBackend { - if (value === 'anthropic' || value === 'vertex' || value === 'claude-code') { + if (value === 'anthropic' || value === 'vertex' || value === 'claude-code' || value === 'codex') { return value; } throw new InvalidArgumentError(`invalid choice '${value}'`); @@ -308,9 +308,14 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo .addOption(new Option('--source-git-url ', 'Git URL for dbt, MetricFlow, or LookML').hideHelp()) .addOption(new Option('--source-branch ', 'Git branch for source setup').hideHelp()) .addOption(new Option('--source-subpath ', 'Repo subpath for source setup').hideHelp()) - .addOption(new Option('--source-auth-token-ref ', 'env: or file: credential ref for source repo auth').hideHelp()) + .addOption( + new Option( + '--source-auth-token-ref ', + 'env: or file: credential ref for source repo auth or Notion integration token', + ).hideHelp(), + ) .addOption(new Option('--source-url ', 'Source service URL for Metabase or Looker').hideHelp()) - .addOption(new Option('--source-api-key-ref ', 'env: or file: API key ref for Metabase or Notion').hideHelp()) + .addOption(new Option('--source-api-key-ref ', 'env: or file: API key ref for Metabase').hideHelp()) .addOption(new Option('--source-client-id ', 'Looker client id').hideHelp()) .addOption(new Option('--source-client-secret-ref ', 'env: or file: Looker client secret ref').hideHelp()) .addOption(new Option('--source-warehouse-connection-id ', 'Mapped warehouse connection id').hideHelp()) @@ -401,6 +406,8 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo } const resolvedAgentScope = options.local ? 'local' : options.global ? 'global' : 'project'; + const debugEnabled = + ((command.optsWithGlobals ? command.optsWithGlobals() : command.opts()) as { debug?: unknown }).debug === true; await runSetupArgs(context, { command: 'run', projectDir: resolveCommandProjectDir(command), @@ -410,6 +417,7 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo agentScope: resolvedAgentScope, skipAgents: options.skipAgents === true, inputMode: options.input === false ? 'disabled' : 'auto', + ...(debugEnabled ? { debug: true } : {}), yes: options.yes === true, cliVersion: context.packageInfo.version, ...(options.llmBackend ? { llmBackend: options.llmBackend } : {}), diff --git a/packages/cli/src/commands/sl-commands.ts b/packages/cli/src/commands/sl-commands.ts index a4cb644c..8f2f05a3 100644 --- a/packages/cli/src/commands/sl-commands.ts +++ b/packages/cli/src/commands/sl-commands.ts @@ -94,19 +94,28 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte }, ); - sl.command('validate') - .description('Validate a semantic-layer source (set --connection-id on `ktx sl`)') + sl.command('read') + .description('Read a semantic-layer source YAML file') + .argument('', 'Semantic-layer source name') + .action(async (sourceName: string, _options, command) => { + const parentOpts = command.parent?.opts() as { connectionId?: string } | undefined; + await runSlArgs(context, { + command: 'read', + projectDir: resolveCommandProjectDir(command), + connectionId: parentOpts?.connectionId, + sourceName, + }); + }); + + sl.command('validate') + .description('Validate a semantic-layer source') .argument('', 'Semantic-layer source name') .action(async (sourceName: string, _options, command) => { const parentOpts = command.parent?.opts() as { connectionId?: string } | undefined; - const connectionId = parentOpts?.connectionId; - if (connectionId === undefined) { - command.error("error: required option '--connection-id ' not specified"); - } await runSlArgs(context, { command: 'validate', projectDir: resolveCommandProjectDir(command), - connectionId: connectionId as string, + connectionId: parentOpts?.connectionId, sourceName, }); }); @@ -131,10 +140,14 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte throw new Error('sl query requires at least one --measure'); } const parentOpts = command.parent?.opts() as { connectionId?: string } | undefined; + const connectionId = parentOpts?.connectionId; + if (connectionId === undefined) { + command.error("error: required option '--connection-id ' not specified"); + } const args = slQueryCommandSchema.parse({ command: 'query', projectDir: resolveCommandProjectDir(command), - connectionId: parentOpts?.connectionId, + connectionId, ...(options.queryFile ? { queryFile: options.queryFile } : { diff --git a/packages/cli/src/completion/complete-engine.ts b/packages/cli/src/completion/complete-engine.ts new file mode 100644 index 00000000..7268d397 --- /dev/null +++ b/packages/cli/src/completion/complete-engine.ts @@ -0,0 +1,172 @@ +import type { CommandUnknownOpts, Option } from '@commander-js/extra-typings'; + +/** + * Dynamic completion candidates that depend on project state (semantic-layer + * source names, wiki page keys, connection ids). Injected so the engine stays + * pure and unit-testable without touching the filesystem. + */ +export interface CompletionProviders { + /** Candidate operands for a positional argument of the active command path. */ + positionalCandidates(commandPath: string[], typedTokens: string[]): Promise; + /** Candidate values for an option that has no static `choices` (e.g. `--connection-id`). */ + optionValueCandidates(commandPath: string[], optionFlag: string, typedTokens: string[]): Promise; +} + +interface ResolvedCommand { + command: CommandUnknownOpts; + /** Subcommand names from the root down to the active command (root name excluded). */ + commandPath: string[]; +} + +function isHiddenCommand(command: CommandUnknownOpts): boolean { + // Completion mirrors `ktx --help`: commands registered with `{ hidden: true }` + // (the `__complete` helper and `mcp serve-internal`) are internal and must not + // surface. Commander exposes this only through the private `_hidden` field its + // own help renderer reads, so a name heuristic like a `__` prefix is not enough. + return (command as { _hidden?: boolean })._hidden === true; +} + +function resolveCommand(program: CommandUnknownOpts, typedTokens: string[]): ResolvedCommand { + let command: CommandUnknownOpts = program; + const commandPath: string[] = []; + for (let index = 0; index < typedTokens.length; index += 1) { + const token = typedTokens[index]; + if (token.startsWith('-')) { + // A value-taking option in the `--flag value` form consumes the next token + // as its value, so skip that value before matching subcommands. Otherwise a + // connection id like `query` would be resolved as the `sl query` subcommand + // instead of being treated as the `--connection-id` value. The `--flag=value` + // form carries its own value and consumes nothing extra. + if (!token.includes('=')) { + const option = findOption(command, token); + if (option && !option.isBoolean()) { + index += 1; + } + } + continue; + } + const sub = command.commands.find((candidate) => candidate.name() === token || candidate.aliases().includes(token)); + if (sub) { + command = sub; + commandPath.push(sub.name()); + } + } + return { command, commandPath }; +} + +function collectOptions(command: CommandUnknownOpts): Option[] { + const options: Option[] = []; + let current: CommandUnknownOpts | null = command; + while (current) { + options.push(...current.options); + current = current.parent; + } + return options; +} + +function findOption(command: CommandUnknownOpts, flag: string): Option | undefined { + return collectOptions(command).find((option) => option.long === flag || option.short === flag); +} + +function isRepeatableOption(option: Option): boolean { + // Variadic options, and options backed by a collector with an array default + // (e.g. `--measure`/`--dimension`), may be supplied more than once. + return option.variadic || Array.isArray(option.defaultValue); +} + +function flagCandidates(command: CommandUnknownOpts, typedTokens: string[]): string[] { + const present = new Set(typedTokens.filter((token) => token.startsWith('-'))); + const candidates: string[] = []; + for (const option of collectOptions(command)) { + if (option.hidden || !option.long) { + continue; + } + if (present.has(option.long) && !isRepeatableOption(option)) { + continue; + } + candidates.push(option.long); + } + return candidates; +} + +async function optionValueCandidates( + resolved: ResolvedCommand, + option: Option, + typedTokens: string[], + providers: CompletionProviders, +): Promise { + if (option.argChoices && option.argChoices.length > 0) { + return option.argChoices; + } + return providers.optionValueCandidates(resolved.commandPath, option.long ?? option.name(), typedTokens); +} + +function dedupeSortFilter(candidates: string[], partial: string): string[] { + const seen = new Set(); + const matches: string[] = []; + for (const candidate of candidates) { + if (!candidate.startsWith(partial) || seen.has(candidate)) { + continue; + } + seen.add(candidate); + matches.push(candidate); + } + return matches.sort(); +} + +/** + * Compute completion candidates for the partial last element of `words` + * (everything the shell has on the line after `ktx`). The active command and + * its flags are derived by walking the live Commander tree, so completion never + * drifts from the real command structure. + */ +export async function computeCompletions( + program: CommandUnknownOpts, + words: string[], + providers: CompletionProviders, +): Promise { + const partial = words.length > 0 ? (words[words.length - 1] ?? '') : ''; + const typedTokens = words.slice(0, -1); + const resolved = resolveCommand(program, typedTokens); + + // (a) Option value via the `--opt=value` form. + const equalsMatch = /^(--[^=]+)=(.*)$/.exec(partial); + if (equalsMatch) { + const [, flag, valuePartial] = equalsMatch; + const option = findOption(resolved.command, flag); + if (!option || option.isBoolean()) { + return []; + } + const values = await optionValueCandidates(resolved, option, typedTokens, providers); + return dedupeSortFilter( + values.map((value) => `${flag}=${value}`), + `${flag}=${valuePartial}`, + ); + } + + // (b) Option value via the `--opt value` form (previous token is a value-taking option). + const previous = typedTokens[typedTokens.length - 1]; + if (previous && previous.startsWith('-') && !partial.startsWith('-')) { + const option = findOption(resolved.command, previous); + if (option && !option.isBoolean()) { + return dedupeSortFilter(await optionValueCandidates(resolved, option, typedTokens, providers), partial); + } + } + + // (c) Flag completion. + if (partial.startsWith('-')) { + return dedupeSortFilter(flagCandidates(resolved.command, typedTokens), partial); + } + + // (d) Positional: subcommand names union static argument choices union dynamic operand candidates. + const candidates: string[] = resolved.command.commands + .filter((sub) => !isHiddenCommand(sub)) + .map((sub) => sub.name()); + for (const argument of resolved.command.registeredArguments) { + if (argument.argChoices) { + candidates.push(...argument.argChoices); + } + } + candidates.push(...(await providers.positionalCandidates(resolved.commandPath, typedTokens))); + return dedupeSortFilter(candidates, partial); +} diff --git a/packages/cli/src/completion/completion-scripts.ts b/packages/cli/src/completion/completion-scripts.ts new file mode 100644 index 00000000..5761c6e0 --- /dev/null +++ b/packages/cli/src/completion/completion-scripts.ts @@ -0,0 +1,39 @@ +// Static shell completion scripts emitted by `ktx completion `. +// +// Both scripts gather the words on the current command line (excluding the +// leading `ktx`), append the partial word under the cursor, and delegate to the +// hidden `ktx __complete` command, which prints newline-separated candidates. +// All command/flag/entity knowledge lives in `ktx __complete` so these scripts +// never have to encode the command tree. +// +// Lines are single-quoted JS strings so the shell `${...}` expansions are +// emitted verbatim (a template literal would try to interpolate them). + +const ZSH_SCRIPT = [ + '#compdef ktx', + '_ktx() {', + ' local -a candidates', + ' local out', + ' out="$(ktx __complete -- "${words[@]:1:$((CURRENT-1))}" 2>/dev/null)" || return 0', + ' candidates=("${(@f)out}")', + ' compadd -- $candidates', + '}', + 'compdef _ktx ktx', + '', +].join('\n'); + +const BASH_SCRIPT = [ + '_ktx() {', + ' local cur out', + ' cur="${COMP_WORDS[COMP_CWORD]}"', + ' out="$(ktx __complete -- "${COMP_WORDS[@]:1:COMP_CWORD}" 2>/dev/null)" || { COMPREPLY=(); return 0; }', + " local IFS=$'\\n'", + ' COMPREPLY=($(compgen -W "${out}" -- "$cur"))', + '}', + 'complete -F _ktx ktx', + '', +].join('\n'); + +export function completionScript(shell: 'zsh' | 'bash'): string { + return shell === 'zsh' ? ZSH_SCRIPT : BASH_SCRIPT; +} diff --git a/packages/cli/src/completion/dynamic-candidates.ts b/packages/cli/src/completion/dynamic-candidates.ts new file mode 100644 index 00000000..2be512c9 --- /dev/null +++ b/packages/cli/src/completion/dynamic-candidates.ts @@ -0,0 +1,103 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import type { KtxLocalProject } from '../context/project/project.js'; +import { resolveKtxProjectDir } from '../project-resolver.js'; +import type { CompletionProviders } from './complete-engine.js'; + +/** Extract an option value from already-typed tokens (`--flag value` or `--flag=value`). */ +function extractOptionValue(tokens: string[], flag: string): string | undefined { + const prefix = `${flag}=`; + for (let index = 0; index < tokens.length; index += 1) { + const token = tokens[index]; + if (token === flag) { + const next = tokens[index + 1]; + if (next !== undefined && !next.startsWith('-')) { + return next; + } + } else if (token.startsWith(prefix)) { + return token.slice(prefix.length); + } + } + return undefined; +} + +/** + * Resolve and load the project the user is completing against. Honors a + * `--project-dir` typed on the line, then `KTX_PROJECT_DIR`, then the nearest + * `ktx.yaml`. Returns null (no completions) when there is no project, without + * creating any files. + */ +async function loadCompletionProject(typedTokens: string[]): Promise { + const explicitProjectDir = extractOptionValue(typedTokens, '--project-dir'); + const projectDir = resolveKtxProjectDir(explicitProjectDir !== undefined ? { explicitProjectDir } : {}); + if (!existsSync(join(projectDir, 'ktx.yaml'))) { + return null; + } + const { loadKtxProject } = await import('../context/project/project.js'); + return loadKtxProject({ projectDir }); +} + +async function sourceNames(typedTokens: string[]): Promise { + const project = await loadCompletionProject(typedTokens); + if (!project) { + return []; + } + const connectionId = extractOptionValue(typedTokens, '--connection-id'); + const { listLocalSlSources } = await import('../context/sl/local-sl.js'); + const summaries = await listLocalSlSources(project, connectionId !== undefined ? { connectionId } : {}); + return [...new Set(summaries.map((summary) => summary.name))]; +} + +async function wikiPageKeys(typedTokens: string[]): Promise { + const project = await loadCompletionProject(typedTokens); + if (!project) { + return []; + } + const userId = extractOptionValue(typedTokens, '--user-id'); + const { listLocalKnowledgePageKeys } = await import('../context/wiki/local-knowledge.js'); + return listLocalKnowledgePageKeys(project, userId !== undefined ? { userId } : {}); +} + +async function connectionIds(typedTokens: string[]): Promise { + const project = await loadCompletionProject(typedTokens); + if (!project) { + return []; + } + return Object.keys(project.config.connections).sort(); +} + +/** + * Project-backed completion providers. Every entry swallows its own errors so a + * failed lookup never breaks the shell — completion degrades to commands/flags. + */ +export function createProjectCompletionProviders(): CompletionProviders { + return { + async positionalCandidates(commandPath, typedTokens) { + try { + const key = commandPath.join(' '); + if (key === 'sl read' || key === 'sl validate') { + return await sourceNames(typedTokens); + } + if (key === 'wiki read') { + return await wikiPageKeys(typedTokens); + } + if (key === 'connection test' || key === 'ingest') { + return await connectionIds(typedTokens); + } + return []; + } catch { + return []; + } + }, + async optionValueCandidates(_commandPath, optionFlag, typedTokens) { + try { + if (optionFlag === '--connection-id' || optionFlag === '--connection') { + return await connectionIds(typedTokens); + } + return []; + } catch { + return []; + } + }, + }; +} diff --git a/packages/cli/src/connection-drivers.ts b/packages/cli/src/connection-drivers.ts new file mode 100644 index 00000000..4f10e663 --- /dev/null +++ b/packages/cli/src/connection-drivers.ts @@ -0,0 +1,21 @@ +import type { KtxProjectConnectionConfig } from './context/project/config.js'; + +const KTX_DATABASE_DRIVER_IDS = new Set([ + 'sqlite', + 'postgres', + 'mysql', + 'clickhouse', + 'sqlserver', + 'bigquery', + 'snowflake', +]); + +export function normalizeConnectionDriver(connection: KtxProjectConnectionConfig): string { + return String(connection.driver ?? '') + .trim() + .toLowerCase(); +} + +export function isDatabaseDriver(driver: string): boolean { + return KTX_DATABASE_DRIVER_IDS.has(driver.trim().toLowerCase()); +} diff --git a/packages/cli/src/connection-recovery.ts b/packages/cli/src/connection-recovery.ts new file mode 100644 index 00000000..2cd87448 --- /dev/null +++ b/packages/cli/src/connection-recovery.ts @@ -0,0 +1,132 @@ +import type { KtxCliIo } from './cli-runtime.js'; +import type { KtxSetupPromptOption } from './setup-prompts.js'; + +export type RecoveryOutcome = 'ready' | 'skip' | 'back' | 'failed'; + +/** @internal */ +export interface RecoveryAction { + value: string; + label: string; + run: () => Promise; +} + +export type ConfigureResult = 'configured' | 'back' | 'cancelled'; + +export type ValidateResult = + | { status: 'ok' } + | { status: 'back' } + | { status: 'failed'; extraActions?: RecoveryAction[] }; + +export interface ConnectionRecoveryInput { + label: string; + interactive: boolean; + allowSkip: boolean; + io: KtxCliIo; + prompts: { + select(options: { message: string; options: KtxSetupPromptOption[] }): Promise; + }; + snapshot: () => Promise<() => Promise>; + configure: () => Promise; + validate: () => Promise; +} + +async function runRollbackOnce(input: { + rollback: () => Promise; + state: { rolledBack: boolean }; +}): Promise { + if (input.state.rolledBack) { + return; + } + input.state.rolledBack = true; + await input.rollback(); +} + +function recoveryOptions(input: { + allowSkip: boolean; + extraActions?: RecoveryAction[]; +}): KtxSetupPromptOption[] { + return [ + { value: 'retry', label: 'Retry connection test' }, + { value: 're-enter', label: 'Re-enter connection details' }, + ...(input.extraActions ?? []).map((action) => ({ + value: action.value, + label: action.label, + })), + ...(input.allowSkip ? [{ value: 'skip', label: 'Skip this connection' }] : []), + { value: 'back', label: 'Back' }, + ]; +} + +export async function runConnectionSetupWithRecovery( + input: ConnectionRecoveryInput, +): Promise { + const rollback = await input.snapshot(); + const rollbackState = { rolledBack: false }; + + const firstConfig = await input.configure(); + if (firstConfig === 'back') { + await runRollbackOnce({ rollback, state: rollbackState }); + return 'back'; + } + if (firstConfig === 'cancelled') { + await runRollbackOnce({ rollback, state: rollbackState }); + return 'failed'; + } + + let validation = await input.validate(); + while (validation.status !== 'ok') { + if (validation.status === 'back') { + await runRollbackOnce({ rollback, state: rollbackState }); + return 'back'; + } + + if (!input.interactive) { + return 'failed'; + } + + const action = await input.prompts.select({ + message: `Connection setup failed for ${input.label}`, + options: recoveryOptions({ + allowSkip: input.allowSkip, + extraActions: validation.extraActions, + }), + }); + + if (action === 'back') { + await runRollbackOnce({ rollback, state: rollbackState }); + return 'back'; + } + if (action === 'skip' && input.allowSkip) { + await runRollbackOnce({ rollback, state: rollbackState }); + return 'skip'; + } + if (action === 're-enter') { + const nextConfig = await input.configure(); + if (nextConfig === 'back') { + await runRollbackOnce({ rollback, state: rollbackState }); + return 'back'; + } + if (nextConfig === 'cancelled') { + await runRollbackOnce({ rollback, state: rollbackState }); + return 'failed'; + } + validation = await input.validate(); + continue; + } + if (action === 'retry') { + validation = await input.validate(); + continue; + } + + const extraAction = validation.extraActions?.find((candidate) => candidate.value === action); + if (extraAction) { + await extraAction.run(); + validation = await input.validate(); + continue; + } + + validation = await input.validate(); + } + + return 'ready'; +} diff --git a/packages/cli/src/connection.ts b/packages/cli/src/connection.ts index bb99d4fd..9b6b4294 100644 --- a/packages/cli/src/connection.ts +++ b/packages/cli/src/connection.ts @@ -6,6 +6,7 @@ import { type NotionBotInfo, NotionClient } from './context/ingest/adapters/noti import { createLocalLookerCredentialResolver } from './context/ingest/adapters/looker/local-looker.adapter.js'; import { metabaseRuntimeConfigFromLocalConnection } from './context/ingest/adapters/metabase/local-metabase.adapter.js'; import { testRepoConnection } from './context/ingest/repo-fetch.js'; +import { getDriverRegistration } from './context/connections/drivers.js'; import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from './context/connections/notion-config.js'; import { resolveKtxConfigReference } from './context/core/config-reference.js'; import { type KtxLocalProject, loadKtxProject } from './context/project/project.js'; @@ -15,8 +16,9 @@ import { bold, dim, green, red, SYMBOLS } from './io/symbols.js'; import { createKtxCliScanConnector } from './local-scan-connectors.js'; import { profileMark } from './startup-profile.js'; import { isDemoConnection } from './telemetry/demo-detect.js'; -import { emitTelemetryEvent } from './telemetry/index.js'; -import { scrubErrorClass } from './telemetry/scrubber.js'; +import { emitTelemetryEvent, reportException } from './telemetry/index.js'; +import { collectTelemetryRedactionSecrets } from './telemetry/redaction-secrets.js'; +import { formatErrorDetail, scrubErrorClass } from './telemetry/scrubber.js'; profileMark('module:connection'); @@ -73,6 +75,12 @@ async function testNativeConnection( } const result = await connector.testConnection(); if (!result.success) { + // Re-throw the driver's original error so connection_test telemetry records + // its real class (e.g. ConnectionError) and code (e.g. ELOGIN) instead of + // collapsing every native failure to a generic Error with no code. + if (result.cause instanceof Error) { + throw result.cause; + } throw new Error(result.error ?? 'connection test failed'); } return { driver: connector.driver }; @@ -272,17 +280,7 @@ async function testConnectionByDriver( return { driver, detailKey: 'Repo', detailValue: result.repoUrl }; } - if ( - driver === 'sqlite' || - driver === 'sqlite3' || - driver === 'postgres' || - driver === 'postgresql' || - driver === 'mysql' || - driver === 'clickhouse' || - driver === 'sqlserver' || - driver === 'bigquery' || - driver === 'snowflake' - ) { + if (getDriverRegistration(driver)) { const result = await testNativeConnection( project, connectionId, @@ -313,6 +311,7 @@ async function emitConnectionTest(input: { io: KtxCliIo; }): Promise { const errorClass = input.error ? scrubErrorClass(input.error) : undefined; + const errorDetail = input.error ? formatErrorDetail(input.error) : undefined; await emitTelemetryEvent({ name: 'connection_test', projectDir: input.project.projectDir, @@ -323,8 +322,24 @@ async function emitConnectionTest(input: { outcome: input.outcome, durationMs: input.durationMs, ...(errorClass ? { errorClass } : {}), + ...(errorDetail ? { errorDetail } : {}), }, }); + if (input.error) { + await reportException({ + error: input.error, + context: { source: 'connection test', handled: true, fatal: false }, + projectDir: input.project.projectDir, + io: input.io, + redactionSecrets: await collectTelemetryRedactionSecrets({ + project: input.project, + connectionId: input.connectionId, + includeLlm: false, + includeEmbeddings: false, + env: process.env, + }), + }); + } } function visualWidth(text: string): number { diff --git a/packages/cli/src/connectors/bigquery/connector.ts b/packages/cli/src/connectors/bigquery/connector.ts index 7810e251..eae0f2ed 100644 --- a/packages/cli/src/connectors/bigquery/connector.ts +++ b/packages/cli/src/connectors/bigquery/connector.ts @@ -1,12 +1,34 @@ import { BigQuery, type TableField } from '@google-cloud/bigquery'; import { normalizeBigQueryProjectId, normalizeBigQueryRegion } from '../../context/connections/bigquery-identifiers.js'; +import { getDialectForDriver } from '../../context/connections/dialects.js'; import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; -import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js'; +import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js'; import { scopedTableNames } from '../../context/scan/table-ref.js'; +import { + connectorTestFailure, + createKtxConnectorCapabilities, + type KtxConnectorTestResult, + type KtxColumnSampleInput, + type KtxColumnSampleResult, + type KtxColumnStatsInput, + type KtxColumnStatsResult, + type KtxQueryResult, + type KtxReadOnlyQueryInput, + type KtxScanConnector, + type KtxScanContext, + type KtxScanInput, + type KtxScanWarning, + type KtxSchemaColumn, + type KtxSchemaSnapshot, + type KtxSchemaTable, + type KtxTableListEntry, + type KtxTableRef, + type KtxTableSampleInput, + type KtxTableSampleResult, +} from '../../context/scan/types.js'; import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; -import { KtxBigQueryDialect } from './dialect.js'; export interface KtxBigQueryConnectionConfig { driver?: string; @@ -185,6 +207,17 @@ function firstNumber(value: unknown): number | null { return Number.isFinite(numberValue) ? numberValue : null; } +function isDeniedError(error: unknown): boolean { + if (!error || typeof error !== 'object') { + return false; + } + const candidate = error as { code?: unknown; errors?: Array<{ reason?: unknown }> }; + return ( + candidate.code === 403 || + candidate.errors?.some((item) => item.reason === 'accessDenied' || item.reason === 'notFound') === true + ); +} + function normalizeValue(value: unknown): unknown { if (value === null || value === undefined) { return null; @@ -204,6 +237,23 @@ function normalizeValue(value: unknown): unknown { return value; } +/** @internal */ +export function prepareBigQueryReadOnlyQuery( + sql: string, + params?: Record, +): { sql: string; params?: Record } { + if (!params) { + return { sql, params: undefined }; + } + let processedSql = sql; + const processedParams: Record = {}; + for (const [key, value] of Object.entries(params)) { + processedSql = processedSql.replace(new RegExp(`:${key}\\b`, 'g'), `@${key}`); + processedParams[key] = value; + } + return { sql: processedSql, params: Object.keys(processedParams).length > 0 ? processedParams : undefined }; +} + export function isKtxBigQueryConnectionConfig( connection: KtxBigQueryConnectionConfig | undefined, ): connection is KtxBigQueryConnectionConfig { @@ -255,7 +305,7 @@ export class KtxBigQueryScanConnector implements KtxScanConnector { private readonly now: () => Date; private readonly maxBytesBilled?: number | string; private readonly queryTimeoutMs?: number; - private readonly dialect = new KtxBigQueryDialect(); + private readonly dialect = getDialectForDriver('bigquery'); private client: KtxBigQueryClient | null = null; constructor(options: KtxBigQueryScanConnectorOptions) { @@ -272,7 +322,7 @@ export class KtxBigQueryScanConnector implements KtxScanConnector { this.id = `bigquery:${options.connectionId}`; } - async testConnection(): Promise<{ success: boolean; error?: string }> { + async testConnection(): Promise { try { const client = this.getClient(); await client.getDatasets({ maxResults: 1 }); @@ -281,7 +331,7 @@ export class KtxBigQueryScanConnector implements KtxScanConnector { } return { success: true }; } catch (error) { - return { success: false, error: error instanceof Error ? error.message : String(error) }; + return connectorTestFailure(error); } } @@ -289,11 +339,12 @@ export class KtxBigQueryScanConnector implements KtxScanConnector { this.assertConnection(input.connectionId); const tables: KtxSchemaTable[] = []; const datasetIds = this.requireDatasetIdsForScan(); + const snapshotWarnings: KtxScanWarning[] = []; for (const datasetId of datasetIds) { const scopedNames = input.tableScope ? scopedTableNames(input.tableScope, { catalog: this.resolved.projectId, db: datasetId }) : null; - tables.push(...(await this.introspectDataset(datasetId, scopedNames))); + tables.push(...(await this.introspectDataset(datasetId, scopedNames, snapshotWarnings))); } return { connectionId: this.connectionId, @@ -307,6 +358,7 @@ export class KtxBigQueryScanConnector implements KtxScanConnector { total_columns: tables.reduce((sum, table) => sum + table.columns.length, 0), }, tables, + warnings: snapshotWarnings, }; } @@ -331,7 +383,7 @@ export class KtxBigQueryScanConnector implements KtxScanConnector { async executeReadOnly(input: KtxBigQueryReadOnlyQueryInput, _ctx: KtxScanContext): Promise { this.assertConnection(input.connectionId); const limitedSql = limitSqlForExecution(assertReadOnlySql(input.sql), input.maxRows); - const prepared = this.dialect.prepareQuery(limitedSql, input.params); + const prepared = prepareBigQueryReadOnlyQuery(limitedSql, input.params); const result = await this.query(prepared.sql, prepared.params); return { ...result, rowCount: result.rows.length }; } @@ -366,7 +418,7 @@ export class KtxBigQueryScanConnector implements KtxScanConnector { if (!datasetId) { return 0; } - const tables = await this.introspectDataset(datasetId, null); + const tables = await this.introspectDataset(datasetId, null, []); return tables.find((table) => table.name === tableName)?.estimatedRows ?? 0; } @@ -378,7 +430,7 @@ export class KtxBigQueryScanConnector implements KtxScanConnector { return this.dialect.quoteIdentifier(identifier); } - async listDatasets(): Promise { + async listSchemas(): Promise { const [datasets] = await this.getClient().getDatasets(); return datasets.map((dataset) => dataset.id).filter((id): id is string => Boolean(id)); } @@ -404,6 +456,7 @@ export class KtxBigQueryScanConnector implements KtxScanConnector { params, ); return rows.map((row) => ({ + catalog: this.resolved.projectId, schema: row.table_schema, name: row.table_name, kind: @@ -467,13 +520,24 @@ export class KtxBigQueryScanConnector implements KtxScanConnector { return firstNumber(rows[0]?.[header]); } - private async introspectDataset(datasetId: string, scopedNames: readonly string[] | null): Promise { + private async introspectDataset( + datasetId: string, + scopedNames: readonly string[] | null, + snapshotWarnings: KtxScanWarning[], + ): Promise { if (scopedNames && scopedNames.length === 0) return []; const dataset = this.getClient().dataset(datasetId); const [tableRefs] = await dataset.getTables(); const scopeSet = scopedNames ? new Set(scopedNames) : null; const filteredTableRefs = scopeSet ? tableRefs.filter((tableRef) => scopeSet.has(tableRef.id ?? '')) : tableRefs; - const primaryKeys = await this.primaryKeys(datasetId); + const primaryKeysResult = await tryConstraintQuery( + { schema: datasetId, kind: 'primary_key', isDeniedError }, + () => this.primaryKeys(datasetId), + ); + const primaryKeys = primaryKeysResult.ok ? primaryKeysResult.value : new Map>(); + if (!primaryKeysResult.ok) { + snapshotWarnings.push(primaryKeysResult.warning); + } const tables: KtxSchemaTable[] = []; for (const tableRef of filteredTableRefs) { const tableName = tableRef.id || ''; diff --git a/packages/cli/src/connectors/bigquery/dialect.ts b/packages/cli/src/connectors/bigquery/dialect.ts index 02d904ed..0e2f883e 100644 --- a/packages/cli/src/connectors/bigquery/dialect.ts +++ b/packages/cli/src/connectors/bigquery/dialect.ts @@ -1,9 +1,18 @@ +import type { KtxDialect } from '../../context/connections/dialects.js'; +import { + columnDisplayPartCount, + formatDialectDisplayRef, + formatDialectTableName, + limitOffsetClause, + parseDialectDisplayRef, +} from '../../context/connections/dialect-helpers.js'; import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type BigQueryTableNameRef = Pick & Partial>; -export class KtxBigQueryDialect { - readonly type = 'bigquery'; +/** @internal */ +export class KtxBigQueryDialect implements KtxDialect { + readonly type = 'bigquery' as const; private readonly typeMappings: Record = { TIMESTAMP: 'time', @@ -27,13 +36,19 @@ export class KtxBigQueryDialect { } formatTableName(table: BigQueryTableNameRef): string { - if (table.catalog && table.db) { - return `${this.quoteIdentifier(table.catalog)}.${this.quoteIdentifier(table.db)}.${this.quoteIdentifier(table.name)}`; - } - if (table.db) { - return `${this.quoteIdentifier(table.db)}.${this.quoteIdentifier(table.name)}`; - } - return this.quoteIdentifier(table.name); + return formatDialectTableName(table, this.quoteIdentifier.bind(this), 'three-part'); + } + + formatDisplayRef(table: BigQueryTableNameRef): string { + return formatDialectDisplayRef(table, 'three-part'); + } + + parseDisplayRef(display: string): KtxTableRef | null { + return parseDialectDisplayRef(display, 'three-part'); + } + + columnDisplayTablePartCount(): 1 | 2 | 3 { + return columnDisplayPartCount('three-part'); } mapDataType(nativeType: string): string { @@ -93,19 +108,6 @@ export class KtxBigQueryDialect { return `SELECT ${quotedColumn} FROM ${tableName} WHERE ${quotedColumn} IS NOT NULL AND TRIM(CAST(${quotedColumn} AS STRING)) != '' ORDER BY RAND() LIMIT ${limit}`; } - prepareQuery(sql: string, params?: Record): { sql: string; params?: Record } { - if (!params) { - return { sql, params: undefined }; - } - let processedSql = sql; - const processedParams: Record = {}; - for (const [key, value] of Object.entries(params)) { - processedSql = processedSql.replace(new RegExp(`:${key}\\b`, 'g'), `@${key}`); - processedParams[key] = value; - } - return { sql: processedSql, params: Object.keys(processedParams).length > 0 ? processedParams : undefined }; - } - getRandomSampleFilter(samplePct: number): string { if (samplePct <= 0 || samplePct >= 1) { return ''; @@ -121,7 +123,11 @@ export class KtxBigQueryDialect { } getLimitOffsetClause(limit: number, offset?: number): string { - return offset !== undefined && offset > 0 ? `LIMIT ${limit} OFFSET ${offset}` : `LIMIT ${limit}`; + return limitOffsetClause(limit, offset); + } + + getTopClause(_limit: number): string { + return ''; } getNullCountExpression(column: string): string { @@ -132,6 +138,18 @@ export class KtxBigQueryDialect { return `APPROX_COUNT_DISTINCT(${column})`; } + textLengthExpression(columnSql: string): string { + return `LENGTH(CAST(${columnSql} AS STRING))`; + } + + castToText(columnSql: string): string { + return `CAST(${columnSql} AS STRING)`; + } + + getSampleValueAggregation(innerSql: string): string { + return `(SELECT STRING_AGG(CAST(value AS STRING), '\\u001F') FROM (${innerSql}) AS relationship_profile_values)`; + } + generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string { return ` WITH sampled AS ( @@ -172,36 +190,4 @@ export class KtxBigQueryDialect { FROM sampled `; } - - getTimeTruncExpression( - column: string, - granularity: 'day' | 'week' | 'month' | 'quarter' | 'year', - timezone?: string, - ): string { - const bigQueryGranularity = granularity.toUpperCase(); - if (timezone) { - return `DATE_TRUNC(DATETIME(${column}, '${timezone}'), ${bigQueryGranularity})`; - } - return `DATE_TRUNC(${column}, ${bigQueryGranularity})`; - } - - getCustomTimeTruncExpression(column: string, interval: string, origin?: string, timezone?: string): string { - const col = timezone ? `DATETIME(${column}, '${timezone}')` : column; - const [rawAmount, rawUnit] = interval.split(' '); - let diffUnit = rawUnit!.toUpperCase(); - let amount = Number(rawAmount); - let addUnit = diffUnit; - if (diffUnit === 'WEEK') { - diffUnit = 'DAY'; - amount = amount * 7; - addUnit = 'DAY'; - } - const originExpr = origin ? `TIMESTAMP '${origin}'` : `TIMESTAMP '1970-01-01'`; - return `TIMESTAMP_ADD(${originExpr}, INTERVAL CAST(FLOOR(TIMESTAMP_DIFF(${col}, ${originExpr}, ${diffUnit}) / ${amount}) * ${amount} AS INT64) ${addUnit})`; - } - - parseIntervalToSql(interval: string): string { - const [amount, unit] = interval.split(' '); - return `INTERVAL ${amount} ${unit!.toUpperCase()}`; - } } diff --git a/packages/cli/src/connectors/clickhouse/connector.ts b/packages/cli/src/connectors/clickhouse/connector.ts index a2ee568c..23622701 100644 --- a/packages/cli/src/connectors/clickhouse/connector.ts +++ b/packages/cli/src/connectors/clickhouse/connector.ts @@ -1,12 +1,12 @@ import { createClient } from '@clickhouse/client'; +import { getDialectForDriver } from '../../context/connections/dialects.js'; import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; -import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js'; +import { connectorTestFailure, createKtxConnectorCapabilities, type KtxConnectorTestResult, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js'; import { scopedTableNames } from '../../context/scan/table-ref.js'; import { readFileSync } from 'node:fs'; import { Agent as HttpsAgent } from 'node:https'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; -import { KtxClickHouseDialect } from './dialect.js'; export interface KtxClickHouseConnectionConfig { driver?: string; @@ -198,6 +198,49 @@ function clickHouseTableKey(database: string, table: string): string { return `${database}.${table}`; } +function inferClickHouseQueryParamType(value: unknown): string { + if (value === null || value === undefined) { + return 'String'; + } + if (typeof value === 'boolean') { + return 'Bool'; + } + if (typeof value === 'number') { + return Number.isInteger(value) ? 'Int64' : 'Float64'; + } + if (value instanceof Date) { + return 'DateTime'; + } + return 'String'; +} + +/** @internal */ +export function prepareClickHouseReadOnlyQuery( + sql: string, + params?: Record, +): { sql: string; params?: Record } { + if (!params) { + return { sql, params: undefined }; + } + + let parameterizedQuery = sql; + const queryParams: Record = {}; + const sortedKeys = Object.keys(params).sort((a, b) => b.length - a.length); + + for (const key of sortedKeys) { + const placeholder = `:${key}`; + if (parameterizedQuery.includes(placeholder)) { + parameterizedQuery = parameterizedQuery.replace( + new RegExp(`:${key}\\b`, 'g'), + `{${key}:${inferClickHouseQueryParamType(params[key])}}`, + ); + queryParams[key] = params[key]; + } + } + + return { sql: parameterizedQuery, params: Object.keys(queryParams).length > 0 ? queryParams : undefined }; +} + export function isKtxClickHouseConnectionConfig( connection: KtxClickHouseConnectionConfig | undefined, ): connection is KtxClickHouseConnectionConfig { @@ -256,7 +299,7 @@ export class KtxClickHouseScanConnector implements KtxScanConnector { private readonly clientFactory: KtxClickHouseClientFactory; private readonly endpointResolver?: KtxClickHouseEndpointResolver; private readonly now: () => Date; - private readonly dialect = new KtxClickHouseDialect(); + private readonly dialect = getDialectForDriver('clickhouse'); private client: KtxClickHouseClient | null = null; private resolvedEndpoint: KtxClickHouseResolvedEndpoint | null = null; @@ -274,12 +317,12 @@ export class KtxClickHouseScanConnector implements KtxScanConnector { this.id = `clickhouse:${options.connectionId}`; } - async testConnection(): Promise<{ success: boolean; error?: string }> { + async testConnection(): Promise { try { await this.query('SELECT 1'); return { success: true }; } catch (error) { - return { success: false, error: error instanceof Error ? error.message : String(error) }; + return connectorTestFailure(error); } } @@ -408,7 +451,7 @@ export class KtxClickHouseScanConnector implements KtxScanConnector { async executeReadOnly(input: KtxClickHouseReadOnlyQueryInput, _ctx: KtxScanContext): Promise { this.assertConnection(input.connectionId); const limitedSql = limitSqlForExecution(assertReadOnlySql(input.sql), input.maxRows); - const prepared = this.dialect.prepareQuery(limitedSql, input.params); + const prepared = prepareClickHouseReadOnlyQuery(limitedSql, input.params); const result = await this.query(prepared.sql, prepared.params); return { ...result, rowCount: result.rows.length }; } @@ -488,6 +531,7 @@ export class KtxClickHouseScanConnector implements KtxScanConnector { { schemas: filterSchemas }, ); return rows.map((row) => ({ + catalog: null, schema: row.database, name: row.name, kind: row.engine === 'View' || row.engine === 'MaterializedView' ? ('view' as const) : ('table' as const), diff --git a/packages/cli/src/connectors/clickhouse/dialect.ts b/packages/cli/src/connectors/clickhouse/dialect.ts index 48452ea6..9e470cae 100644 --- a/packages/cli/src/connectors/clickhouse/dialect.ts +++ b/packages/cli/src/connectors/clickhouse/dialect.ts @@ -1,9 +1,18 @@ +import type { KtxDialect } from '../../context/connections/dialects.js'; +import { + columnDisplayPartCount, + formatDialectDisplayRef, + formatDialectTableName, + limitOffsetClause, + parseDialectDisplayRef, +} from '../../context/connections/dialect-helpers.js'; import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type ClickHouseTableNameRef = Pick & Partial>; -export class KtxClickHouseDialect { - readonly type = 'clickhouse'; +/** @internal */ +export class KtxClickHouseDialect implements KtxDialect { + readonly type = 'clickhouse' as const; private readonly typeMappings: Record = { date: 'time', @@ -45,9 +54,19 @@ export class KtxClickHouseDialect { } formatTableName(table: ClickHouseTableNameRef): string { - return table.db - ? `${this.quoteIdentifier(table.db)}.${this.quoteIdentifier(table.name)}` - : this.quoteIdentifier(table.name); + return formatDialectTableName(table, this.quoteIdentifier.bind(this), 'ansi'); + } + + formatDisplayRef(table: ClickHouseTableNameRef): string { + return formatDialectDisplayRef(table, 'ansi'); + } + + parseDisplayRef(display: string): KtxTableRef | null { + return parseDialectDisplayRef(display, 'ansi'); + } + + columnDisplayTablePartCount(): 1 | 2 | 3 { + return columnDisplayPartCount('ansi'); } mapDataType(nativeType: string): string { @@ -97,29 +116,6 @@ export class KtxClickHouseDialect { return `SELECT ${quotedColumn} FROM ${tableName} WHERE ${quotedColumn} IS NOT NULL AND trim(toString(${quotedColumn})) != '' LIMIT ${limit}`; } - prepareQuery(sql: string, params?: Record): { sql: string; params?: Record } { - if (!params) { - return { sql, params: undefined }; - } - - let parameterizedQuery = sql; - const queryParams: Record = {}; - const sortedKeys = Object.keys(params).sort((a, b) => b.length - a.length); - - for (const key of sortedKeys) { - const placeholder = `:${key}`; - if (parameterizedQuery.includes(placeholder)) { - parameterizedQuery = parameterizedQuery.replace( - new RegExp(`:${key}\\b`, 'g'), - `{${key}:${this.inferClickHouseType(params[key])}}`, - ); - queryParams[key] = params[key]; - } - } - - return { sql: parameterizedQuery, params: queryParams }; - } - getRandomSampleFilter(samplePct: number): string { if (samplePct <= 0 || samplePct >= 1) { return ''; @@ -132,7 +128,11 @@ export class KtxClickHouseDialect { } getLimitOffsetClause(limit: number, offset?: number): string { - return offset !== undefined && offset > 0 ? `LIMIT ${limit} OFFSET ${offset}` : `LIMIT ${limit}`; + return limitOffsetClause(limit, offset); + } + + getTopClause(_limit: number): string { + return ''; } getNullCountExpression(column: string): string { @@ -143,6 +143,18 @@ export class KtxClickHouseDialect { return `COUNT(DISTINCT ${column})`; } + textLengthExpression(columnSql: string): string { + return `length(toString(${columnSql}))`; + } + + castToText(columnSql: string): string { + return `toString(${columnSql})`; + } + + getSampleValueAggregation(innerSql: string): string { + return `(SELECT arrayStringConcat(groupArray(toString(value)), '\\x1F') FROM (${innerSql}) AS relationship_profile_values)`; + } + generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string { return ` SELECT COUNT(DISTINCT val) AS cardinality @@ -181,99 +193,9 @@ export class KtxClickHouseDialect { ) `; } - - getTimeTruncExpression( - column: string, - granularity: 'day' | 'week' | 'month' | 'quarter' | 'year', - timezone?: string, - ): string { - const tz = timezone ? `, '${timezone}'` : ''; - switch (granularity) { - case 'day': - return `toStartOfDay(${column}${tz})`; - case 'week': - return `toStartOfWeek(${column}, 1${tz})`; - case 'month': - return `toStartOfMonth(${column}${tz})`; - case 'quarter': - return `toStartOfQuarter(${column}${tz})`; - case 'year': - return `toStartOfYear(${column}${tz})`; - } - } - - getCustomTimeTruncExpression(column: string, interval: string, origin?: string, timezone?: string): string { - const col = timezone ? `toTimezone(${column}, '${timezone}')` : column; - const [rawAmount, rawUnit] = interval.split(' '); - const amount = Number(rawAmount); - const unit = rawUnit!.toLowerCase(); - const originExpr = origin ? `toDateTime('${origin}')` : "toDateTime('1970-01-01')"; - const calendarUnit = this.toClickHouseDateDiffUnit(unit); - if (calendarUnit) { - return `dateAdd(${calendarUnit}, intDiv(dateDiff(${calendarUnit}, ${originExpr}, ${col}), ${amount}) * ${amount}, ${originExpr})`; - } - const seconds = this.intervalToSeconds(amount, unit); - return `addSeconds(${originExpr}, intDiv(toUInt64(dateDiff('second', ${originExpr}, ${col})), ${seconds}) * ${seconds})`; - } - - parseIntervalToSql(interval: string): string { - const [amount, unit] = interval.split(' '); - return `INTERVAL ${amount} ${unit!.toUpperCase()}`; - } - private unwrapClickHouseType(value: string, wrapper: string): string { const prefix = `${wrapper}(`; return value.startsWith(prefix) && value.endsWith(')') ? value.slice(prefix.length, -1) : value; } - private inferClickHouseType(value: unknown): string { - if (value === null || value === undefined) { - return 'String'; - } - if (typeof value === 'boolean') { - return 'Bool'; - } - if (typeof value === 'number') { - return Number.isInteger(value) ? 'Int64' : 'Float64'; - } - if (value instanceof Date) { - return 'DateTime'; - } - return 'String'; - } - - private toClickHouseDateDiffUnit(unit: string): string | null { - if (unit === 'month' || unit === 'months') { - return "'month'"; - } - if (unit === 'quarter' || unit === 'quarters') { - return "'quarter'"; - } - if (unit === 'year' || unit === 'years') { - return "'year'"; - } - return null; - } - - private intervalToSeconds(amount: number, unit: string): number { - switch (unit) { - case 'second': - case 'seconds': - return amount; - case 'minute': - case 'minutes': - return amount * 60; - case 'hour': - case 'hours': - return amount * 3600; - case 'day': - case 'days': - return amount * 86400; - case 'week': - case 'weeks': - return amount * 604800; - default: - return amount * 86400; - } - } } diff --git a/packages/cli/src/connectors/mysql/connector.ts b/packages/cli/src/connectors/mysql/connector.ts index 82a2384c..c147c7dd 100644 --- a/packages/cli/src/connectors/mysql/connector.ts +++ b/packages/cli/src/connectors/mysql/connector.ts @@ -2,10 +2,37 @@ import mysql, { type FieldPacket, type Pool, type RowDataPacket } from 'mysql2/p import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; +import { getDialectForDriver } from '../../context/connections/dialects.js'; import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; -import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxTableListEntry, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js'; +import { + constraintDiscoveryWarning, + tryConstraintQuery, + type ConstraintDiscoveryKind, +} from '../../context/scan/constraint-discovery.js'; import { scopedTableNames } from '../../context/scan/table-ref.js'; -import { KtxMysqlDialect } from './dialect.js'; +import { + connectorTestFailure, + createKtxConnectorCapabilities, + type KtxConnectorTestResult, + type KtxColumnSampleInput, + type KtxColumnSampleResult, + type KtxColumnStatsInput, + type KtxColumnStatsResult, + type KtxQueryResult, + type KtxReadOnlyQueryInput, + type KtxScanConnector, + type KtxScanContext, + type KtxScanInput, + type KtxScanWarning, + type KtxSchemaColumn, + type KtxSchemaForeignKey, + type KtxSchemaSnapshot, + type KtxSchemaTable, + type KtxTableListEntry, + type KtxTableRef, + type KtxTableSampleInput, + type KtxTableSampleResult, +} from '../../context/scan/types.js'; export interface KtxMysqlConnectionConfig { driver?: string; @@ -18,6 +45,7 @@ export interface KtxMysqlConnectionConfig { password?: string; url?: string; ssl?: boolean | { rejectUnauthorized?: boolean }; + maxConnections?: number; [key: string]: unknown; } @@ -163,6 +191,23 @@ function maybeNumber(value: unknown): number | undefined { return typeof value === 'number' && Number.isFinite(value) ? value : undefined; } +function positiveIntegerConfigValue(input: { + connection: KtxMysqlConnectionConfig; + key: keyof KtxMysqlConnectionConfig; + connectionId: string; + defaultValue: number; +}): number { + const value = input.connection[input.key]; + if (value === undefined) { + return input.defaultValue; + } + const numberValue = Number(value); + if (!Number.isInteger(numberValue) || numberValue < 1) { + throw new Error(`connections.${input.connectionId}.${String(input.key)} must be a positive integer`); + } + return numberValue; +} + function parseMysqlUrl(url: string): Partial { const parsed = new URL(url); const sslParam = parsed.searchParams.get('ssl') ?? parsed.searchParams.get('sslmode'); @@ -231,6 +276,28 @@ function primaryKeyMap(rows: MysqlPrimaryKeyRow[], fallbackDatabase: string): Ma return grouped; } +function isDeniedError(error: unknown): boolean { + if (!error || typeof error !== 'object') { + return false; + } + const code = (error as { code?: unknown }).code; + return ( + code === 'ER_TABLEACCESS_DENIED_ERROR' || + code === 'ER_SPECIFIC_ACCESS_DENIED_ERROR' || + code === 'ER_DBACCESS_DENIED_ERROR' + ); +} + +function pushConstraintWarnings( + warnings: KtxScanWarning[], + schemas: readonly string[], + kind: ConstraintDiscoveryKind, +): void { + for (const schema of schemas) { + warnings.push(constraintDiscoveryWarning({ schema, kind })); + } +} + function queryParams(params: Record | unknown[] | undefined): unknown[] | undefined { if (!params) { return undefined; @@ -238,6 +305,25 @@ function queryParams(params: Record | unknown[] | undefined): u return Array.isArray(params) ? params : Object.values(params); } +/** @internal */ +export function prepareMysqlReadOnlyQuery( + sql: string, + params?: Record, +): { sql: string; params?: unknown[] } { + if (!params) { + return { sql, params: undefined }; + } + const values: unknown[] = []; + const parameterizedQuery = sql.replace(/:([A-Za-z_][A-Za-z0-9_]*)\b/g, (placeholder, key: string) => { + if (!(key in params)) { + return placeholder; + } + values.push(params[key]); + return '?'; + }); + return { sql: parameterizedQuery, params: values }; +} + export function isKtxMysqlConnectionConfig( connection: KtxMysqlConnectionConfig | undefined, ): connection is KtxMysqlConnectionConfig { @@ -262,6 +348,12 @@ export function mysqlConnectionPoolConfigFromConfig(input: { const host = stringConfigValue(merged, 'host', env); const database = stringConfigValue(merged, 'database', env); const user = stringConfigValue(merged, 'username', env) ?? stringConfigValue(merged, 'user', env); + const maxConnections = positiveIntegerConfigValue({ + connection: merged, + key: 'maxConnections', + connectionId: input.connectionId, + defaultValue: 10, + }); if (!host) { throw new Error(`Native MySQL connector requires connections.${input.connectionId}.host or url`); @@ -280,7 +372,7 @@ export function mysqlConnectionPoolConfigFromConfig(input: { database, user, password: stringConfigValue(merged, 'password', env), - connectionLimit: 10, + connectionLimit: maxConnections, waitForConnections: true, ...(ssl ? { ssl: { rejectUnauthorized: ssl.rejectUnauthorized ?? false } } : {}), }; @@ -305,7 +397,7 @@ export class KtxMysqlScanConnector implements KtxScanConnector { private readonly poolFactory: KtxMysqlPoolFactory; private readonly endpointResolver?: KtxMysqlEndpointResolver; private readonly now: () => Date; - private readonly dialect = new KtxMysqlDialect(); + private readonly dialect = getDialectForDriver('mysql'); private pool: KtxMysqlPool | null = null; private resolvedEndpoint: KtxMysqlResolvedEndpoint | null = null; @@ -323,18 +415,19 @@ export class KtxMysqlScanConnector implements KtxScanConnector { this.id = `mysql:${options.connectionId}`; } - async testConnection(): Promise<{ success: boolean; error?: string }> { + async testConnection(): Promise { try { await this.query('SELECT 1'); return { success: true }; } catch (error) { - return { success: false, error: error instanceof Error ? error.message : String(error) }; + return connectorTestFailure(error); } } async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise { this.assertConnection(input.connectionId); const databases = configuredMysqlSchemas(this.connection, this.poolConfig.database); + const snapshotWarnings: KtxScanWarning[] = []; const placeholders = databases.map(() => '?').join(', '); let allScopedTables: string[] | null = null; if (input.tableScope) { @@ -368,8 +461,11 @@ export class KtxMysqlScanConnector implements KtxScanConnector { `, [...databases, ...tableNameParams], ); - const primaryKeys = await this.queryRaw( - ` + const primaryKeysResult = await tryConstraintQuery( + { schema: databases[0] ?? this.poolConfig.database, kind: 'primary_key', isDeniedError }, + () => + this.queryRaw( + ` SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA IN (${placeholders}) @@ -377,10 +473,18 @@ export class KtxMysqlScanConnector implements KtxScanConnector { ${tableNameClause} ORDER BY TABLE_SCHEMA, TABLE_NAME, ORDINAL_POSITION `, - [...databases, ...tableNameParams], + [...databases, ...tableNameParams], + ), ); - const foreignKeys = await this.queryRaw( - ` + const primaryKeys = primaryKeysResult.ok ? primaryKeysResult.value : []; + if (!primaryKeysResult.ok) { + pushConstraintWarnings(snapshotWarnings, databases, 'primary_key'); + } + const foreignKeysResult = await tryConstraintQuery( + { schema: databases[0] ?? this.poolConfig.database, kind: 'foreign_key', isDeniedError }, + () => + this.queryRaw( + ` SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME, CONSTRAINT_NAME FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA IN (${placeholders}) @@ -388,8 +492,13 @@ export class KtxMysqlScanConnector implements KtxScanConnector { ${tableNameClause} ORDER BY TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME `, - [...databases, ...tableNameParams], + [...databases, ...tableNameParams], + ), ); + const foreignKeys = foreignKeysResult.ok ? foreignKeysResult.value : []; + if (!foreignKeysResult.ok) { + pushConstraintWarnings(snapshotWarnings, databases, 'foreign_key'); + } const columnsByTable = groupByTable(columns, this.poolConfig.database); const primaryKeysByTable = primaryKeyMap(primaryKeys, this.poolConfig.database); @@ -417,6 +526,7 @@ export class KtxMysqlScanConnector implements KtxScanConnector { total_columns: schemaTables.reduce((sum, table) => sum + table.columns.length, 0), }, tables: schemaTables, + warnings: snapshotWarnings, }; } @@ -461,7 +571,7 @@ export class KtxMysqlScanConnector implements KtxScanConnector { const limitedSql = limitSqlForExecution(assertReadOnlySql(input.sql), input.maxRows); const prepared = Array.isArray(input.params) ? { sql: limitedSql, params: input.params } - : this.dialect.prepareQuery(limitedSql, input.params); + : prepareMysqlReadOnlyQuery(limitedSql, input.params); const result = await this.query(prepared.sql, prepared.params); return { ...result, rowCount: result.rows.length }; } @@ -536,6 +646,7 @@ export class KtxMysqlScanConnector implements KtxScanConnector { filterSchemas, ); return rows.map((row) => ({ + catalog: null, schema: row.TABLE_SCHEMA, name: row.TABLE_NAME, kind: row.TABLE_TYPE === 'VIEW' ? ('view' as const) : ('table' as const), diff --git a/packages/cli/src/connectors/mysql/dialect.ts b/packages/cli/src/connectors/mysql/dialect.ts index d61db36c..7f9cc725 100644 --- a/packages/cli/src/connectors/mysql/dialect.ts +++ b/packages/cli/src/connectors/mysql/dialect.ts @@ -1,9 +1,18 @@ +import type { KtxDialect } from '../../context/connections/dialects.js'; +import { + columnDisplayPartCount, + formatDialectDisplayRef, + formatDialectTableName, + limitOffsetClause, + parseDialectDisplayRef, +} from '../../context/connections/dialect-helpers.js'; import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type MysqlTableNameRef = Pick & Partial>; -export class KtxMysqlDialect { - readonly type = 'mysql'; +/** @internal */ +export class KtxMysqlDialect implements KtxDialect { + readonly type = 'mysql' as const; private readonly typeMappings: Record = { datetime: 'time', @@ -41,9 +50,19 @@ export class KtxMysqlDialect { } formatTableName(table: MysqlTableNameRef): string { - return table.db - ? `${this.quoteIdentifier(table.db)}.${this.quoteIdentifier(table.name)}` - : this.quoteIdentifier(table.name); + return formatDialectTableName(table, this.quoteIdentifier.bind(this), 'ansi'); + } + + formatDisplayRef(table: MysqlTableNameRef): string { + return formatDialectDisplayRef(table, 'ansi'); + } + + parseDisplayRef(display: string): KtxTableRef | null { + return parseDialectDisplayRef(display, 'ansi'); + } + + columnDisplayTablePartCount(): 1 | 2 | 3 { + return columnDisplayPartCount('ansi'); } mapDataType(nativeType: string): string { @@ -91,21 +110,6 @@ export class KtxMysqlDialect { return `SELECT ${quotedColumn} FROM ${tableName} WHERE ${quotedColumn} IS NOT NULL AND TRIM(CAST(${quotedColumn} AS CHAR)) != '' LIMIT ${limit}`; } - prepareQuery(sql: string, params?: Record): { sql: string; params?: unknown[] } { - if (!params) { - return { sql, params: undefined }; - } - const values: unknown[] = []; - const parameterizedQuery = sql.replace(/:([A-Za-z_][A-Za-z0-9_]*)\b/g, (placeholder, key: string) => { - if (!(key in params)) { - return placeholder; - } - values.push(params[key]); - return '?'; - }); - return { sql: parameterizedQuery, params: values }; - } - getRandomSampleFilter(samplePct: number): string { if (samplePct <= 0 || samplePct >= 1) { return ''; @@ -118,7 +122,11 @@ export class KtxMysqlDialect { } getLimitOffsetClause(limit: number, offset?: number): string { - return offset !== undefined && offset > 0 ? `LIMIT ${limit} OFFSET ${offset}` : `LIMIT ${limit}`; + return limitOffsetClause(limit, offset); + } + + getTopClause(_limit: number): string { + return ''; } getNullCountExpression(column: string): string { @@ -129,6 +137,18 @@ export class KtxMysqlDialect { return `COUNT(DISTINCT ${column})`; } + textLengthExpression(columnSql: string): string { + return `CHAR_LENGTH(CAST(${columnSql} AS CHAR))`; + } + + castToText(columnSql: string): string { + return `CAST(${columnSql} AS CHAR)`; + } + + getSampleValueAggregation(innerSql: string): string { + return `(SELECT GROUP_CONCAT(CAST(value AS CHAR) SEPARATOR CHAR(31)) FROM (${innerSql}) AS relationship_profile_values)`; + } + generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string { return ` SELECT COUNT(DISTINCT val) AS cardinality @@ -167,36 +187,4 @@ export class KtxMysqlDialect { ) AS sampled `; } - - getTimeTruncExpression( - column: string, - granularity: 'day' | 'week' | 'month' | 'quarter' | 'year', - timezone?: string, - ): string { - const col = timezone ? `CONVERT_TZ(${column}, '+00:00', '${timezone}')` : column; - switch (granularity) { - case 'day': - return `DATE(${col})`; - case 'week': - return `DATE(${col} - INTERVAL WEEKDAY(${col}) DAY)`; - case 'month': - return `DATE_FORMAT(${col}, '%Y-%m-01')`; - case 'quarter': - return `MAKEDATE(YEAR(${col}), 1) + INTERVAL (QUARTER(${col}) - 1) QUARTER`; - case 'year': - return `DATE_FORMAT(${col}, '%Y-01-01')`; - } - } - - getCustomTimeTruncExpression(column: string, interval: string, origin?: string, timezone?: string): string { - const col = timezone ? `CONVERT_TZ(${column}, '+00:00', '${timezone}')` : column; - const [amount, unit] = interval.split(' '); - const originExpr = origin ? `'${origin}'` : `'1970-01-01'`; - return `DATE_ADD(${originExpr}, INTERVAL FLOOR(TIMESTAMPDIFF(${unit!.toUpperCase()}, ${originExpr}, ${col}) / ${amount}) * ${amount} ${unit!.toUpperCase()})`; - } - - parseIntervalToSql(interval: string): string { - const [amount, unit] = interval.split(' '); - return `INTERVAL ${amount} ${unit!.toUpperCase()}`; - } } diff --git a/packages/cli/src/connectors/postgres/connector.ts b/packages/cli/src/connectors/postgres/connector.ts index 5cb94bf4..1a956a3d 100644 --- a/packages/cli/src/connectors/postgres/connector.ts +++ b/packages/cli/src/connectors/postgres/connector.ts @@ -1,11 +1,34 @@ import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; +import { getDialectForDriver } from '../../context/connections/dialects.js'; import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; -import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js'; +import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js'; import { scopedTableNames } from '../../context/scan/table-ref.js'; +import { + connectorTestFailure, + createKtxConnectorCapabilities, + type KtxConnectorTestResult, + type KtxColumnSampleInput, + type KtxColumnSampleResult, + type KtxColumnStatsInput, + type KtxColumnStatsResult, + type KtxQueryResult, + type KtxReadOnlyQueryInput, + type KtxScanConnector, + type KtxScanContext, + type KtxScanInput, + type KtxScanWarning, + type KtxSchemaColumn, + type KtxSchemaForeignKey, + type KtxSchemaSnapshot, + type KtxSchemaTable, + type KtxTableListEntry, + type KtxTableRef, + type KtxTableSampleInput, + type KtxTableSampleResult, +} from '../../context/scan/types.js'; import { Pool } from 'pg'; -import { KtxPostgresDialect } from './dialect.js'; const PG_OID_TYPE_MAP: Record = { 16: 'boolean', @@ -43,6 +66,7 @@ export interface KtxPostgresConnectionConfig { sslmode?: string; sslMode?: string; rejectUnauthorized?: boolean; + maxConnections?: number; [key: string]: unknown; } @@ -197,6 +221,29 @@ function groupByTable(rows: T[]): Map, +): { sql: string; params?: unknown[] } { + if (!params) { + return { sql, params: undefined }; + } + const paramNames = Object.keys(params); + const values: unknown[] = new Array(paramNames.length); + const paramIndexMap = new Map(); + paramNames.forEach((name, index) => { + paramIndexMap.set(name, index + 1); + values[index] = params[name]; + }); + const sortedKeys = [...paramNames].sort((a, b) => b.length - a.length); + let parameterizedQuery = sql; + for (const name of sortedKeys) { + parameterizedQuery = parameterizedQuery.replace(new RegExp(`:${name}\\b`, 'g'), `$${paramIndexMap.get(name)}`); + } + return { sql: parameterizedQuery, params: values }; +} + function primaryKeyMap(rows: PostgresPrimaryKeyRow[]): Map> { const grouped = new Map>(); for (const row of rows) { @@ -207,6 +254,14 @@ function primaryKeyMap(rows: PostgresPrimaryKeyRow[]): Map> return grouped; } +function isDeniedError(error: unknown): boolean { + if (!error || typeof error !== 'object') { + return false; + } + const code = (error as { code?: unknown }).code; + return code === '42501' || code === '42P01'; +} + function queryRows(result: KtxPostgresQueryResult): unknown[][] { const headers = (result.fields ?? []).map((field) => field.name); return result.rows.map((row) => headers.map((header) => row[header])); @@ -242,6 +297,23 @@ function numberValue(value: unknown): number | undefined { return typeof value === 'number' && Number.isFinite(value) ? value : undefined; } +function positiveIntegerConfigValue(input: { + connection: KtxPostgresConnectionConfig; + key: keyof KtxPostgresConnectionConfig; + connectionId: string; + defaultValue: number; +}): number { + const value = input.connection[input.key]; + if (value === undefined) { + return input.defaultValue; + } + const numberValue = Number(value); + if (!Number.isInteger(numberValue) || numberValue < 1) { + throw new Error(`connections.${input.connectionId}.${String(input.key)} must be a positive integer`); + } + return numberValue; +} + function parsePostgresUrl(url: string): Partial { const parsed = new URL(url); const sslmode = parsed.searchParams.get('sslmode') ?? undefined; @@ -276,7 +348,7 @@ export function isKtxPostgresConnectionConfig( connection: KtxPostgresConnectionConfig | undefined, ): connection is KtxPostgresConnectionConfig { const driver = String(connection?.driver ?? '').toLowerCase(); - return driver === 'postgres' || driver === 'postgresql'; + return driver === 'postgres'; } /** @internal */ @@ -299,6 +371,12 @@ export function postgresPoolConfigFromConfig(input: { const user = stringConfigValue(merged, 'username', env) ?? stringConfigValue(merged, 'user', env); const password = stringConfigValue(merged, 'password', env); const sslmode = normalizedSslMode(merged); + const maxConnections = positiveIntegerConfigValue({ + connection: merged, + key: 'maxConnections', + connectionId: input.connectionId, + defaultValue: 10, + }); if (!referencedUrl && !host) { throw new Error(`Native PostgreSQL connector requires connections.${input.connectionId}.host or url`); @@ -311,7 +389,7 @@ export function postgresPoolConfigFromConfig(input: { } const config: KtxPostgresPoolConfig = { - max: 10, + max: maxConnections, idleTimeoutMillis: 30_000, connectionTimeoutMillis: 10_000, ...(referencedUrl && sslmode !== 'prefer' && sslmode !== 'disable' @@ -347,7 +425,7 @@ export class KtxPostgresScanConnector implements KtxScanConnector { private readonly poolFactory: KtxPostgresPoolFactory; private readonly endpointResolver?: KtxPostgresEndpointResolver; private readonly now: () => Date; - private readonly dialect = new KtxPostgresDialect(); + private readonly dialect = getDialectForDriver('postgres'); private pool: KtxPostgresPool | null = null; private lastIdlePoolError: Error | null = null; private resolvedEndpoint: KtxPostgresResolvedEndpoint | null = null; @@ -366,12 +444,12 @@ export class KtxPostgresScanConnector implements KtxScanConnector { this.id = `postgres:${options.connectionId}`; } - async testConnection(): Promise<{ success: boolean; error?: string }> { + async testConnection(): Promise { try { await this.query('SELECT 1'); return { success: true }; } catch (error) { - return { success: false, error: error instanceof Error ? error.message : String(error) }; + return connectorTestFailure(error); } } @@ -379,10 +457,11 @@ export class KtxPostgresScanConnector implements KtxScanConnector { this.assertConnection(input.connectionId); const schemas = schemasFromConnection(this.connection); const allTables: KtxSchemaTable[] = []; + const snapshotWarnings: KtxScanWarning[] = []; for (const schema of schemas) { const scopedNames = input.tableScope ? scopedTableNames(input.tableScope, { catalog: null, db: schema }) : null; if (scopedNames && scopedNames.length === 0) continue; - const tables = await this.loadSchemaTables(schema, scopedNames); + const tables = await this.loadSchemaTables(schema, scopedNames, snapshotWarnings); allTables.push(...tables); } return { @@ -398,6 +477,7 @@ export class KtxPostgresScanConnector implements KtxScanConnector { total_columns: allTables.reduce((sum, table) => sum + table.columns.length, 0), }, tables: allTables, + warnings: snapshotWarnings, }; } @@ -434,7 +514,7 @@ export class KtxPostgresScanConnector implements KtxScanConnector { const limitedSql = limitSqlForExecution(assertReadOnlySql(input.sql), input.maxRows); const prepared = Array.isArray(input.params) ? { sql: limitedSql, params: input.params } - : this.dialect.prepareQuery(limitedSql, input.params); + : preparePostgresReadOnlyQuery(limitedSql, input.params); const result = await this.query(prepared.sql, prepared.params); return { ...result, rowCount: result.rows.length }; } @@ -529,6 +609,7 @@ export class KtxPostgresScanConnector implements KtxScanConnector { [filterSchemas], ); return rows.map((row) => ({ + catalog: null, schema: row.schema_name, name: row.table_name, kind: row.table_kind === 'v' ? ('view' as const) : ('table' as const), @@ -546,7 +627,11 @@ export class KtxPostgresScanConnector implements KtxScanConnector { } } - private async loadSchemaTables(schema: string, scopedNames: readonly string[] | null): Promise { + private async loadSchemaTables( + schema: string, + scopedNames: readonly string[] | null, + snapshotWarnings: KtxScanWarning[], + ): Promise { if (scopedNames && scopedNames.length === 0) return []; const pgCatalogScopeClause = scopedNames ? 'AND c.relname = ANY($2)' : ''; const tableConstraintScopeClause = scopedNames ? 'AND tc.table_name = ANY($2)' : ''; @@ -591,8 +676,11 @@ export class KtxPostgresScanConnector implements KtxScanConnector { `, [schema, ...scopeValues], ); - const primaryKeys = await this.queryRaw( - ` + const primaryKeysResult = await tryConstraintQuery( + { schema, kind: 'primary_key', isDeniedError }, + () => + this.queryRaw( + ` SELECT tc.table_name, kcu.column_name FROM information_schema.table_constraints tc JOIN information_schema.key_column_usage kcu @@ -603,10 +691,18 @@ export class KtxPostgresScanConnector implements KtxScanConnector { ${tableConstraintScopeClause} ORDER BY tc.table_name, kcu.ordinal_position `, - [schema, ...scopeValues], + [schema, ...scopeValues], + ), ); - const foreignKeys = await this.queryRaw( - ` + const primaryKeys = primaryKeysResult.ok ? primaryKeysResult.value : []; + if (!primaryKeysResult.ok) { + snapshotWarnings.push(primaryKeysResult.warning); + } + const foreignKeysResult = await tryConstraintQuery( + { schema, kind: 'foreign_key', isDeniedError }, + () => + this.queryRaw( + ` SELECT tc.table_name, kcu.column_name, @@ -626,8 +722,13 @@ export class KtxPostgresScanConnector implements KtxScanConnector { ${tableConstraintScopeClause} ORDER BY tc.table_name, kcu.column_name `, - [schema, ...scopeValues], + [schema, ...scopeValues], + ), ); + const foreignKeys = foreignKeysResult.ok ? foreignKeysResult.value : []; + if (!foreignKeysResult.ok) { + snapshotWarnings.push(foreignKeysResult.warning); + } const columnsByTable = groupByTable(columns); const primaryKeysByTable = primaryKeyMap(primaryKeys); diff --git a/packages/cli/src/connectors/postgres/dialect.ts b/packages/cli/src/connectors/postgres/dialect.ts index ea0590b8..49d5677d 100644 --- a/packages/cli/src/connectors/postgres/dialect.ts +++ b/packages/cli/src/connectors/postgres/dialect.ts @@ -1,9 +1,18 @@ +import type { KtxDialect } from '../../context/connections/dialects.js'; +import { + columnDisplayPartCount, + formatDialectDisplayRef, + formatDialectTableName, + limitOffsetClause, + parseDialectDisplayRef, +} from '../../context/connections/dialect-helpers.js'; import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type PostgresTableNameRef = Pick & Partial>; -export class KtxPostgresDialect { - readonly type = 'postgresql'; +/** @internal */ +export class KtxPostgresDialect implements KtxDialect { + readonly type = 'postgres' as const; private readonly typeMappings: Record = { timestamp: 'time', @@ -45,9 +54,19 @@ export class KtxPostgresDialect { } formatTableName(table: PostgresTableNameRef): string { - return table.db - ? `${this.quoteIdentifier(table.db)}.${this.quoteIdentifier(table.name)}` - : this.quoteIdentifier(table.name); + return formatDialectTableName(table, this.quoteIdentifier.bind(this), 'ansi'); + } + + formatDisplayRef(table: PostgresTableNameRef): string { + return formatDialectDisplayRef(table, 'ansi'); + } + + parseDisplayRef(display: string): KtxTableRef | null { + return parseDialectDisplayRef(display, 'ansi'); + } + + columnDisplayTablePartCount(): 1 | 2 | 3 { + return columnDisplayPartCount('ansi'); } mapDataType(nativeType: string): string { @@ -92,25 +111,6 @@ export class KtxPostgresDialect { return `SELECT ${quotedColumn} FROM ${tableName} WHERE ${quotedColumn} IS NOT NULL AND TRIM(CAST(${quotedColumn} AS TEXT)) != '' LIMIT ${limit}`; } - prepareQuery(sql: string, params?: Record): { sql: string; params?: unknown[] } { - if (!params) { - return { sql, params: undefined }; - } - const paramNames = Object.keys(params); - const values: unknown[] = new Array(paramNames.length); - const paramIndexMap = new Map(); - paramNames.forEach((name, index) => { - paramIndexMap.set(name, index + 1); - values[index] = params[name]; - }); - const sortedKeys = [...paramNames].sort((a, b) => b.length - a.length); - let parameterizedQuery = sql; - for (const name of sortedKeys) { - parameterizedQuery = parameterizedQuery.replace(new RegExp(`:${name}\\b`, 'g'), `$${paramIndexMap.get(name)}`); - } - return { sql: parameterizedQuery, params: values }; - } - getRandomSampleFilter(samplePct: number): string { if (samplePct <= 0 || samplePct >= 1) { return ''; @@ -126,7 +126,11 @@ export class KtxPostgresDialect { } getLimitOffsetClause(limit: number, offset?: number): string { - return offset !== undefined && offset > 0 ? `LIMIT ${limit} OFFSET ${offset}` : `LIMIT ${limit}`; + return limitOffsetClause(limit, offset); + } + + getTopClause(_limit: number): string { + return ''; } getNullCountExpression(column: string): string { @@ -137,6 +141,18 @@ export class KtxPostgresDialect { return `COUNT(DISTINCT ${column})`; } + textLengthExpression(columnSql: string): string { + return `LENGTH(CAST(${columnSql} AS TEXT))`; + } + + castToText(columnSql: string): string { + return `CAST(${columnSql} AS TEXT)`; + } + + getSampleValueAggregation(innerSql: string): string { + return `(SELECT STRING_AGG(CAST(value AS TEXT), CHR(31)) FROM (${innerSql}) AS relationship_profile_values)`; + } + generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string { return ` WITH sampled AS ( @@ -191,23 +207,4 @@ export class KtxPostgresDialect { FROM sampled `; } - - getTimeTruncExpression( - column: string, - granularity: 'day' | 'week' | 'month' | 'quarter' | 'year', - timezone?: string, - ): string { - const col = timezone ? `(${column} AT TIME ZONE '${timezone.replace(/'/g, "''")}')` : column; - return `DATE_TRUNC('${granularity}', ${col})`; - } - - getCustomTimeTruncExpression(column: string, interval: string, origin?: string, timezone?: string): string { - const col = timezone ? `(${column} AT TIME ZONE '${timezone.replace(/'/g, "''")}')` : column; - const originExpr = origin ? `TIMESTAMP '${origin.replace(/'/g, "''")}'` : "TIMESTAMP '1970-01-01'"; - return `${originExpr} + FLOOR(EXTRACT(EPOCH FROM (${col} - ${originExpr})) / EXTRACT(EPOCH FROM INTERVAL '${interval.replace(/'/g, "''")}')) * INTERVAL '${interval.replace(/'/g, "''")}'`; - } - - parseIntervalToSql(interval: string): string { - return `INTERVAL '${interval.replace(/'/g, "''")}'`; - } } diff --git a/packages/cli/src/connectors/snowflake/connector.ts b/packages/cli/src/connectors/snowflake/connector.ts index 0281b298..51a91e52 100644 --- a/packages/cli/src/connectors/snowflake/connector.ts +++ b/packages/cli/src/connectors/snowflake/connector.ts @@ -2,12 +2,34 @@ import { createPrivateKey } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; +import { getDialectForDriver } from '../../context/connections/dialects.js'; import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; -import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js'; +import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js'; import { scopedTableNames } from '../../context/scan/table-ref.js'; +import { + connectorTestFailure, + createKtxConnectorCapabilities, + type KtxConnectorTestResult, + type KtxColumnSampleInput, + type KtxColumnSampleResult, + type KtxColumnStatsInput, + type KtxColumnStatsResult, + type KtxQueryResult, + type KtxReadOnlyQueryInput, + type KtxScanConnector, + type KtxScanContext, + type KtxScanInput, + type KtxScanWarning, + type KtxSchemaColumn, + type KtxSchemaSnapshot, + type KtxSchemaTable, + type KtxTableListEntry, + type KtxTableRef, + type KtxTableSampleInput, + type KtxTableSampleResult, +} from '../../context/scan/types.js'; import snowflake from 'snowflake-sdk'; import type { Bind, Binds, Connection, ConnectionOptions } from 'snowflake-sdk'; -import { KtxSnowflakeDialect } from './dialect.js'; import { assertSafeSnowflakeIdentifier, quoteSnowflakeIdentifier } from './identifiers.js'; import { configureSnowflakeSdkLogger } from './sdk-logger.js'; @@ -24,7 +46,7 @@ export interface KtxSnowflakeConnectionConfig { privateKey?: string; passphrase?: string; role?: string; - maxSessions?: number; + maxConnections?: number; [key: string]: unknown; } @@ -39,7 +61,7 @@ export interface KtxSnowflakeResolvedConnectionConfig { privateKey?: string; passphrase?: string; role?: string; - maxSessions: number; + maxConnections: number; } export interface KtxSnowflakeRawColumnMetadata { @@ -166,6 +188,13 @@ function firstNumber(value: unknown): number | null { return Number.isFinite(numberValue) ? numberValue : null; } +function isDeniedError(error: unknown): boolean { + if (error instanceof Error) { + return /insufficient privileges|does not exist or not authorized/i.test(error.message); + } + return false; +} + function normalizeSnowflakeValue(value: unknown, columnType?: string): unknown { if (columnType && DATE_TYPES.some((type) => columnType.toUpperCase().includes(type))) { if (typeof value === 'number') { @@ -202,6 +231,14 @@ function toSnowflakeBinds(params: unknown[] | undefined): Binds | undefined { return params?.map((value) => toSnowflakeBind(value)); } +/** @internal */ +export function prepareSnowflakeReadOnlyQuery( + sql: string, + params?: Record, +): { sql: string; params?: unknown[] } { + return { sql, params: params ? Object.values(params) : undefined }; +} + export function isKtxSnowflakeConnectionConfig( connection: KtxSnowflakeConnectionConfig | undefined, ): connection is KtxSnowflakeConnectionConfig { @@ -218,6 +255,10 @@ export function snowflakeConnectionConfigFromConfig(input: { if (!isKtxSnowflakeConnectionConfig(input.connection)) { throw new Error(`Native Snowflake connector cannot run driver "${inputDriver}"`); } + const staleMaxSessionsKey = 'max' + 'Sessions'; + if (Object.prototype.hasOwnProperty.call(input.connection, staleMaxSessionsKey)) { + throw new Error(`connections.${input.connectionId}.maxSessions has been renamed to maxConnections`); + } const env = input.env ?? process.env; const authMethod = input.connection?.authMethod ?? 'password'; const account = stringConfigValue(input.connection, 'account', env); @@ -249,9 +290,9 @@ export function snowflakeConnectionConfigFromConfig(input: { database, schemas: resolvedSchemas, username, - maxSessions: positiveIntegerConfigValue({ + maxConnections: positiveIntegerConfigValue({ connection: input.connection, - key: 'maxSessions', + key: 'maxConnections', connectionId: input.connectionId, defaultValue: 4, }), @@ -322,7 +363,7 @@ class SnowflakeSdkDriver implements KtxSnowflakeDriver { const message = error instanceof Error ? error.message : String(error); if (/timeout/i.test(message) && /pool|acquire/i.test(message)) { throw new Error( - "Snowflake session pool exhausted after 60s - consider lowering maxSessions or increasing your account's concurrent-statement limit.", + "Snowflake session pool exhausted after 60s - consider lowering maxConnections or increasing your account's concurrent-statement limit.", ); } throw error; @@ -399,6 +440,7 @@ class SnowflakeSdkDriver implements KtxSnowflakeDriver { [this.resolved.database, ...(schemas ?? [])], ); return result.rows.map((row) => ({ + catalog: this.resolved.database, schema: String(row[0]), name: String(row[1]), kind: String(row[2]) === 'VIEW' ? ('view' as const) : ('table' as const), @@ -424,7 +466,7 @@ class SnowflakeSdkDriver implements KtxSnowflakeDriver { await this.query('SELECT 1'); return { success: true }; } catch (error) { - return { success: false, error: error instanceof Error ? error.message : String(error) }; + return connectorTestFailure(error); } } @@ -432,7 +474,7 @@ class SnowflakeSdkDriver implements KtxSnowflakeDriver { if (!this.pool) { this.pool = snowflake.createPool(await this.resolveConnectionOptions(), { min: 0, - max: this.resolved.maxSessions, + max: this.resolved.maxConnections, evictionRunIntervalMillis: 30_000, acquireTimeoutMillis: 60_000, }); @@ -519,7 +561,7 @@ export class KtxSnowflakeScanConnector implements KtxScanConnector { private readonly resolved: KtxSnowflakeResolvedConnectionConfig; private readonly driverFactory: KtxSnowflakeDriverFactory; - private readonly dialect = new KtxSnowflakeDialect(); + private readonly dialect = getDialectForDriver('snowflake'); private readonly now: () => Date; private driverInstance: KtxSnowflakeDriver | null = null; @@ -533,20 +575,30 @@ export class KtxSnowflakeScanConnector implements KtxScanConnector { } } - async testConnection(): Promise<{ success: boolean; error?: string }> { + async testConnection(): Promise { return this.getDriver().test(); } async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise { this.assertConnection(input.connectionId); const tables: KtxSchemaTable[] = []; + const snapshotWarnings: KtxScanWarning[] = []; for (const schemaName of this.resolved.schemas) { const scopedNames = input.tableScope ? scopedTableNames(input.tableScope, { catalog: this.resolved.database, db: schemaName }) : null; if (scopedNames && scopedNames.length === 0) continue; const rawTables = await this.getDriver().getSchemaMetadata(schemaName, scopedNames); - const primaryKeys = await this.primaryKeys(rawTables.map((table) => table.name), schemaName); + const primaryKeysResult = await tryConstraintQuery( + { schema: schemaName, kind: 'primary_key', isDeniedError }, + () => this.primaryKeys(rawTables.map((table) => table.name), schemaName), + ); + const primaryKeys = primaryKeysResult.ok + ? primaryKeysResult.value + : new Map(rawTables.map((table) => [table.name, new Set()])); + if (!primaryKeysResult.ok) { + snapshotWarnings.push(primaryKeysResult.warning); + } tables.push(...rawTables.map((table) => this.toSchemaTable(table, primaryKeys))); } return { @@ -563,6 +615,7 @@ export class KtxSnowflakeScanConnector implements KtxScanConnector { total_columns: tables.reduce((sum, table) => sum + table.columns.length, 0), }, tables, + warnings: snapshotWarnings, }; } @@ -593,7 +646,7 @@ export class KtxSnowflakeScanConnector implements KtxScanConnector { async executeReadOnly(input: KtxSnowflakeReadOnlyQueryInput, _ctx: KtxScanContext): Promise { this.assertConnection(input.connectionId); const limitedSql = limitSqlForExecution(assertReadOnlySql(input.sql), input.maxRows); - const prepared = this.dialect.prepareQuery(limitedSql, input.params); + const prepared = prepareSnowflakeReadOnlyQuery(limitedSql, input.params); return this.getDriver().query(prepared.sql, prepared.params); } @@ -654,6 +707,7 @@ export class KtxSnowflakeScanConnector implements KtxScanConnector { [this.resolved.database, ...(schemas ?? [])], ); return result.rows.map((row) => ({ + catalog: this.resolved.database, schema: String(row[0]), name: String(row[1]), kind: String(row[2]) === 'VIEW' ? ('view' as const) : ('table' as const), @@ -686,9 +740,8 @@ export class KtxSnowflakeScanConnector implements KtxScanConnector { return grouped; } const tableNamePlaceholders = tableNames.map(() => '?').join(', '); - try { - const result = await this.getDriver().query( - ` + const result = await this.getDriver().query( + ` SELECT tc.TABLE_NAME, kcu.COLUMN_NAME FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu @@ -701,16 +754,12 @@ export class KtxSnowflakeScanConnector implements KtxScanConnector { AND tc.TABLE_NAME IN (${tableNamePlaceholders}) ORDER BY tc.TABLE_NAME, kcu.ORDINAL_POSITION `, - [schemaName, this.resolved.database, ...tableNames], - ); - for (const row of result.rows) { - const tableName = String(row[0]); - const columnName = String(row[1]); - grouped.get(tableName)?.add(columnName); - } - } catch { - // INFORMATION_SCHEMA.KEY_COLUMN_USAGE often isn't granted to read-only roles; - // continue with empty PK map and let FK inference + profiling carry the slack. + [schemaName, this.resolved.database, ...tableNames], + ); + for (const row of result.rows) { + const tableName = String(row[0]); + const columnName = String(row[1]); + grouped.get(tableName)?.add(columnName); } return grouped; } diff --git a/packages/cli/src/connectors/snowflake/dialect.ts b/packages/cli/src/connectors/snowflake/dialect.ts index db508134..3fe04101 100644 --- a/packages/cli/src/connectors/snowflake/dialect.ts +++ b/packages/cli/src/connectors/snowflake/dialect.ts @@ -1,9 +1,18 @@ +import type { KtxDialect } from '../../context/connections/dialects.js'; +import { + columnDisplayPartCount, + formatDialectDisplayRef, + formatDialectTableName, + limitOffsetClause, + parseDialectDisplayRef, +} from '../../context/connections/dialect-helpers.js'; import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type SnowflakeTableNameRef = Pick & Partial>; -export class KtxSnowflakeDialect { - readonly type = 'snowflake'; +/** @internal */ +export class KtxSnowflakeDialect implements KtxDialect { + readonly type = 'snowflake' as const; private readonly typeMappings: Record = { TIMESTAMP_NTZ: 'time', @@ -45,13 +54,19 @@ export class KtxSnowflakeDialect { } formatTableName(table: SnowflakeTableNameRef): string { - if (table.catalog && table.db) { - return `${this.quoteIdentifier(table.catalog)}.${this.quoteIdentifier(table.db)}.${this.quoteIdentifier(table.name)}`; - } - if (table.db) { - return `${this.quoteIdentifier(table.db)}.${this.quoteIdentifier(table.name)}`; - } - return this.quoteIdentifier(table.name); + return formatDialectTableName(table, this.quoteIdentifier.bind(this), 'three-part'); + } + + formatDisplayRef(table: SnowflakeTableNameRef): string { + return formatDialectDisplayRef(table, 'three-part'); + } + + parseDisplayRef(display: string): KtxTableRef | null { + return parseDialectDisplayRef(display, 'three-part'); + } + + columnDisplayTablePartCount(): 1 | 2 | 3 { + return columnDisplayPartCount('three-part'); } mapDataType(nativeType: string): string { @@ -96,10 +111,6 @@ export class KtxSnowflakeDialect { return `SELECT ${quotedColumn} FROM ${tableName} WHERE ${quotedColumn} IS NOT NULL AND TRIM(CAST(${quotedColumn} AS STRING)) != '' LIMIT ${limit}`; } - prepareQuery(sql: string, params?: Record): { sql: string; params?: unknown[] } { - return { sql, params: params ? Object.values(params) : undefined }; - } - getRandomSampleFilter(samplePct: number): string { if (samplePct <= 0 || samplePct >= 1) { return ''; @@ -115,7 +126,11 @@ export class KtxSnowflakeDialect { } getLimitOffsetClause(limit: number, offset?: number): string { - return offset !== undefined && offset > 0 ? `LIMIT ${limit} OFFSET ${offset}` : `LIMIT ${limit}`; + return limitOffsetClause(limit, offset); + } + + getTopClause(_limit: number): string { + return ''; } getNullCountExpression(column: string): string { @@ -126,6 +141,18 @@ export class KtxSnowflakeDialect { return `APPROX_COUNT_DISTINCT(${column})`; } + textLengthExpression(columnSql: string): string { + return `LENGTH(CAST(${columnSql} AS TEXT))`; + } + + castToText(columnSql: string): string { + return `CAST(${columnSql} AS VARCHAR)`; + } + + getSampleValueAggregation(innerSql: string): string { + return `(SELECT LISTAGG(CAST(value AS VARCHAR), '\\x1f') FROM (${innerSql}) AS relationship_profile_values)`; + } + generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string { return ` WITH sampled AS ( @@ -164,24 +191,4 @@ export class KtxSnowflakeDialect { FROM sampled `; } - - getTimeTruncExpression( - column: string, - granularity: 'day' | 'week' | 'month' | 'quarter' | 'year', - timezone?: string, - ): string { - const target = timezone ? `CONVERT_TIMEZONE('UTC', '${timezone}', ${column})` : column; - return `DATE_TRUNC('${granularity}', ${target})`; - } - - getCustomTimeTruncExpression(column: string, interval: string, origin?: string, timezone?: string): string { - const target = timezone ? `CONVERT_TIMEZONE('UTC', '${timezone}', ${column})` : column; - const [amount, unit] = interval.split(' '); - const originExpr = origin ? `'${origin}'::TIMESTAMP` : `'1970-01-01'::TIMESTAMP`; - return `DATEADD(${unit}, FLOOR(DATEDIFF(${unit}, ${originExpr}, ${target}) / ${amount}) * ${amount}, ${originExpr})`; - } - - parseIntervalToSql(interval: string): string { - return `INTERVAL '${interval}'`; - } } diff --git a/packages/cli/src/connectors/sqlite/connector.ts b/packages/cli/src/connectors/sqlite/connector.ts index 17b33a71..f5ba2a55 100644 --- a/packages/cli/src/connectors/sqlite/connector.ts +++ b/packages/cli/src/connectors/sqlite/connector.ts @@ -3,11 +3,11 @@ import { existsSync, readFileSync, statSync } from 'node:fs'; import { homedir } from 'node:os'; import { isAbsolute, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { getDialectForDriver } from '../../context/connections/dialects.js'; import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; import { normalizeQueryRows } from '../../context/connections/query-executor.js'; -import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js'; +import { connectorTestFailure, createKtxConnectorCapabilities, type KtxConnectorTestResult, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js'; import { scopedTableNames } from '../../context/scan/table-ref.js'; -import { KtxSqliteDialect } from './dialect.js'; export interface KtxSqliteConnectionConfig { driver?: string; @@ -125,7 +125,7 @@ export function isKtxSqliteConnectionConfig( connection: KtxSqliteConnectionConfig | undefined, ): connection is KtxSqliteConnectionConfig { const driver = String(connection?.driver ?? '').toLowerCase(); - return driver === 'sqlite' || driver === 'sqlite3'; + return driver === 'sqlite'; } /** @internal */ @@ -157,7 +157,7 @@ export class KtxSqliteScanConnector implements KtxScanConnector { private readonly connectionId: string; private readonly dbPath: string; private readonly now: () => Date; - private readonly dialect = new KtxSqliteDialect(); + private readonly dialect = getDialectForDriver('sqlite'); private db: Database.Database | null = null; constructor(options: KtxSqliteScanConnectorOptions) { @@ -167,7 +167,7 @@ export class KtxSqliteScanConnector implements KtxScanConnector { this.id = `sqlite:${options.connectionId}`; } - async testConnection(): Promise<{ success: boolean; error?: string }> { + async testConnection(): Promise { try { if (!existsSync(this.dbPath) || !statSync(this.dbPath).isFile()) { return { success: false, error: `File not found: ${this.dbPath}` }; @@ -175,7 +175,7 @@ export class KtxSqliteScanConnector implements KtxScanConnector { this.database().prepare('SELECT 1').get(); return { success: true }; } catch (error) { - return { success: false, error: error instanceof Error ? error.message : String(error) }; + return connectorTestFailure(error); } } @@ -209,6 +209,31 @@ export class KtxSqliteScanConnector implements KtxScanConnector { }; } + async listSchemas(): Promise { + return []; + } + + async listTables(_schemas?: string[]): Promise { + const rows = this.database() + .prepare( + ` + SELECT name, type + FROM sqlite_master + WHERE type IN ('table', 'view') + AND name NOT LIKE 'sqlite_%' + ORDER BY name + `, + ) + .all() as SqliteMasterRow[]; + + return rows.map((row) => ({ + catalog: null, + schema: '', + name: row.name, + kind: row.type === 'view' ? ('view' as const) : ('table' as const), + })); + } + async sampleTable(input: KtxTableSampleInput, _ctx: KtxScanContext): Promise { this.assertConnection(input.connectionId); const result = this.query(this.dialect.generateSampleQuery(this.qTableName(input.table), input.limit, input.columns)); diff --git a/packages/cli/src/connectors/sqlite/dialect.ts b/packages/cli/src/connectors/sqlite/dialect.ts index b5771b62..5472b674 100644 --- a/packages/cli/src/connectors/sqlite/dialect.ts +++ b/packages/cli/src/connectors/sqlite/dialect.ts @@ -1,9 +1,18 @@ +import type { KtxDialect } from '../../context/connections/dialects.js'; +import { + columnDisplayPartCount, + formatDialectDisplayRef, + formatDialectTableName, + limitOffsetClause, + parseDialectDisplayRef, +} from '../../context/connections/dialect-helpers.js'; import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type SqliteTableNameRef = Pick & Partial>; -export class KtxSqliteDialect { - readonly type = 'sqlite'; +/** @internal */ +export class KtxSqliteDialect implements KtxDialect { + readonly type = 'sqlite' as const; private readonly typeMappings: Record = { DATETIME: 'time', @@ -29,7 +38,19 @@ export class KtxSqliteDialect { } formatTableName(table: SqliteTableNameRef): string { - return this.quoteIdentifier(table.name); + return formatDialectTableName(table, this.quoteIdentifier.bind(this), 'sqlite'); + } + + formatDisplayRef(table: SqliteTableNameRef): string { + return formatDialectDisplayRef(table, 'sqlite'); + } + + parseDisplayRef(display: string): KtxTableRef | null { + return parseDialectDisplayRef(display, 'sqlite'); + } + + columnDisplayTablePartCount(): 1 | 2 | 3 { + return columnDisplayPartCount('sqlite'); } mapDataType(nativeType: string): string { @@ -76,10 +97,6 @@ export class KtxSqliteDialect { return `SELECT ${quoted} FROM ${tableName} WHERE ${quoted} IS NOT NULL AND TRIM(CAST(${quoted} AS TEXT)) != '' LIMIT ${limit}`; } - prepareQuery(sql: string, params?: Record): { sql: string; params?: unknown } { - return params ? { sql, params } : { sql }; - } - getRandomSampleFilter(samplePct: number): string { if (samplePct <= 0 || samplePct >= 1) { return ''; @@ -92,7 +109,11 @@ export class KtxSqliteDialect { } getLimitOffsetClause(limit: number, offset?: number): string { - return offset !== undefined && offset > 0 ? `LIMIT ${limit} OFFSET ${offset}` : `LIMIT ${limit}`; + return limitOffsetClause(limit, offset); + } + + getTopClause(_limit: number): string { + return ''; } getNullCountExpression(column: string): string { @@ -103,6 +124,18 @@ export class KtxSqliteDialect { return `COUNT(DISTINCT ${column})`; } + textLengthExpression(columnSql: string): string { + return `LENGTH(CAST(${columnSql} AS TEXT))`; + } + + castToText(columnSql: string): string { + return `CAST(${columnSql} AS TEXT)`; + } + + getSampleValueAggregation(innerSql: string): string { + return `(SELECT GROUP_CONCAT(CAST(value AS TEXT), char(31)) FROM (${innerSql}) AS relationship_profile_values)`; + } + generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string { return ` WITH sampled AS ( @@ -143,35 +176,4 @@ export class KtxSqliteDialect { FROM sampled `; } - - getTimeTruncExpression( - column: string, - granularity: 'day' | 'week' | 'month' | 'quarter' | 'year', - _timezone?: string, - ): string { - switch (granularity) { - case 'day': - return `DATE(${column})`; - case 'week': - return `DATE(${column}, 'weekday 0', '-6 days')`; - case 'month': - return `DATE(${column}, 'start of month')`; - case 'quarter': - return `DATE(${column}, 'start of month', '-' || ((CAST(STRFTIME('%m', ${column}) AS INTEGER) - 1) % 3) || ' months')`; - case 'year': - return `DATE(${column}, 'start of year')`; - } - } - - getCustomTimeTruncExpression(column: string, interval: string, origin?: string, _timezone?: string): string { - const [amount, unit] = interval.split(' '); - const originExpr = origin ? `julianday('${origin}')` : `julianday('1970-01-01')`; - const unitDays = unit === 'day' ? 1 : unit === 'week' ? 7 : 30; - const intervalDays = Number(amount) * unitDays; - return `DATE(julianday('1970-01-01') + (CAST((julianday(${column}) - ${originExpr}) / ${intervalDays} AS INTEGER) * ${intervalDays}))`; - } - - parseIntervalToSql(interval: string): string { - return `'${interval}'`; - } } diff --git a/packages/cli/src/connectors/sqlserver/connector.ts b/packages/cli/src/connectors/sqlserver/connector.ts index 64b8075e..5dd9969b 100644 --- a/packages/cli/src/connectors/sqlserver/connector.ts +++ b/packages/cli/src/connectors/sqlserver/connector.ts @@ -1,11 +1,34 @@ import { assertReadOnlySql } from '../../context/connections/read-only-sql.js'; -import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js'; +import { getDialectForDriver } from '../../context/connections/dialects.js'; +import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js'; import { scopedTableNames } from '../../context/scan/table-ref.js'; +import { + connectorTestFailure, + createKtxConnectorCapabilities, + type KtxConnectorTestResult, + type KtxColumnSampleInput, + type KtxColumnSampleResult, + type KtxColumnStatsInput, + type KtxColumnStatsResult, + type KtxQueryResult, + type KtxReadOnlyQueryInput, + type KtxScanConnector, + type KtxScanContext, + type KtxScanInput, + type KtxScanWarning, + type KtxSchemaColumn, + type KtxSchemaForeignKey, + type KtxSchemaSnapshot, + type KtxSchemaTable, + type KtxTableListEntry, + type KtxTableRef, + type KtxTableSampleInput, + type KtxTableSampleResult, +} from '../../context/scan/types.js'; import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; import sql from 'mssql'; -import { KtxSqlServerDialect } from './dialect.js'; export interface KtxSqlServerConnectionConfig { driver?: string; @@ -19,6 +42,7 @@ export interface KtxSqlServerConnectionConfig { schema?: string; schemas?: string[]; trustServerCertificate?: boolean; + maxConnections?: number; [key: string]: unknown; } @@ -136,6 +160,21 @@ function tableScopeSql( return { clause: `AND ${columnExpression} IN (${placeholders.join(', ')})`, params }; } +/** @internal */ +export function prepareSqlServerReadOnlyQuery( + sql: string, + params?: Record, +): { sql: string; params?: Record } { + if (!params) { + return { sql, params: undefined }; + } + let parameterizedQuery = sql; + for (const key of Object.keys(params)) { + parameterizedQuery = parameterizedQuery.replace(new RegExp(`:${key}\\b`, 'g'), `@${key}`); + } + return { sql: parameterizedQuery, params }; +} + class DefaultSqlServerPoolFactory implements KtxSqlServerPoolFactory { async createPool(config: KtxSqlServerPoolConfig): Promise { const pool = await new sql.ConnectionPool(config as sql.config).connect(); @@ -197,6 +236,23 @@ function maybeNumber(value: unknown): number | undefined { return typeof value === 'number' && Number.isFinite(value) ? value : undefined; } +function positiveIntegerConfigValue(input: { + connection: KtxSqlServerConnectionConfig; + key: keyof KtxSqlServerConnectionConfig; + connectionId: string; + defaultValue: number; +}): number { + const value = input.connection[input.key]; + if (value === undefined) { + return input.defaultValue; + } + const numberValue = Number(value); + if (!Number.isInteger(numberValue) || numberValue < 1) { + throw new Error(`connections.${input.connectionId}.${String(input.key)} must be a positive integer`); + } + return numberValue; +} + function schemaNames(connection: KtxSqlServerConnectionConfig, env: NodeJS.ProcessEnv): string[] { if (Array.isArray(connection.schemas) && connection.schemas.length > 0) { return connection.schemas.filter((schema) => schema.trim().length > 0).map((schema) => resolveStringReference(schema, env)); @@ -219,6 +275,14 @@ function firstNumber(value: unknown): number | null { return Number.isFinite(numberValue) ? numberValue : null; } +function isDeniedError(error: unknown): boolean { + if (!error || typeof error !== 'object') { + return false; + } + const number = (error as { number?: unknown }).number; + return number === 229 || number === 230 || number === 297; +} + function limitSqlForSqlServerExecution(sqlText: string, maxRows: number | undefined): string { const trimmed = assertReadOnlySql(sqlText).replace(/;+\s*$/, ''); if (!maxRows) { @@ -254,6 +318,12 @@ export function sqlServerConnectionPoolConfigFromConfig(input: { const server = stringConfigValue(merged, 'host', env); const database = stringConfigValue(merged, 'database', env); const user = stringConfigValue(merged, 'username', env) ?? stringConfigValue(merged, 'user', env); + const maxConnections = positiveIntegerConfigValue({ + connection: merged, + key: 'maxConnections', + connectionId: input.connectionId, + defaultValue: 10, + }); if (!server) { throw new Error(`Native SQL Server connector requires connections.${input.connectionId}.host or url`); @@ -272,7 +342,7 @@ export function sqlServerConnectionPoolConfigFromConfig(input: { user, password: stringConfigValue(merged, 'password', env), options: { encrypt: true, trustServerCertificate: merged.trustServerCertificate ?? true }, - pool: { max: 10, min: 0, idleTimeoutMillis: 30000 }, + pool: { max: maxConnections, min: 0, idleTimeoutMillis: 30000 }, }; } @@ -296,7 +366,7 @@ export class KtxSqlServerScanConnector implements KtxScanConnector { private readonly poolFactory: KtxSqlServerPoolFactory; private readonly endpointResolver?: KtxSqlServerEndpointResolver; private readonly now: () => Date; - private readonly dialect = new KtxSqlServerDialect(); + private readonly dialect = getDialectForDriver('sqlserver'); private pool: KtxSqlServerPool | null = null; private resolvedEndpoint: KtxSqlServerResolvedEndpoint | null = null; @@ -316,23 +386,24 @@ export class KtxSqlServerScanConnector implements KtxScanConnector { this.id = `sqlserver:${options.connectionId}`; } - async testConnection(): Promise<{ success: boolean; error?: string }> { + async testConnection(): Promise { try { await this.query('SELECT 1'); return { success: true }; } catch (error) { - return { success: false, error: error instanceof Error ? error.message : String(error) }; + return connectorTestFailure(error); } } async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise { this.assertConnection(input.connectionId); const tables: KtxSchemaTable[] = []; + const snapshotWarnings: KtxScanWarning[] = []; for (const schemaName of this.schemas) { const scopedNames = input.tableScope ? scopedTableNames(input.tableScope, { catalog: this.poolConfig.database, db: schemaName }) : null; - tables.push(...(await this.introspectSchema(schemaName, scopedNames))); + tables.push(...(await this.introspectSchema(schemaName, scopedNames, snapshotWarnings))); } return { connectionId: this.connectionId, @@ -347,6 +418,7 @@ export class KtxSqlServerScanConnector implements KtxScanConnector { total_columns: tables.reduce((sum, table) => sum + table.columns.length, 0), }, tables, + warnings: snapshotWarnings, }; } @@ -372,7 +444,7 @@ export class KtxSqlServerScanConnector implements KtxScanConnector { async executeReadOnly(input: KtxSqlServerReadOnlyQueryInput, _ctx: KtxScanContext): Promise { this.assertConnection(input.connectionId); const limitedSql = limitSqlForSqlServerExecution(input.sql, input.maxRows); - const prepared = this.dialect.prepareQuery(limitedSql, input.params); + const prepared = prepareSqlServerReadOnlyQuery(limitedSql, input.params); const result = await this.query(prepared.sql, prepared.params); return { ...result, rowCount: result.rows.length }; } @@ -462,6 +534,7 @@ export class KtxSqlServerScanConnector implements KtxScanConnector { params, ); return rows.map((row) => ({ + catalog: this.poolConfig.database, schema: row.schema_name, name: row.table_name, kind: row.table_type === 'VIEW' ? ('view' as const) : ('table' as const), @@ -479,7 +552,11 @@ export class KtxSqlServerScanConnector implements KtxScanConnector { } } - private async introspectSchema(schemaName: string, scopedNames: readonly string[] | null): Promise { + private async introspectSchema( + schemaName: string, + scopedNames: readonly string[] | null, + snapshotWarnings: KtxScanWarning[], + ): Promise { if (scopedNames && scopedNames.length === 0) return []; const tableScope = tableScopeSql(scopedNames, 'TABLE_NAME'); const tables = await this.queryRaw<{ table_name: string; table_type: string }>( @@ -510,8 +587,22 @@ export class KtxSqlServerScanConnector implements KtxScanConnector { ); const tableComments = await this.tableComments(schemaName, scopedNames); const columnComments = await this.columnComments(schemaName, scopedNames); - const primaryKeys = await this.primaryKeys(schemaName, scopedNames); - const foreignKeys = await this.foreignKeys(schemaName, scopedNames); + const primaryKeysResult = await tryConstraintQuery( + { schema: schemaName, kind: 'primary_key', isDeniedError }, + () => this.primaryKeys(schemaName, scopedNames), + ); + const foreignKeysResult = await tryConstraintQuery( + { schema: schemaName, kind: 'foreign_key', isDeniedError }, + () => this.foreignKeys(schemaName, scopedNames), + ); + const primaryKeys = primaryKeysResult.ok ? primaryKeysResult.value : new Map>(); + const foreignKeys = foreignKeysResult.ok ? foreignKeysResult.value : []; + if (!primaryKeysResult.ok) { + snapshotWarnings.push(primaryKeysResult.warning); + } + if (!foreignKeysResult.ok) { + snapshotWarnings.push(foreignKeysResult.warning); + } const rowCounts = await this.rowCounts(schemaName, scopedNames); const columnsByTable = groupByTable(columns); const foreignKeysByTable = groupByTable(foreignKeys); diff --git a/packages/cli/src/connectors/sqlserver/dialect.ts b/packages/cli/src/connectors/sqlserver/dialect.ts index 8444317d..6b1804f4 100644 --- a/packages/cli/src/connectors/sqlserver/dialect.ts +++ b/packages/cli/src/connectors/sqlserver/dialect.ts @@ -1,9 +1,18 @@ +import type { KtxDialect } from '../../context/connections/dialects.js'; +import { + columnDisplayPartCount, + formatDialectDisplayRef, + formatDialectTableName, + parseDialectDisplayRef, + safeSqlLimit, +} from '../../context/connections/dialect-helpers.js'; import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type SqlServerTableNameRef = Pick & Partial>; -export class KtxSqlServerDialect { - readonly type = 'sqlserver'; +/** @internal */ +export class KtxSqlServerDialect implements KtxDialect { + readonly type = 'sqlserver' as const; private readonly typeMappings: Record = { datetime: 'time', @@ -39,9 +48,19 @@ export class KtxSqlServerDialect { } formatTableName(table: SqlServerTableNameRef): string { - return table.db - ? `${this.quoteIdentifier(table.db)}.${this.quoteIdentifier(table.name)}` - : this.quoteIdentifier(table.name); + return formatDialectTableName(table, this.quoteIdentifier.bind(this), 'three-part'); + } + + formatDisplayRef(table: SqlServerTableNameRef): string { + return formatDialectDisplayRef(table, 'three-part'); + } + + parseDisplayRef(display: string): KtxTableRef | null { + return parseDialectDisplayRef(display, 'three-part'); + } + + columnDisplayTablePartCount(): 1 | 2 | 3 { + return columnDisplayPartCount('three-part'); } mapDataType(nativeType: string): string { @@ -86,17 +105,6 @@ export class KtxSqlServerDialect { return `SELECT TOP ${limit} ${quotedColumn} FROM ${tableName} WHERE ${quotedColumn} IS NOT NULL AND LTRIM(RTRIM(CAST(${quotedColumn} AS NVARCHAR(MAX)))) != ''`; } - prepareQuery(sql: string, params?: Record): { sql: string; params?: Record } { - if (!params) { - return { sql, params: undefined }; - } - let parameterizedQuery = sql; - for (const key of Object.keys(params)) { - parameterizedQuery = parameterizedQuery.replace(new RegExp(`:${key}\\b`, 'g'), `@${key}`); - } - return { sql: parameterizedQuery, params }; - } - getRandomSampleFilter(samplePct: number): string { if (samplePct <= 0 || samplePct >= 1) { return ''; @@ -111,12 +119,12 @@ export class KtxSqlServerDialect { return `TABLESAMPLE (${samplePct * 100} PERCENT)`; } - getLimitOffsetClause(limit: number, offset?: number): string { - return offset !== undefined && offset > 0 ? `OFFSET ${offset} ROWS FETCH NEXT ${limit} ROWS ONLY` : ''; + getLimitOffsetClause(_limit: number, _offset?: number): string { + return ''; } getTopClause(limit: number): string { - return `TOP ${limit}`; + return `TOP (${safeSqlLimit(limit)})`; } getNullCountExpression(column: string): string { @@ -127,6 +135,18 @@ export class KtxSqlServerDialect { return `COUNT(DISTINCT ${column})`; } + textLengthExpression(columnSql: string): string { + return `LEN(CAST(${columnSql} AS NVARCHAR(MAX)))`; + } + + castToText(columnSql: string): string { + return `CAST(${columnSql} AS NVARCHAR(MAX))`; + } + + getSampleValueAggregation(innerSql: string): string { + return `(SELECT STRING_AGG(CAST(value AS NVARCHAR(MAX)), CHAR(31)) FROM (${innerSql}) AS relationship_profile_values)`; + } + generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string { return ` WITH sampled AS ( @@ -167,35 +187,4 @@ export class KtxSqlServerDialect { FROM sampled `; } - - getTimeTruncExpression( - column: string, - granularity: 'day' | 'week' | 'month' | 'quarter' | 'year', - timezone?: string, - ): string { - const col = timezone ? `${column} AT TIME ZONE 'UTC' AT TIME ZONE '${timezone}'` : column; - switch (granularity) { - case 'day': - return `CAST(${col} AS DATE)`; - case 'week': - return `DATEADD(WEEK, DATEDIFF(WEEK, 0, ${col}), 0)`; - case 'month': - return `DATEFROMPARTS(YEAR(${col}), MONTH(${col}), 1)`; - case 'quarter': - return `DATEFROMPARTS(YEAR(${col}), (DATEPART(QUARTER, ${col}) - 1) * 3 + 1, 1)`; - case 'year': - return `DATEFROMPARTS(YEAR(${col}), 1, 1)`; - } - } - - getCustomTimeTruncExpression(column: string, interval: string, origin?: string, timezone?: string): string { - const col = timezone ? `${column} AT TIME ZONE 'UTC' AT TIME ZONE '${timezone}'` : column; - const [amount, unit] = interval.split(' '); - const originExpr = origin ? `'${origin}'` : `'1970-01-01'`; - return `DATEADD(${unit}, (DATEDIFF(${unit}, ${originExpr}, ${col}) / ${amount}) * ${amount}, ${originExpr})`; - } - - parseIntervalToSql(interval: string): string { - return `'${interval}'`; - } } diff --git a/packages/cli/src/context-build-view.ts b/packages/cli/src/context-build-view.ts index 49ddd3eb..0ddd4922 100644 --- a/packages/cli/src/context-build-view.ts +++ b/packages/cli/src/context-build-view.ts @@ -1,8 +1,6 @@ -import type { KtxProgressPort, KtxProgressUpdateOptions } from './context/scan/types.js'; import type { KtxCliIo } from './index.js'; import type { KtxIngestProgressUpdate } from './ingest.js'; import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; -import { publicDatabaseIngestMessage, publicQueryHistoryMessage } from './public-ingest-copy.js'; import type { KtxPublicIngestArgs, KtxPublicIngestDeps, @@ -10,7 +8,8 @@ import type { KtxPublicIngestProject, KtxPublicIngestTargetResult, } from './public-ingest.js'; -import { buildPublicIngestPlan, executePublicIngestTarget } from './public-ingest.js'; +import { buildPublicIngestPlan, executePublicIngestTarget, publicProgressMessage } from './public-ingest.js'; +import { createAggregateProgressPort } from './progress-port-adapter.js'; import { formatDuration } from './demo-metrics.js'; import { profileMark } from './startup-profile.js'; @@ -88,7 +87,6 @@ export interface ContextBuildArgs { targetConnectionId?: string; all?: boolean; entrypoint?: 'setup' | 'ingest'; - depth?: Extract['depth']; queryHistory?: Extract['queryHistory']; queryHistoryWindowDays?: number; scanMode?: Extract['scanMode']; @@ -371,19 +369,17 @@ function retryCommand(input: { projectDir?: string; entrypoint?: 'setup' | 'ingest'; connectionId?: string; - depth?: 'fast' | 'deep'; queryHistory?: boolean; queryHistoryWindowDays?: number; }): string { const projectPart = input.projectDir ? ` --project-dir ${input.projectDir}` : ''; if (input.entrypoint === 'ingest' && input.connectionId) { - const depthPart = input.depth ? ` --${input.depth}` : ''; const queryHistoryPart = input.queryHistory ? ' --query-history' : ''; const windowPart = input.queryHistory && input.queryHistoryWindowDays !== undefined ? ` --query-history-window-days ${input.queryHistoryWindowDays}` : ''; - return `ktx ingest ${input.connectionId}${projectPart}${depthPart}${queryHistoryPart}${windowPart}`; + return `ktx ingest ${input.connectionId}${projectPart}${queryHistoryPart}${windowPart}`; } return input.projectDir ? `ktx setup --project-dir ${input.projectDir}` : 'ktx setup'; } @@ -694,7 +690,7 @@ function isLocalSqlAnalysisConnectionRefused(input: { capturedOutput?: string; f function friendlyDriverName(driver: string): string { const normalized = driver.toLowerCase(); - if (normalized === 'postgres' || normalized === 'postgresql') return 'PostgreSQL'; + if (normalized === 'postgres') return 'PostgreSQL'; if (normalized === 'mysql') return 'MySQL'; if (normalized === 'sqlserver') return 'SQL Server'; if (normalized === 'bigquery') return 'BigQuery'; @@ -746,7 +742,6 @@ function appendRetryIfNeeded(input: { projectDir: input.projectDir, entrypoint: input.entrypoint, connectionId: input.target.connectionId, - depth: input.target.databaseDepth, queryHistory: input.target.queryHistory?.enabled === true, queryHistoryWindowDays: input.target.queryHistory?.windowDays, })}`; @@ -769,7 +764,6 @@ function failureTextForTarget(input: { projectDir: input.projectDir, entrypoint: input.entrypoint, connectionId: input.target.connectionId, - depth: input.target.databaseDepth, queryHistory: input.target.queryHistory?.enabled === true, queryHistoryWindowDays: input.target.queryHistory?.windowDays, })}`, @@ -784,7 +778,6 @@ function failureTextForTarget(input: { projectDir: input.projectDir, entrypoint: input.entrypoint, connectionId: input.target.connectionId, - depth: input.target.databaseDepth, queryHistory: input.target.queryHistory?.enabled === true, queryHistoryWindowDays: input.target.queryHistory?.windowDays, })}`, @@ -816,17 +809,6 @@ export function initViewState(targets: KtxPublicIngestPlanTarget[]): ContextBuil }; } -function publicProgressMessage(message: string, target: KtxPublicIngestPlanTarget): string { - let current = message; - if (target.operation === 'database-ingest') { - current = publicDatabaseIngestMessage(current); - } - if (target.steps.includes('query-history')) { - current = publicQueryHistoryMessage(current, target.connectionId); - } - return current; -} - function formatProgressDetail( update: Pick, target: KtxPublicIngestPlanTarget, @@ -835,29 +817,6 @@ function formatProgressDetail( return `[${percent}%] ${publicProgressMessage(update.message, target)}`; } -function createContextBuildProgressPort( - onProgress: (update: KtxIngestProgressUpdate) => void, - state: { progress: number } = { progress: 0 }, - start = 0, - weight = 1, -): KtxProgressPort { - return { - async update(value: number, message?: string, options?: KtxProgressUpdateOptions): Promise { - const absoluteValue = start + Math.max(0, Math.min(1, value)) * weight; - state.progress = Math.max(state.progress, Math.min(1, absoluteValue)); - if (!message) return; - onProgress({ - percent: Math.max(0, Math.min(100, Math.round(state.progress * 100))), - message, - ...(options?.transient !== undefined ? { transient: options.transient } : {}), - }); - }, - startPhase(phaseWeight: number): KtxProgressPort { - return createContextBuildProgressPort(onProgress, state, state.progress, weight * phaseWeight); - }, - }; -} - export async function runContextBuild( project: KtxPublicIngestProject, args: ContextBuildArgs, @@ -868,7 +827,6 @@ export async function runContextBuild( projectDir: args.projectDir, ...(args.targetConnectionId ? { targetConnectionId: args.targetConnectionId } : {}), all: args.all ?? true, - ...(args.depth ? { depth: args.depth } : {}), ...(args.queryHistory ? { queryHistory: args.queryHistory } : {}), ...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}), ...(args.scanMode ? { scanMode: args.scanMode } : {}), @@ -935,7 +893,6 @@ export async function runContextBuild( all: args.all ?? true, json: false, inputMode: args.inputMode, - ...(args.depth ? { depth: args.depth } : {}), ...(args.queryHistory ? { queryHistory: args.queryHistory } : {}), ...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}), ...(args.scanMode ? { scanMode: args.scanMode } : {}), @@ -1030,7 +987,7 @@ export async function runContextBuild( }; const progressDeps: KtxPublicIngestDeps = { - scanProgress: createContextBuildProgressPort(updateSchemaPhase), + scanProgress: createAggregateProgressPort(updateSchemaPhase), ingestProgress: updateIngestPhase, runtimeIo: io, onPhaseStart, @@ -1040,7 +997,7 @@ export async function runContextBuild( let result: KtxPublicIngestTargetResult | null = null; let thrownError: unknown = null; try { - result = await execTarget(targetState.target, runArgs, capture.io, progressDeps); + result = await execTarget(targetState.target, runArgs, capture.io, progressDeps, project); } catch (error) { thrownError = error; } diff --git a/packages/cli/src/context/connections/dialect-helpers.ts b/packages/cli/src/context/connections/dialect-helpers.ts new file mode 100644 index 00000000..04ed569b --- /dev/null +++ b/packages/cli/src/context/connections/dialect-helpers.ts @@ -0,0 +1,87 @@ +import type { KtxTableRef } from '../scan/types.js'; + +export type KtxDialectIdentifierShape = 'ansi' | 'sqlite' | 'three-part'; + +export type KtxDialectTableRef = Pick & Partial>; + +export function safeSqlLimit(limit: number): number { + return Math.max(1, Math.floor(limit)); +} + +function safeSqlOffset(offset: number | undefined): number | null { + if (offset === undefined) { + return null; + } + const normalized = Math.floor(offset); + return normalized > 0 ? normalized : null; +} + +function cleanIdentifierPart(part: string): string { + return part.trim().replace(/^["'`\[]|["'`\]]$/g, ''); +} + +function splitDisplay(display: string): string[] { + return display.trim().split('.').map(cleanIdentifierPart).filter(Boolean); +} + +function tableParts(table: KtxDialectTableRef, shape: KtxDialectIdentifierShape): string[] { + if (shape === 'sqlite') { + return [table.name]; + } + return [table.catalog ?? null, table.db ?? null, table.name].filter((part): part is string => Boolean(part)); +} + +function acceptedDisplayPartCounts(shape: KtxDialectIdentifierShape): readonly number[] { + if (shape === 'sqlite') { + return [1]; + } + if (shape === 'three-part') { + return [3]; + } + return [2, 3]; +} + +export function formatDialectTableName( + table: KtxDialectTableRef, + quoteIdentifier: (identifier: string) => string, + shape: KtxDialectIdentifierShape, +): string { + return tableParts(table, shape).map(quoteIdentifier).join('.'); +} + +export function formatDialectDisplayRef(table: KtxDialectTableRef, shape: KtxDialectIdentifierShape): string { + return tableParts(table, shape).join('.'); +} + +export function parseDialectDisplayRef(display: string, shape: KtxDialectIdentifierShape): KtxTableRef | null { + const parts = splitDisplay(display); + if (!acceptedDisplayPartCounts(shape).includes(parts.length)) { + return null; + } + if (parts.length === 1) { + return { catalog: null, db: null, name: parts[0]! }; + } + if (parts.length === 2) { + return { catalog: null, db: parts[0]!, name: parts[1]! }; + } + if (parts.length === 3) { + return { catalog: parts[0]!, db: parts[1]!, name: parts[2]! }; + } + return null; +} + +export function columnDisplayPartCount(shape: KtxDialectIdentifierShape): 1 | 2 | 3 { + if (shape === 'sqlite') { + return 1; + } + if (shape === 'three-part') { + return 3; + } + return 2; +} + +export function limitOffsetClause(limit: number, offset?: number): string { + const safeLimit = safeSqlLimit(limit); + const safeOffset = safeSqlOffset(offset); + return safeOffset === null ? `LIMIT ${safeLimit}` : `LIMIT ${safeLimit} OFFSET ${safeOffset}`; +} diff --git a/packages/cli/src/context/connections/dialects.test.ts b/packages/cli/src/context/connections/dialects.test.ts deleted file mode 100644 index 6c9b6c41..00000000 --- a/packages/cli/src/context/connections/dialects.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { getDialectForDriver } from './dialects.js'; - -describe('getDialectForDriver', () => { - it.each([ - ['postgres', '"public"."orders"'], - ['postgresql', '"public"."orders"'], - ['mysql', '`public`.`orders`'], - ['clickhouse', '`public`.`orders`'], - ['sqlite', '"orders"'], - ['snowflake', '"analytics"."public"."orders"'], - ['bigquery', '`analytics`.`public`.`orders`'], - ['sqlserver', '[analytics].[public].[orders]'], - ] as const)('formats table names for %s', (driver, expected) => { - const dialect = getDialectForDriver(driver); - expect( - dialect.formatTableName({ - catalog: driver === 'snowflake' || driver === 'bigquery' || driver === 'sqlserver' ? 'analytics' : null, - db: driver === 'sqlite' ? null : 'public', - name: 'orders', - }), - ).toBe(expected); - }); - - it('throws with a supported-driver list for unknown drivers', () => { - expect(() => getDialectForDriver('oracle')).toThrow( - 'Unsupported warehouse driver "oracle". Supported drivers: bigquery, clickhouse, mysql, postgres, postgresql, sqlite, sqlite3, snowflake, sqlserver', - ); - }); -}); diff --git a/packages/cli/src/context/connections/dialects.ts b/packages/cli/src/context/connections/dialects.ts index 75a8ae4c..c7929cea 100644 --- a/packages/cli/src/context/connections/dialects.ts +++ b/packages/cli/src/context/connections/dialects.ts @@ -1,102 +1,64 @@ -import type { KtxSchemaDimensionType, KtxTableRef } from '../scan/types.js'; - -type SupportedDriver = - | 'postgres' - | 'postgresql' - | 'mysql' - | 'sqlserver' - | 'snowflake' - | 'bigquery' - | 'clickhouse' - | 'sqlite' - | 'sqlite3'; +import { KtxBigQueryDialect } from '../../connectors/bigquery/dialect.js'; +import { KtxClickHouseDialect } from '../../connectors/clickhouse/dialect.js'; +import { KtxMysqlDialect } from '../../connectors/mysql/dialect.js'; +import { KtxPostgresDialect } from '../../connectors/postgres/dialect.js'; +import { KtxSqliteDialect } from '../../connectors/sqlite/dialect.js'; +import { KtxSnowflakeDialect } from '../../connectors/snowflake/dialect.js'; +import { KtxSqlServerDialect } from '../../connectors/sqlserver/dialect.js'; +import type { KtxConnectionDriver, KtxSchemaDimensionType, KtxTableRef } from '../scan/types.js'; +import type { KtxDialectTableRef } from './dialect-helpers.js'; export interface KtxDialect { - readonly type: SupportedDriver; + readonly type: KtxConnectionDriver; quoteIdentifier(identifier: string): string; - formatTableName(table: KtxTableRef): string; + formatTableName(table: KtxDialectTableRef): string; + formatDisplayRef(table: KtxDialectTableRef): string; + parseDisplayRef(display: string): KtxTableRef | null; + columnDisplayTablePartCount(): 1 | 2 | 3; + getLimitOffsetClause(limit: number, offset?: number): string; + getTopClause(limit: number): string; + getRandomSampleFilter(samplePct: number): string; + getTableSampleClause(samplePct: number): string; + generateSampleQuery(tableName: string, limit: number, columns?: string[]): string; + generateColumnSampleQuery(tableName: string, columnName: string, limit: number): string; + getSampleValueAggregation(innerSql: string): string; + generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string; + generateRandomizedCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string; + generateDistinctValuesQuery(tableName: string, columnName: string, limit: number): string; + generateColumnStatisticsQuery(schemaName: string, tableName: string): string | null; + getNullCountExpression(column: string): string; + getDistinctCountExpression(column: string): string; + textLengthExpression(columnSql: string): string; + castToText(columnSql: string): string; mapToDimensionType(nativeType: string): KtxSchemaDimensionType; + mapDataType(nativeType: string): string; } -const supportedDrivers: SupportedDriver[] = [ +const supportedDrivers: KtxConnectionDriver[] = [ 'bigquery', 'clickhouse', 'mysql', 'postgres', - 'postgresql', 'sqlite', - 'sqlite3', 'snowflake', 'sqlserver', ]; -function doubleQuoted(identifier: string): string { - return `"${identifier.replace(/"/g, '""')}"`; -} - -function backtickQuoted(identifier: string): string { - return `\`${identifier.replace(/`/g, '``')}\``; -} - -function bigQueryQuoted(identifier: string): string { - return `\`${identifier.replace(/`/g, '\\`')}\``; -} - -function bracketQuoted(identifier: string): string { - return `[${identifier.replace(/\]/g, ']]')}]`; -} - -function inferDimensionType(nativeType: string): KtxSchemaDimensionType { - const normalized = nativeType.toLowerCase().trim(); - if (normalized.includes('date') || normalized.includes('time')) { - return 'time'; - } - if ( - normalized.includes('int') || - normalized.includes('num') || - normalized.includes('dec') || - normalized.includes('float') || - normalized.includes('double') || - normalized.includes('real') - ) { - return 'number'; - } - if (normalized.includes('bool') || normalized === 'bit') { - return 'boolean'; - } - return 'string'; -} - -function formatWithParts(table: KtxTableRef, quote: (identifier: string) => string, sqlite = false): string { - const parts = sqlite ? [table.name] : [table.catalog, table.db, table.name].filter((part): part is string => !!part); - return parts.map(quote).join('.'); -} - -function createDialect(type: SupportedDriver, quote: (identifier: string) => string, sqlite = false): KtxDialect { - return { - type, - quoteIdentifier: quote, - formatTableName: (table) => formatWithParts(table, quote, sqlite), - mapToDimensionType: inferDimensionType, - }; -} - -const dialects: Record = { - postgres: createDialect('postgres', doubleQuoted), - postgresql: createDialect('postgresql', doubleQuoted), - mysql: createDialect('mysql', backtickQuoted), - clickhouse: createDialect('clickhouse', backtickQuoted), - sqlite: createDialect('sqlite', doubleQuoted, true), - sqlite3: createDialect('sqlite3', doubleQuoted, true), - snowflake: createDialect('snowflake', doubleQuoted), - bigquery: createDialect('bigquery', bigQueryQuoted), - sqlserver: createDialect('sqlserver', bracketQuoted), +const dialectFactories: Record KtxDialect> = { + bigquery: () => new KtxBigQueryDialect(), + clickhouse: () => new KtxClickHouseDialect(), + mysql: () => new KtxMysqlDialect(), + postgres: () => new KtxPostgresDialect(), + sqlite: () => new KtxSqliteDialect(), + snowflake: () => new KtxSnowflakeDialect(), + sqlserver: () => new KtxSqlServerDialect(), }; export function getDialectForDriver(driver: string): KtxDialect { const normalized = driver.toLowerCase().trim(); - if (normalized in dialects) { - return dialects[normalized as SupportedDriver]; + const factory = dialectFactories[normalized as KtxConnectionDriver]; + if (factory) { + return factory(); } throw new Error(`Unsupported warehouse driver "${driver}". Supported drivers: ${supportedDrivers.join(', ')}`); } diff --git a/packages/cli/src/context/connections/drivers.ts b/packages/cli/src/context/connections/drivers.ts new file mode 100644 index 00000000..1b87984b --- /dev/null +++ b/packages/cli/src/context/connections/drivers.ts @@ -0,0 +1,199 @@ +import type { KtxConnectionDriver, KtxScanConnector } from '../scan/types.js'; + +/** @internal */ +export type KtxScopeConfigKey = 'dataset_ids' | 'databases' | 'schemas' | 'schema_names'; + +/** @internal */ +export interface KtxDriverConnectorModule { + isConnectionConfig(connection: unknown): boolean; + createScanConnector(args: { + connectionId: string; + connection: unknown; + projectDir: string; + }): KtxScanConnector; +} + +export interface KtxDriverRegistration { + readonly driver: KtxConnectionDriver; + readonly scopeConfigKey: KtxScopeConfigKey | null; + readonly hasHistoricSqlReader: boolean; + readonly hasLocalQueryExecutor: boolean; + load(): Promise; +} + +function invalidConnectionConfig(driver: KtxConnectionDriver): Error { + return new Error(`Connection config does not match warehouse driver "${driver}".`); +} + +/** @internal */ +export const driverRegistrations: Record = { + bigquery: { + driver: 'bigquery', + scopeConfigKey: 'dataset_ids', + hasHistoricSqlReader: true, + hasLocalQueryExecutor: false, + load: async () => { + const m = await import('../../connectors/bigquery/connector.js'); + return { + isConnectionConfig: (connection) => { + const typedConnection = connection as Parameters[0]; + return m.isKtxBigQueryConnectionConfig(typedConnection); + }, + createScanConnector: ({ connectionId, connection }) => { + const typedConnection = connection as Parameters[0]; + if (!m.isKtxBigQueryConnectionConfig(typedConnection)) { + throw invalidConnectionConfig('bigquery'); + } + return new m.KtxBigQueryScanConnector({ connectionId, connection: typedConnection }); + }, + }; + }, + }, + clickhouse: { + driver: 'clickhouse', + scopeConfigKey: 'databases', + hasHistoricSqlReader: false, + hasLocalQueryExecutor: false, + load: async () => { + const m = await import('../../connectors/clickhouse/connector.js'); + return { + isConnectionConfig: (connection) => { + const typedConnection = connection as Parameters[0]; + return m.isKtxClickHouseConnectionConfig(typedConnection); + }, + createScanConnector: ({ connectionId, connection }) => { + const typedConnection = connection as Parameters[0]; + if (!m.isKtxClickHouseConnectionConfig(typedConnection)) { + throw invalidConnectionConfig('clickhouse'); + } + return new m.KtxClickHouseScanConnector({ connectionId, connection: typedConnection }); + }, + }; + }, + }, + mysql: { + driver: 'mysql', + scopeConfigKey: 'schemas', + hasHistoricSqlReader: false, + hasLocalQueryExecutor: false, + load: async () => { + const m = await import('../../connectors/mysql/connector.js'); + return { + isConnectionConfig: (connection) => { + const typedConnection = connection as Parameters[0]; + return m.isKtxMysqlConnectionConfig(typedConnection); + }, + createScanConnector: ({ connectionId, connection }) => { + const typedConnection = connection as Parameters[0]; + if (!m.isKtxMysqlConnectionConfig(typedConnection)) { + throw invalidConnectionConfig('mysql'); + } + return new m.KtxMysqlScanConnector({ connectionId, connection: typedConnection }); + }, + }; + }, + }, + postgres: { + driver: 'postgres', + scopeConfigKey: 'schemas', + hasHistoricSqlReader: true, + hasLocalQueryExecutor: true, + load: async () => { + const m = await import('../../connectors/postgres/connector.js'); + return { + isConnectionConfig: (connection) => { + const typedConnection = connection as Parameters[0]; + return m.isKtxPostgresConnectionConfig(typedConnection); + }, + createScanConnector: ({ connectionId, connection }) => { + const typedConnection = connection as Parameters[0]; + if (!m.isKtxPostgresConnectionConfig(typedConnection)) { + throw invalidConnectionConfig('postgres'); + } + return new m.KtxPostgresScanConnector({ connectionId, connection: typedConnection }); + }, + }; + }, + }, + sqlite: { + driver: 'sqlite', + scopeConfigKey: null, + hasHistoricSqlReader: false, + hasLocalQueryExecutor: true, + load: async () => { + const m = await import('../../connectors/sqlite/connector.js'); + return { + isConnectionConfig: (connection) => { + const typedConnection = connection as Parameters[0]; + return m.isKtxSqliteConnectionConfig(typedConnection); + }, + createScanConnector: ({ connectionId, connection, projectDir }) => { + const typedConnection = connection as Parameters[0]; + if (!m.isKtxSqliteConnectionConfig(typedConnection)) { + throw invalidConnectionConfig('sqlite'); + } + return new m.KtxSqliteScanConnector({ connectionId, connection: typedConnection, projectDir }); + }, + }; + }, + }, + snowflake: { + driver: 'snowflake', + scopeConfigKey: 'schema_names', + hasHistoricSqlReader: true, + hasLocalQueryExecutor: false, + load: async () => { + const m = await import('../../connectors/snowflake/connector.js'); + return { + isConnectionConfig: (connection) => { + const typedConnection = connection as Parameters[0]; + return m.isKtxSnowflakeConnectionConfig(typedConnection); + }, + createScanConnector: ({ connectionId, connection, projectDir }) => { + const typedConnection = connection as Parameters[0]; + if (!m.isKtxSnowflakeConnectionConfig(typedConnection)) { + throw invalidConnectionConfig('snowflake'); + } + return new m.KtxSnowflakeScanConnector({ connectionId, connection: typedConnection, projectDir }); + }, + }; + }, + }, + sqlserver: { + driver: 'sqlserver', + scopeConfigKey: 'schemas', + hasHistoricSqlReader: false, + hasLocalQueryExecutor: false, + load: async () => { + const m = await import('../../connectors/sqlserver/connector.js'); + return { + isConnectionConfig: (connection) => { + const typedConnection = connection as Parameters[0]; + return m.isKtxSqlServerConnectionConfig(typedConnection); + }, + createScanConnector: ({ connectionId, connection }) => { + const typedConnection = connection as Parameters[0]; + if (!m.isKtxSqlServerConnectionConfig(typedConnection)) { + throw invalidConnectionConfig('sqlserver'); + } + return new m.KtxSqlServerScanConnector({ connectionId, connection: typedConnection }); + }, + }; + }, + }, +}; + +const supportedDrivers = Object.keys(driverRegistrations).sort() as KtxConnectionDriver[]; + +function isRegisteredDriver(driver: string): driver is KtxConnectionDriver { + return Object.prototype.hasOwnProperty.call(driverRegistrations, driver); +} + +export function getDriverRegistration(driver: string): KtxDriverRegistration | undefined { + const normalized = driver.toLowerCase().trim(); + return isRegisteredDriver(normalized) ? driverRegistrations[normalized] : undefined; +} + +export function listSupportedDrivers(): KtxConnectionDriver[] { + return [...supportedDrivers]; +} diff --git a/packages/cli/src/context/connections/local-query-executor.ts b/packages/cli/src/context/connections/local-query-executor.ts index 9b5f2032..3a2e34c9 100644 --- a/packages/cli/src/context/connections/local-query-executor.ts +++ b/packages/cli/src/context/connections/local-query-executor.ts @@ -1,3 +1,4 @@ +import { driverRegistrations, getDriverRegistration } from './drivers.js'; import { createPostgresQueryExecutor } from './postgres-query-executor.js'; import type { KtxSqlQueryExecutionInput, @@ -5,6 +6,7 @@ import type { KtxSqlQueryExecutorPort, } from './query-executor.js'; import { createSqliteQueryExecutor } from './sqlite-query-executor.js'; +import type { KtxConnectionDriver } from '../scan/types.js'; export interface DefaultLocalQueryExecutorOptions { postgres?: KtxSqlQueryExecutorPort; @@ -15,20 +17,43 @@ function driverFor(input: KtxSqlQueryExecutionInput): string { return String(input.connection?.driver ?? '').toLowerCase(); } +function localExecutorMap( + options: DefaultLocalQueryExecutorOptions, +): Partial> { + const wiredExecutors: Partial> = { + postgres: options.postgres ?? createPostgresQueryExecutor(), + sqlite: options.sqlite ?? createSqliteQueryExecutor(), + }; + + const executors: Partial> = {}; + for (const registration of Object.values(driverRegistrations)) { + if (!registration.hasLocalQueryExecutor) continue; + const executor = wiredExecutors[registration.driver]; + if (executor) { + executors[registration.driver] = executor; + } + } + return executors; +} + export function createDefaultLocalQueryExecutor(options: DefaultLocalQueryExecutorOptions = {}): KtxSqlQueryExecutorPort { - const postgres = options.postgres ?? createPostgresQueryExecutor(); - const sqlite = options.sqlite ?? createSqliteQueryExecutor(); + const executors = localExecutorMap(options); return { async execute(input: KtxSqlQueryExecutionInput): Promise { const driver = driverFor(input); - if (driver === 'postgres' || driver === 'postgresql') { - return postgres.execute(input); + const registration = getDriverRegistration(driver); + if (!registration?.hasLocalQueryExecutor) { + throw new Error(`No local query executor is configured for driver "${input.connection?.driver ?? 'unknown'}".`); } - if (driver === 'sqlite' || driver === 'sqlite3') { - return sqlite.execute(input); + + const executor = executors[registration.driver]; + if (!executor) { + throw new Error( + `Local query executor flag is enabled for driver "${registration.driver}", but no executor factory is wired.`, + ); } - throw new Error(`No local query executor is configured for driver "${input.connection?.driver ?? 'unknown'}".`); + return executor.execute(input); }, }; } diff --git a/packages/cli/src/context/connections/local-warehouse-descriptor.ts b/packages/cli/src/context/connections/local-warehouse-descriptor.ts index c2cc6516..4ad926df 100644 --- a/packages/cli/src/context/connections/local-warehouse-descriptor.ts +++ b/packages/cli/src/context/connections/local-warehouse-descriptor.ts @@ -20,10 +20,8 @@ export interface LocalConnectionInfo { const DRIVER_TO_CONNECTION_TYPE: Record = { postgres: 'POSTGRESQL', - postgresql: 'POSTGRESQL', sqlite: 'SQLITE', sqlserver: 'SQLSERVER', - mssql: 'SQLSERVER', mysql: 'MYSQL', clickhouse: 'CLICKHOUSE', snowflake: 'SNOWFLAKE', diff --git a/packages/cli/src/context/connections/postgres-query-executor.ts b/packages/cli/src/context/connections/postgres-query-executor.ts index b5f2d02e..842609f4 100644 --- a/packages/cli/src/context/connections/postgres-query-executor.ts +++ b/packages/cli/src/context/connections/postgres-query-executor.ts @@ -38,7 +38,7 @@ export function createPostgresQueryExecutor(options: PostgresQueryExecutorOption async execute(input: KtxSqlQueryExecutionInput): Promise { const driver = connectionDriver(input); const connection = input.connection; - if (driver !== 'postgres' && driver !== 'postgresql') { + if (driver !== 'postgres') { throw new Error(`Local Postgres execution cannot run driver "${connection?.driver ?? 'unknown'}".`); } if (typeof connection?.url !== 'string' || connection.url.trim().length === 0) { diff --git a/packages/cli/src/context/connections/sqlite-query-executor.ts b/packages/cli/src/context/connections/sqlite-query-executor.ts index 22c69005..40710c96 100644 --- a/packages/cli/src/context/connections/sqlite-query-executor.ts +++ b/packages/cli/src/context/connections/sqlite-query-executor.ts @@ -52,7 +52,7 @@ function sqlitePathFromUrl(url: string): string { /** @internal */ export function sqliteDatabasePathFromConnection(input: KtxSqlQueryExecutionInput): string { const driver = connectionDriver(input); - if (driver !== 'sqlite' && driver !== 'sqlite3') { + if (driver !== 'sqlite') { throw new Error(`Local SQLite execution cannot run driver "${input.connection?.driver ?? 'unknown'}".`); } diff --git a/packages/cli/src/context/core/abort.ts b/packages/cli/src/context/core/abort.ts new file mode 100644 index 00000000..95467c52 --- /dev/null +++ b/packages/cli/src/context/core/abort.ts @@ -0,0 +1,39 @@ +/** @internal */ +export function createAbortError(message = 'Aborted'): DOMException { + return new DOMException(message, 'AbortError'); +} + +export function isAbortError(error: unknown): boolean { + if (error instanceof DOMException && error.name === 'AbortError') { + return true; + } + if (!error || typeof error !== 'object') { + return false; + } + const record = error as { name?: unknown; code?: unknown }; + return record.name === 'AbortError' || record.code === 'ABORT_ERR'; +} + +/** @internal */ +export function throwIfAborted(signal?: AbortSignal): void { + if (signal?.aborted) { + throw createAbortError(); + } +} + +export function linkAbortSignal(parent?: AbortSignal): { controller: AbortController; dispose: () => void } { + const controller = new AbortController(); + if (!parent) { + return { controller, dispose: () => undefined }; + } + if (parent.aborted) { + controller.abort(createAbortError()); + return { controller, dispose: () => undefined }; + } + const onAbort = () => controller.abort(createAbortError()); + parent.addEventListener('abort', onAbort, { once: true }); + return { + controller, + dispose: () => parent.removeEventListener('abort', onAbort), + }; +} diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/bigquery-query-history-reader.ts b/packages/cli/src/context/ingest/adapters/historic-sql/bigquery-query-history-reader.ts index ac4d4c71..fe637078 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/bigquery-query-history-reader.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/bigquery-query-history-reader.ts @@ -200,27 +200,78 @@ export class BigQueryHistoricSqlQueryHistoryReader { config: HistoricSqlUnifiedPullConfig, ): AsyncIterable { const sql = ` +WITH filtered_jobs AS ( + SELECT + COALESCE(query_info.query_hashes.normalized_literals, TO_HEX(SHA256(query))) AS template_id, + query, + user_email, + creation_time, + end_time, + error_result + FROM ${this.viewPath} + WHERE job_type = 'QUERY' + AND statement_type IN ('SELECT', 'MERGE') + AND creation_time >= ${timestampExpression(window.start)} + AND creation_time < ${timestampExpression(window.end)} + AND query IS NOT NULL +), +template_stats AS ( + SELECT + template_id, + MIN(query) AS canonical_sql, + COUNT(*) AS executions, + COUNT(DISTINCT user_email) AS distinct_users, + MIN(creation_time) AS first_seen, + MAX(creation_time) AS last_seen, + APPROX_QUANTILES(TIMESTAMP_DIFF(end_time, creation_time, MILLISECOND), 100)[OFFSET(50)] AS p50_ms, + APPROX_QUANTILES(TIMESTAMP_DIFF(end_time, creation_time, MILLISECOND), 100)[OFFSET(95)] AS p95_ms, + SAFE_DIVIDE(COUNTIF(error_result IS NOT NULL), COUNT(*)) AS error_rate, + CAST(NULL AS INT64) AS rows_produced + FROM filtered_jobs + GROUP BY template_id + HAVING COUNT(*) >= ${config.minExecutions} +), +template_users AS ( + SELECT + template_id, + user_email AS user, + COUNT(*) AS executions, + MAX(creation_time) AS last_seen + FROM filtered_jobs + GROUP BY template_id, user_email +) SELECT - query_hash AS template_id, - MIN(query) AS canonical_sql, - COUNT(*) AS executions, - COUNT(DISTINCT user_email) AS distinct_users, - MIN(creation_time) AS first_seen, - MAX(creation_time) AS last_seen, - APPROX_QUANTILES(TIMESTAMP_DIFF(end_time, creation_time, MILLISECOND), 100)[OFFSET(50)] AS p50_ms, - APPROX_QUANTILES(TIMESTAMP_DIFF(end_time, creation_time, MILLISECOND), 100)[OFFSET(95)] AS p95_ms, - SAFE_DIVIDE(COUNTIF(error_result IS NOT NULL), COUNT(*)) AS error_rate, - CAST(NULL AS INT64) AS rows_produced, - TO_JSON_STRING(ARRAY_AGG(STRUCT(user_email AS user, 1 AS executions) ORDER BY creation_time DESC LIMIT 5)) AS top_users -FROM ${this.viewPath} -WHERE job_type = 'QUERY' - AND statement_type IN ('SELECT', 'MERGE') - AND creation_time >= ${timestampExpression(window.start)} - AND creation_time < ${timestampExpression(window.end)} - AND query IS NOT NULL -GROUP BY query_hash -HAVING COUNT(*) >= ${config.minExecutions} -ORDER BY executions DESC`.trim(); + stats.template_id, + stats.canonical_sql, + stats.executions, + stats.distinct_users, + stats.first_seen, + stats.last_seen, + stats.p50_ms, + stats.p95_ms, + stats.error_rate, + stats.rows_produced, + TO_JSON_STRING( + ARRAY_AGG( + STRUCT(users.user AS user, users.executions AS executions) + ORDER BY users.executions DESC, users.last_seen DESC + ) + ) AS top_users +FROM template_stats AS stats +JOIN template_users AS users + ON users.template_id = stats.template_id +GROUP BY + stats.template_id, + stats.canonical_sql, + stats.executions, + stats.distinct_users, + stats.first_seen, + stats.last_seen, + stats.p50_ms, + stats.p95_ms, + stats.error_rate, + stats.rows_produced +ORDER BY stats.executions DESC`.trim(); const result = await queryClient(client).executeQuery(sql); if (result.error) { throw grantsError(result.error); diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/chunk-unified.ts b/packages/cli/src/context/ingest/adapters/historic-sql/chunk-unified.ts index 4e6dfeda..56cc32e7 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/chunk-unified.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/chunk-unified.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; import { readFile, readdir } from 'node:fs/promises'; import { join, relative } from 'node:path'; +import { tableRefKey } from '../../../scan/table-ref.js'; import type { ChunkResult, DiffSet, ScopeDescriptor, WorkUnit } from '../../types.js'; import { isHistoricSqlPatternInputShardPath } from './pattern-inputs.js'; import { stagedManifestSchema, stagedPatternsInputSchema, stagedTableInputSchema } from './types.js'; @@ -37,7 +38,7 @@ export async function chunkHistoricSqlUnifiedStagedDir(stagedDir: string, diffSe } const table = stagedTableInputSchema.parse(await readJson(stagedDir, path)); workUnits.push({ - unitKey: `historic-sql-table-${safeUnitKey(table.table)}`, + unitKey: `historic-sql-table-${safeUnitKey(tableRefKey(table.tableRef))}`, displayLabel: `Historic SQL usage: ${table.table}`, rawFiles: [path], dependencyPaths: ['manifest.json'], @@ -60,7 +61,7 @@ export async function chunkHistoricSqlUnifiedStagedDir(stagedDir: string, diffSe dependencyPaths: ['manifest.json'], peerFileIndex: files.filter((file) => file !== path && file !== 'manifest.json').sort(), notes: - `Use historic_sql_patterns. Read ${path} and emit pattern objects with emit_historic_sql_evidence using rawPath "${path}". Do not call wiki_write or sl_write_source.`, + `Use historic_sql_patterns. Read ${path} and emit pattern objects with emit_historic_sql_evidence. Do not call wiki_write or sl_write_source.`, }); } diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/connection-dialect.ts b/packages/cli/src/context/ingest/adapters/historic-sql/connection-dialect.ts index 846ce098..7845cbbc 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/connection-dialect.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/connection-dialect.ts @@ -1,5 +1,9 @@ +import { getDriverRegistration } from '../../../connections/drivers.js'; +import type { KtxConnectionDriver } from '../../../scan/types.js'; import type { HistoricSqlDialect } from './types.js'; +const historicSqlDialects: readonly HistoricSqlDialect[] = ['postgres', 'bigquery', 'snowflake']; + function recordOrNull(value: unknown): Record | null { return value && typeof value === 'object' && !Array.isArray(value) ? (value as Record) : null; } @@ -10,10 +14,33 @@ function queryHistoryRecord(connection: unknown): Record | null return context ? recordOrNull(context.queryHistory) : null; } +function historicSqlDialectForDriver(driver: KtxConnectionDriver): HistoricSqlDialect { + const dialect = historicSqlDialects.find((candidate) => candidate === driver); + if (!dialect) { + throw new Error(`Driver "${driver}" is marked as historic-SQL capable but has no HistoricSqlDialect mapping.`); + } + return dialect; +} + export function isQueryHistoryEnabled(connection: unknown): boolean { return queryHistoryRecord(connection)?.enabled === true; } +/** + * Resolves the query-history dialect from the connection's driver capability + * alone, ignoring whether query history is enabled in ktx.yaml. Use this on the + * adapter-registration path when query history has been explicitly requested + * for the run (e.g. via `--query-history`, which is itself the opt-in): the + * persisted `context.queryHistory.enabled` flag must not gate registration. + * Returns null when the connection's driver has no query-history reader. + */ +export function historicSqlDialectForConnectionDriver(connection: unknown): HistoricSqlDialect | null { + const conn = recordOrNull(connection); + const driver = String(conn?.driver ?? '').toLowerCase(); + const registration = getDriverRegistration(driver); + return registration?.hasHistoricSqlReader ? historicSqlDialectForDriver(registration.driver) : null; +} + /** * Resolves the query-history dialect for a connection. Returns null when * query history is disabled, or when the connection's driver has no @@ -23,10 +50,5 @@ export function queryHistoryDialectForConnection(connection: unknown): HistoricS if (!isQueryHistoryEnabled(connection)) { return null; } - const conn = recordOrNull(connection); - const driver = String(conn?.driver ?? '').toLowerCase(); - if (driver === 'postgres' || driver === 'postgresql') return 'postgres'; - if (driver === 'bigquery') return 'bigquery'; - if (driver === 'snowflake') return 'snowflake'; - return null; + return historicSqlDialectForConnectionDriver(connection); } diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/evidence-tool.ts b/packages/cli/src/context/ingest/adapters/historic-sql/evidence-tool.ts index 29d66cb2..1b03e1c6 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/evidence-tool.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/evidence-tool.ts @@ -10,7 +10,6 @@ const emitHistoricSqlEvidenceInputSchema = z .object({ kind: z.enum(['table_usage', 'pattern']), table: z.string().min(1).optional(), - rawPath: z.string().min(1), usage: tableUsageOutputSchema.optional(), pattern: patternOutputSchema.optional(), }) @@ -46,6 +45,7 @@ interface EmitHistoricSqlEvidenceToolContext { connectionId?: string | null; session?: { ingest?: { runId: string; sourceKey: string }; + allowedRawPaths?: ReadonlySet; configService?: { writeFile( path: string, @@ -66,7 +66,7 @@ function unitKeyForEvidence(input: EmitHistoricSqlEvidenceInput): string { return `historic-sql-pattern-${String(input.pattern?.slug).replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '')}`; } -function evidenceEnvelope(input: EmitHistoricSqlEvidenceInput, connectionId: string) { +function evidenceEnvelope(input: EmitHistoricSqlEvidenceInput, connectionId: string, rawPaths: string[]) { if (input.kind === 'table_usage') { if (!input.table || !input.usage) { throw new Error('Invalid historic-SQL table usage evidence input.'); @@ -75,7 +75,7 @@ function evidenceEnvelope(input: EmitHistoricSqlEvidenceInput, connectionId: str kind: 'table_usage' as const, connectionId, table: input.table, - rawPath: input.rawPath, + rawPaths, usage: input.usage, }; } @@ -85,7 +85,7 @@ function evidenceEnvelope(input: EmitHistoricSqlEvidenceInput, connectionId: str return { kind: 'pattern' as const, connectionId, - rawPath: input.rawPath, + rawPaths, pattern: input.pattern, }; } @@ -102,9 +102,13 @@ export function createEmitHistoricSqlEvidenceTool(defaultContext?: EmitHistoricS if (!ingest || ingest.sourceKey !== 'historic-sql' || !configService || !context?.connectionId) { return 'Error: emit_historic_sql_evidence is only available during historic-sql ingest.'; } + const rawPaths = context.session?.allowedRawPaths ? [...context.session.allowedRawPaths].sort() : []; + if (rawPaths.length === 0) { + return 'Error: emit_historic_sql_evidence requires a WorkUnit context with at least one raw file.'; + } const unitKey = unitKeyForEvidence(input); - const evidence = evidenceEnvelope(input, context.connectionId); + const evidence = evidenceEnvelope(input, context.connectionId, rawPaths); const content = serializeHistoricSqlEvidence(evidence); await configService.writeFile( historicSqlEvidencePath(ingest.runId, unitKey), diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/evidence.ts b/packages/cli/src/context/ingest/adapters/historic-sql/evidence.ts index ddf26aed..6445decf 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/evidence.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/evidence.ts @@ -14,7 +14,7 @@ export const historicSqlTableUsageEvidenceSchema = z.object({ kind: z.literal('table_usage'), connectionId: z.string().min(1), table: z.string().min(1), - rawPath: z.string().min(1), + rawPaths: z.array(z.string().min(1)).min(1), usage: tableUsageOutputSchema, }); @@ -22,7 +22,7 @@ export const historicSqlTableUsageEvidenceSchema = z.object({ export const historicSqlPatternEvidenceSchema = z.object({ kind: z.literal('pattern'), connectionId: z.string().min(1), - rawPath: z.string().min(1), + rawPaths: z.array(z.string().min(1)).min(1), pattern: patternOutputSchema, }); diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/pattern-inputs.ts b/packages/cli/src/context/ingest/adapters/historic-sql/pattern-inputs.ts index 025fa43c..2e99836e 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/pattern-inputs.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/pattern-inputs.ts @@ -1,4 +1,5 @@ import { Buffer } from 'node:buffer'; +import { tableRefKey } from '../../../scan/table-ref.js'; import type { StagedPatternsInput } from './types.js'; const HISTORIC_SQL_PATTERN_WORKUNIT_DIR = 'patterns-input'; @@ -44,11 +45,16 @@ function sortedAuditTemplates(templates: readonly PatternTemplate[]): PatternTem function sortedPatternCandidates(templates: readonly PatternTemplate[]): PatternTemplate[] { return [...templates] .filter((template) => template.tablesTouched.length >= 2) - .map((template) => ({ ...template, tablesTouched: [...template.tablesTouched].sort() })) + .map((template) => ({ + ...template, + tablesTouched: [...template.tablesTouched].sort((left, right) => tableRefKey(left).localeCompare(tableRefKey(right))), + })) .sort((left, right) => { const cardinality = right.tablesTouched.length - left.tablesTouched.length; if (cardinality !== 0) return cardinality; - const tableSignature = left.tablesTouched.join('\0').localeCompare(right.tablesTouched.join('\0')); + const leftSignature = left.tablesTouched.map(tableRefKey).join('\0'); + const rightSignature = right.tablesTouched.map(tableRefKey).join('\0'); + const tableSignature = leftSignature.localeCompare(rightSignature); if (tableSignature !== 0) return tableSignature; return left.id.localeCompare(right.id); }); diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/projection.ts b/packages/cli/src/context/ingest/adapters/historic-sql/projection.ts index 272a96dc..2c7830a2 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/projection.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/projection.ts @@ -278,7 +278,7 @@ export async function projectHistoricSqlEvidence(input: HistoricSqlProjectionInp key: sourceName, targetConnectionId: input.connectionId, detail: `Merged historic-SQL usage for ${matchingEvidence.table}`, - rawPaths: [matchingEvidence.rawPath], + rawPaths: matchingEvidence.rawPaths, }); } } else if (entry.usage && !currentTables.has(tableRef)) { @@ -298,6 +298,7 @@ export async function projectHistoricSqlEvidence(input: HistoricSqlProjectionInp key: sourceName, targetConnectionId: input.connectionId, detail: `Marked historic-SQL usage stale for ${tableRef}`, + rawPaths: ['manifest.json'], }); } } @@ -341,7 +342,7 @@ export async function projectHistoricSqlEvidence(input: HistoricSqlProjectionInp type: reusable ? 'updated' : 'created', key, detail: `Projected historic-SQL pattern ${pattern.pattern.title}`, - rawPaths: [pattern.rawPath], + rawPaths: pattern.rawPaths, }); } @@ -361,6 +362,7 @@ export async function projectHistoricSqlEvidence(input: HistoricSqlProjectionInp type: 'updated', key: page.key, detail: `Archived stale historic-SQL pattern page ${page.key}`, + rawPaths: ['manifest.json'], }); continue; } @@ -377,6 +379,7 @@ export async function projectHistoricSqlEvidence(input: HistoricSqlProjectionInp type: 'updated', key: page.key, detail: `Marked historic-SQL pattern page ${page.key} stale`, + rawPaths: ['manifest.json'], }); } diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/query-history-filter-picker.ts b/packages/cli/src/context/ingest/adapters/historic-sql/query-history-filter-picker.ts new file mode 100644 index 00000000..3f77900d --- /dev/null +++ b/packages/cli/src/context/ingest/adapters/historic-sql/query-history-filter-picker.ts @@ -0,0 +1,283 @@ +import { z } from 'zod'; +import type { KtxLlmRuntimePort } from '../../../../context/llm/runtime-port.js'; +import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js'; +import { tableRefKey } from '../../../scan/table-ref.js'; +import type { KtxTableRef } from '../../../scan/types.js'; +import { bucketDistinctUsers, bucketExecutions, bucketRecency } from './buckets.js'; +import { + compileHistoricSqlRedactionPatterns, + redactHistoricSqlText, + type HistoricSqlRedactionPattern, +} from './redaction.js'; +import { includedQueryHistoryTableRefs } from './scope-membership.js'; +import { + aggregatedTemplateSchema, + historicSqlUnifiedPullConfigSchema, + type AggregatedTemplate, + type HistoricSqlDialect, + type HistoricSqlReader, +} from './types.js'; + +export interface QueryHistoryFilterProposal { + excludedRoles: Array<{ role: string; reason: string; pattern: string }>; + consideredRoleCount: number; + skipped: { reason: 'no-llm' | 'no-daemon' | 'no-in-scope-history' | 'user-block-present' } | null; + warnings: string[]; + parseFailedTemplateIds: string[]; +} + +export interface ProposeQueryHistoryServiceAccountFiltersInput { + connectionId: string; + dialect: HistoricSqlDialect; + queryClient: unknown; + reader: HistoricSqlReader; + sqlAnalysis: SqlAnalysisPort; + llmRuntime: KtxLlmRuntimePort | null; + pullConfig: unknown; + now?: Date; + userServiceAccountsPresent?: boolean; +} + +interface ParsedTemplateForPicker { + template: AggregatedTemplate; + tablesTouched: KtxTableRef[]; + includedTables: KtxTableRef[]; +} + +interface RoleAccumulator { + role: string; + executions: number; + distinctUsers: number; + lastSeen: string; + tables: Map; + templates: AggregatedTemplate[]; +} + +interface QueryHistoryRoleRecord { + role: string; + inScopeTables: string[]; + executionsBucket: string; + distinctUsersBucket: string; + recencyBucket: string; + representativeTemplates: Array<{ id: string; canonicalSql: string; dialect: HistoricSqlDialect }>; +} + +const queryHistoryFilterAdjudicationSchema = z.object({ + roles: z.array( + z.object({ + role: z.string().min(1), + exclude: z.boolean(), + reason: z.string().min(1), + }).strict(), + ), +}).strict(); + +type QueryHistoryFilterAdjudication = z.infer; + +function emptyProposal(skipped: QueryHistoryFilterProposal['skipped'], warnings: string[] = []): QueryHistoryFilterProposal { + return { excludedRoles: [], consideredRoleCount: 0, skipped, warnings, parseFailedTemplateIds: [] }; +} + +function displayTableRef(ref: KtxTableRef): string { + return [ref.catalog, ref.db, ref.name].filter((part): part is string => !!part && part.length > 0).join('.'); +} + +function redactTemplateSqlForPicker( + template: AggregatedTemplate, + redactors: readonly HistoricSqlRedactionPattern[], +): AggregatedTemplate { + if (redactors.length === 0) { + return template; + } + return { + ...template, + canonicalSql: redactHistoricSqlText(template.canonicalSql, redactors), + }; +} + +/** @internal */ +export function regexEscapeForExactRolePattern(role: string): string { + return `^${role.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&')}$`; +} + +function recordRole( + acc: RoleAccumulator, + template: AggregatedTemplate, + tables: readonly KtxTableRef[], + executions: number, +): void { + acc.executions += executions; + acc.distinctUsers = Math.max(acc.distinctUsers, template.stats.distinctUsers); + acc.lastSeen = template.stats.lastSeen > acc.lastSeen ? template.stats.lastSeen : acc.lastSeen; + for (const table of tables) { + acc.tables.set(tableRefKey(table), table); + } + acc.templates.push(template); +} + +function roleRecords(parsedTemplates: readonly ParsedTemplateForPicker[], now: Date): QueryHistoryRoleRecord[] { + const byRole = new Map(); + for (const parsed of parsedTemplates) { + for (const entry of parsed.template.topUsers) { + if (!entry.user || entry.user.trim().length === 0 || entry.executions <= 0) { + continue; + } + const role = entry.user.trim(); + const acc = + byRole.get(role) ?? + { + role, + executions: 0, + distinctUsers: 0, + lastSeen: '1970-01-01T00:00:00.000Z', + tables: new Map(), + templates: [], + }; + recordRole(acc, parsed.template, parsed.includedTables, entry.executions); + byRole.set(role, acc); + } + } + + return [...byRole.values()] + .sort((left, right) => right.executions - left.executions || left.role.localeCompare(right.role)) + .map((acc) => ({ + role: acc.role, + inScopeTables: [...acc.tables.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .slice(0, 25) + .map(([, ref]) => displayTableRef(ref)), + executionsBucket: bucketExecutions(acc.executions), + distinctUsersBucket: bucketDistinctUsers(acc.distinctUsers), + recencyBucket: bucketRecency(acc.lastSeen, now), + representativeTemplates: [...acc.templates] + .sort((left, right) => right.stats.executions - left.stats.executions || left.templateId.localeCompare(right.templateId)) + .slice(0, 3) + .map((template) => ({ + id: template.templateId, + canonicalSql: template.canonicalSql, + dialect: template.dialect, + })), + })); +} + +function adjudicationSystemPrompt(): string { + return [ + 'You are helping ktx decide whether observed query-history roles are operational service accounts.', + 'Default every role to keep. Mark exclude true only when the aggregate evidence clearly shows loader, ELT, reverse-ETL, export, refresh, or maintenance traffic rather than analyst or BI-dashboard usage.', + 'Use only the observed role records. Do not rely on a hardcoded denylist. Return structured output only.', + ].join('\n'); +} + +export async function proposeQueryHistoryServiceAccountFilters( + input: ProposeQueryHistoryServiceAccountFiltersInput, +): Promise { + if (!input.llmRuntime) { + return emptyProposal({ reason: 'no-llm' }); + } + + const config = historicSqlUnifiedPullConfigSchema.parse(input.pullConfig); + const redactors = compileHistoricSqlRedactionPatterns(config.redactionPatterns); + const now = input.now ?? new Date(); + const windowDays = 'windowDays' in config ? config.windowDays : 90; + const windowStart = new Date(now.getTime() - windowDays * 24 * 60 * 60 * 1000); + const warnings: string[] = []; + const parseFailedTemplateIds: string[] = []; + const snapshot: AggregatedTemplate[] = []; + + try { + for await (const row of input.reader.fetchAggregated(input.queryClient, { start: windowStart, end: now }, config)) { + snapshot.push(aggregatedTemplateSchema.parse(row)); + } + } catch (error) { + return emptyProposal(null, [ + `query_history_filter_picker_read_failed:${error instanceof Error ? error.message : String(error)}`, + ]); + } + + if (snapshot.length === 0) { + return emptyProposal({ reason: 'no-in-scope-history' }); + } + + const analysisItems = snapshot.map((template) => ({ id: template.templateId, sql: template.canonicalSql })); + const analysisOptions = + config.modeledTableCatalog.length > 0 ? { catalog: { tables: config.modeledTableCatalog } } : undefined; + let analysis: Awaited>; + try { + analysis = await input.sqlAnalysis.analyzeBatch(analysisItems, input.dialect, analysisOptions); + } catch (error) { + return emptyProposal({ reason: 'no-daemon' }, [ + `query_history_filter_picker_analysis_failed:${error instanceof Error ? error.message : String(error)}`, + ]); + } + + const parsedTemplates: ParsedTemplateForPicker[] = []; + for (const template of snapshot) { + const parsed = analysis.get(template.templateId); + if (!parsed || parsed.error) { + parseFailedTemplateIds.push(template.templateId); + continue; + } + const tablesTouched = [...new Map(parsed.tablesTouched.map((ref) => [tableRefKey(ref), ref])).values()] + .filter((ref) => ref.name.length > 0) + .sort((left, right) => tableRefKey(left).localeCompare(tableRefKey(right))); + const includedTables = includedQueryHistoryTableRefs(tablesTouched, config); + if (includedTables.length === 0) { + continue; + } + parsedTemplates.push({ + template: redactTemplateSqlForPicker(template, redactors), + tablesTouched, + includedTables, + }); + } + + const records = roleRecords(parsedTemplates, now); + if (records.length <= 1) { + return { + excludedRoles: [], + consideredRoleCount: records.length, + skipped: { reason: 'no-in-scope-history' }, + warnings, + parseFailedTemplateIds, + }; + } + + let generated: QueryHistoryFilterAdjudication; + try { + generated = await input.llmRuntime.generateObject({ + role: 'candidateExtraction', + system: adjudicationSystemPrompt(), + prompt: JSON.stringify({ connectionId: input.connectionId, dialect: input.dialect, roles: records }), + schema: queryHistoryFilterAdjudicationSchema, + }); + } catch (error) { + return { + excludedRoles: [], + consideredRoleCount: records.length, + skipped: { reason: 'no-llm' }, + warnings: [ + ...warnings, + `query_history_filter_picker_llm_failed:${error instanceof Error ? error.message : String(error)}`, + ], + parseFailedTemplateIds, + }; + } + + const knownRoles = new Set(records.map((record) => record.role)); + const excludedRoles = generated.roles + .filter((role) => role.exclude && knownRoles.has(role.role)) + .sort((left, right) => left.role.localeCompare(right.role)) + .map((role) => ({ + role: role.role, + reason: role.reason, + pattern: regexEscapeForExactRolePattern(role.role), + })); + + return { + excludedRoles, + consideredRoleCount: records.length, + skipped: input.userServiceAccountsPresent ? { reason: 'user-block-present' } : null, + warnings, + parseFailedTemplateIds, + }; +} diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/scope-floor.ts b/packages/cli/src/context/ingest/adapters/historic-sql/scope-floor.ts new file mode 100644 index 00000000..23b36a0e --- /dev/null +++ b/packages/cli/src/context/ingest/adapters/historic-sql/scope-floor.ts @@ -0,0 +1,260 @@ +import type { Dirent } from 'node:fs'; +import { access, readdir, readFile } from 'node:fs/promises'; +import { join, relative } from 'node:path'; +import YAML from 'yaml'; +import { getDriverRegistration } from '../../../connections/drivers.js'; +import { parseDottedTableEntry } from '../../../scan/enabled-tables.js'; +import { tableRefKey, tableRefSet, type KtxTableRefKey } from '../../../scan/table-ref.js'; +import type { KtxTableRef } from '../../../scan/types.js'; +import { readLiveDatabaseTableFiles } from '../live-database/stage.js'; + +export interface QueryHistoryScopeFloorInput { + projectDir: string; + connectionId: string; + driver: string; + connection: Record; + storedQueryHistory: Record; +} + +export interface QueryHistoryScopeFloor { + enabledTables: KtxTableRef[]; + enabledTableKeys: ReadonlySet | null; + enabledSchemas: string[]; + modeledTableCatalog: KtxTableRef[]; + floorDisabled: boolean; + warnings: string[]; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value + .filter((item): item is string => typeof item === 'string' && item.trim().length > 0) + .map((item) => item.trim()) + : []; +} + +function tableRefsFromValues(values: unknown): KtxTableRef[] { + if (!Array.isArray(values)) return []; + return values.flatMap((value) => { + if (typeof value === 'string') { + const ref = parseDottedTableEntry(value); + return ref ? [ref] : []; + } + if (isRecord(value) && typeof value.name === 'string' && value.name.length > 0) { + return [ + { + catalog: typeof value.catalog === 'string' ? value.catalog : null, + db: typeof value.db === 'string' ? value.db : null, + name: value.name, + }, + ]; + } + return []; + }); +} + +function declaredSchemas(driver: string, connection: Record): string[] { + const key = getDriverRegistration(driver)?.scopeConfigKey; + if (!key) return []; + return [...new Set(stringArray(connection[key]))].sort(); +} + +function uniqueSortedTableRefs(refs: readonly KtxTableRef[]): KtxTableRef[] { + const byKey = new Map(); + for (const ref of refs) { + byKey.set(tableRefKey(ref), ref); + } + return [...byKey.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([, ref]) => ref); +} + +async function latestLiveDatabaseScanDir(projectDir: string, connectionId: string): Promise { + const root = join(projectDir, 'raw-sources', connectionId, 'live-database'); + let entries: Dirent[]; + try { + entries = await readdir(root, { withFileTypes: true }); + } catch (error) { + if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') return null; + throw error; + } + const syncDirs = entries + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort() + .reverse(); + for (const syncDir of syncDirs) { + const absolute = join(root, syncDir); + try { + await access(join(absolute, 'connection.json')); + return absolute; + } catch { + continue; + } + } + return null; +} + +async function scannedTableRefs( + projectDir: string, + connectionId: string, +): Promise<{ refs: KtxTableRef[]; catalogAvailable: boolean; warnings: string[] }> { + const scanDir = await latestLiveDatabaseScanDir(projectDir, connectionId); + if (!scanDir) { + return { refs: [], catalogAvailable: false, warnings: [] }; + } + try { + const tableFiles = await readLiveDatabaseTableFiles(scanDir); + return { + refs: uniqueSortedTableRefs( + tableFiles.map(({ table }) => ({ catalog: table.catalog, db: table.db, name: table.name })), + ), + catalogAvailable: true, + warnings: [], + }; + } catch (error) { + return { + refs: [], + catalogAvailable: false, + warnings: [ + `query_history_scope_floor_catalog_read_failed:live_database_scan:${error instanceof Error ? error.message : String(error)}`, + ], + }; + } +} + +async function listYamlFiles(root: string): Promise { + try { + const entries = await readdir(root, { withFileTypes: true, recursive: true }); + return entries + .filter((entry) => entry.isFile() && /\.ya?ml$/i.test(entry.name)) + .map((entry) => relative(root, join(entry.parentPath, entry.name)).replace(/\\/g, '/')) + .sort(); + } catch (error) { + if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') return []; + throw error; + } +} + +function refsFromManifest(content: string): KtxTableRef[] { + const parsed = YAML.parse(content) as unknown; + if (!isRecord(parsed) || !isRecord(parsed.tables)) return []; + return Object.values(parsed.tables).flatMap((entry) => { + if (!isRecord(entry) || typeof entry.table !== 'string') return []; + const ref = parseDottedTableEntry(entry.table); + return ref ? [ref] : []; + }); +} + +function refsFromStandaloneSource(content: string): KtxTableRef[] { + const parsed = YAML.parse(content) as unknown; + if (!isRecord(parsed) || typeof parsed.table !== 'string') return []; + const ref = parseDottedTableEntry(parsed.table); + return ref ? [ref] : []; +} + +async function semanticTableRefs( + projectDir: string, + connectionId: string, +): Promise<{ refs: KtxTableRef[]; warnings: string[] }> { + const root = join(projectDir, 'semantic-layer', connectionId); + const files = await listYamlFiles(root); + const refs: KtxTableRef[] = []; + const warnings: string[] = []; + for (const file of files) { + try { + const content = await readFile(join(root, file), 'utf-8'); + refs.push(...(file.startsWith('_schema/') ? refsFromManifest(content) : refsFromStandaloneSource(content))); + } catch (error) { + warnings.push( + `query_history_scope_floor_catalog_read_failed:${file}:${error instanceof Error ? error.message : String(error)}`, + ); + } + } + return { refs: uniqueSortedTableRefs(refs), warnings }; +} + +export async function resolveQueryHistoryScopeFloor(input: QueryHistoryScopeFloorInput): Promise { + const explicitEnabledTables = [ + ...tableRefsFromValues(input.storedQueryHistory.enabledTables), + ...tableRefsFromValues(input.connection.enabled_tables), + ]; + const semanticTables = await semanticTableRefs(input.projectDir, input.connectionId); + const scannedTables = await scannedTableRefs(input.projectDir, input.connectionId); + const modeledTables = uniqueSortedTableRefs([ + ...semanticTables.refs, + ...scannedTables.refs, + ...explicitEnabledTables, + ]); + const warnings = [...semanticTables.warnings, ...scannedTables.warnings]; + + if (explicitEnabledTables.length > 0) { + return { + enabledTables: explicitEnabledTables, + enabledTableKeys: tableRefSet(explicitEnabledTables), + enabledSchemas: [], + modeledTableCatalog: modeledTables, + floorDisabled: false, + warnings, + }; + } + + const explicitSchemas = stringArray(input.storedQueryHistory.enabledSchemas); + if (explicitSchemas.includes('*')) { + return { + enabledTables: [], + enabledTableKeys: null, + enabledSchemas: ['*'], + modeledTableCatalog: modeledTables, + floorDisabled: true, + warnings, + }; + } + if (explicitSchemas.length > 0) { + if (!scannedTables.catalogAvailable || modeledTables.length === 0) { + return { + enabledTables: [], + enabledTableKeys: null, + enabledSchemas: ['*'], + modeledTableCatalog: modeledTables, + floorDisabled: true, + warnings: [...warnings, 'query_history_scope_floor_disabled:catalog_unavailable'], + }; + } + return { + enabledTables: [], + enabledTableKeys: null, + enabledSchemas: [...new Set(explicitSchemas)].sort(), + modeledTableCatalog: modeledTables, + floorDisabled: false, + warnings, + }; + } + + const schemas = new Set(declaredSchemas(input.driver, input.connection)); + for (const ref of semanticTables.refs) { + if (ref.db) schemas.add(ref.db); + } + if (schemas.size > 0 && (!scannedTables.catalogAvailable || modeledTables.length === 0)) { + return { + enabledTables: [], + enabledTableKeys: null, + enabledSchemas: ['*'], + modeledTableCatalog: modeledTables, + floorDisabled: true, + warnings: [...warnings, 'query_history_scope_floor_disabled:catalog_unavailable'], + }; + } + return { + enabledTables: [], + enabledTableKeys: null, + enabledSchemas: [...schemas].sort(), + modeledTableCatalog: modeledTables, + floorDisabled: false, + warnings, + }; +} diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/scope-membership.ts b/packages/cli/src/context/ingest/adapters/historic-sql/scope-membership.ts new file mode 100644 index 00000000..8852a82d --- /dev/null +++ b/packages/cli/src/context/ingest/adapters/historic-sql/scope-membership.ts @@ -0,0 +1,45 @@ +import { tableRefKey, tableRefSet } from '../../../scan/table-ref.js'; +import type { KtxTableRef } from '../../../scan/types.js'; + +export interface QueryHistoryScopeMembershipConfig { + enabledTables: readonly KtxTableRef[]; + enabledSchemas: readonly string[]; +} + +function schemaNameForRef(ref: KtxTableRef): string | null { + return ref.db && ref.db.length > 0 ? ref.db : null; +} + +function schemaNamesFromConfig(enabledSchemas: readonly string[]): Set { + return new Set(enabledSchemas.filter((schema) => schema !== '*')); +} + +export function isQueryHistoryScopeFloorDisabled(config: QueryHistoryScopeMembershipConfig): boolean { + return config.enabledSchemas.includes('*'); +} + +export function shouldFailOpenQueryHistoryScope(config: QueryHistoryScopeMembershipConfig): boolean { + return ( + config.enabledTables.length === 0 && + !isQueryHistoryScopeFloorDisabled(config) && + config.enabledSchemas.length === 0 + ); +} + +export function includedQueryHistoryTableRefs( + tablesTouched: readonly KtxTableRef[], + config: QueryHistoryScopeMembershipConfig, +): KtxTableRef[] { + if (config.enabledTables.length > 0) { + const enabled = tableRefSet(config.enabledTables); + return tablesTouched.filter((ref) => enabled.has(tableRefKey(ref))); + } + if (isQueryHistoryScopeFloorDisabled(config) || shouldFailOpenQueryHistoryScope(config)) { + return [...tablesTouched]; + } + const schemas = schemaNamesFromConfig(config.enabledSchemas); + return tablesTouched.filter((ref) => { + const schema = schemaNameForRef(ref); + return schema !== null && schemas.has(schema); + }); +} diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/snowflake-query-history-reader.ts b/packages/cli/src/context/ingest/adapters/historic-sql/snowflake-query-history-reader.ts index 539df3c3..65aafb12 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/snowflake-query-history-reader.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/snowflake-query-history-reader.ts @@ -188,26 +188,75 @@ export class SnowflakeHistoricSqlQueryHistoryReader { config: HistoricSqlUnifiedPullConfig, ): AsyncIterable { const sql = ` +WITH filtered_queries AS ( + SELECT + query_hash, + query_text, + user_name, + start_time, + total_elapsed_time, + execution_status, + rows_produced + FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY + WHERE query_text IS NOT NULL + AND query_type IN ('SELECT', 'MERGE') + AND start_time >= ${timestampLiteral(window.start)} + AND start_time < ${timestampLiteral(window.end)} +), +template_stats AS ( + SELECT + query_hash AS template_id, + MIN(query_text) AS canonical_sql, + COUNT(*) AS executions, + COUNT(DISTINCT user_name) AS distinct_users, + MIN(start_time) AS first_seen, + MAX(start_time) AS last_seen, + APPROX_PERCENTILE(total_elapsed_time, 0.50) AS p50_ms, + APPROX_PERCENTILE(total_elapsed_time, 0.95) AS p95_ms, + DIV0(COUNT_IF(execution_status != 'SUCCESS'), COUNT(*)) AS error_rate, + SUM(rows_produced) AS rows_produced + FROM filtered_queries + GROUP BY query_hash + HAVING COUNT(*) >= ${config.minExecutions} +), +template_users AS ( + SELECT + query_hash AS template_id, + user_name AS user, + COUNT(*) AS executions, + MAX(start_time) AS last_seen + FROM filtered_queries + GROUP BY query_hash, user_name +) SELECT - query_hash AS template_id, - MIN(query_text) AS canonical_sql, - COUNT(*) AS executions, - COUNT(DISTINCT user_name) AS distinct_users, - MIN(start_time) AS first_seen, - MAX(start_time) AS last_seen, - APPROX_PERCENTILE(total_elapsed_time, 0.50) AS p50_ms, - APPROX_PERCENTILE(total_elapsed_time, 0.95) AS p95_ms, - DIV0(COUNT_IF(execution_status != 'SUCCESS'), COUNT(*)) AS error_rate, - SUM(rows_produced) AS rows_produced, - ARRAY_AGG(OBJECT_CONSTRUCT('user', user_name, 'executions', 1)) WITHIN GROUP (ORDER BY start_time DESC)::string AS top_users -FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY -WHERE query_text IS NOT NULL - AND query_type IN ('SELECT', 'MERGE') - AND start_time >= ${timestampLiteral(window.start)} - AND start_time < ${timestampLiteral(window.end)} -GROUP BY query_hash -HAVING COUNT(*) >= ${config.minExecutions} -ORDER BY executions DESC`.trim(); + stats.template_id, + stats.canonical_sql, + stats.executions, + stats.distinct_users, + stats.first_seen, + stats.last_seen, + stats.p50_ms, + stats.p95_ms, + stats.error_rate, + stats.rows_produced, + ARRAY_AGG( + OBJECT_CONSTRUCT('user', users.user, 'executions', users.executions) + ) WITHIN GROUP (ORDER BY users.executions DESC, users.last_seen DESC)::string AS top_users +FROM template_stats AS stats +JOIN template_users AS users + ON users.template_id = stats.template_id +GROUP BY + stats.template_id, + stats.canonical_sql, + stats.executions, + stats.distinct_users, + stats.first_seen, + stats.last_seen, + stats.p50_ms, + stats.p95_ms, + stats.error_rate, + stats.rows_produced +ORDER BY stats.executions DESC`.trim(); const result = await queryClient(client).executeQuery(sql); if (result.error) { throw grantsError(result.error); diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.test.ts deleted file mode 100644 index d49c3a1d..00000000 --- a/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.test.ts +++ /dev/null @@ -1,436 +0,0 @@ -import { mkdtemp, readFile, readdir } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, expect, it, vi } from 'vitest'; -import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js'; -import { stageHistoricSqlAggregatedSnapshot } from './stage-unified.js'; -import type { AggregatedTemplate, HistoricSqlReader } from './types.js'; - -async function tempDir(): Promise { - return mkdtemp(join(tmpdir(), 'historic-sql-unified-stage-')); -} - -async function readJson(root: string, relPath: string): Promise { - return JSON.parse(await readFile(join(root, relPath), 'utf-8')) as T; -} - -function aggregate(overrides: Partial & { templateId: string; canonicalSql: string }): AggregatedTemplate { - return { - templateId: overrides.templateId, - canonicalSql: overrides.canonicalSql, - dialect: overrides.dialect ?? 'postgres', - stats: overrides.stats ?? { - executions: 42, - distinctUsers: 3, - firstSeen: '2026-05-01T00:00:00.000Z', - lastSeen: '2026-05-11T00:00:00.000Z', - p50RuntimeMs: 20, - p95RuntimeMs: 80, - errorRate: 0, - rowsProduced: 100, - }, - topUsers: overrides.topUsers ?? [{ user: 'analyst', executions: 40 }], - }; -} - -describe('stageHistoricSqlAggregatedSnapshot', () => { - it('batch parses templates and writes stable table and patterns artifacts', async () => { - const stagedDir = await tempDir(); - const reader: HistoricSqlReader = { - async probe() { - return { warnings: ['pg_stat_statements.track is none; aggregation still proceeds'], info: [] }; - }, - async *fetchAggregated() { - yield aggregate({ - templateId: 'orders-by-status', - canonicalSql: 'select o.status, count(*) from public.orders o join public.customers c on c.id = o.customer_id where o.created_at >= $1 group by o.status', - }); - yield aggregate({ - templateId: 'service-account-only', - canonicalSql: 'select * from public.orders where id = $1', - stats: { - executions: 20, - distinctUsers: 1, - firstSeen: '2026-05-01T00:00:00.000Z', - lastSeen: '2026-05-11T00:00:00.000Z', - p50RuntimeMs: 5, - p95RuntimeMs: 10, - errorRate: 0, - rowsProduced: 1, - }, - topUsers: [{ user: 'svc_loader', executions: 20 }], - }); - yield aggregate({ - templateId: 'bad-parse', - canonicalSql: 'select broken from', - }); - }, - }; - const sqlAnalysis: SqlAnalysisPort = { - analyzeForFingerprint: vi.fn(), - analyzeBatch: vi.fn(async () => new Map([ - [ - 'orders-by-status', - { - tablesTouched: ['public.orders', 'public.customers'], - columnsByClause: { - select: ['status'], - where: ['created_at'], - join: ['customer_id'], - groupBy: ['status'], - }, - }, - ], - ['bad-parse', { tablesTouched: [], columnsByClause: {}, error: 'parse failed' }], - ])), - validateReadOnly: vi.fn(async () => ({ ok: true })), - }; - - await stageHistoricSqlAggregatedSnapshot({ - stagedDir, - connectionId: 'warehouse', - queryClient: {}, - reader, - sqlAnalysis, - pullConfig: { - dialect: 'postgres', - filters: { - serviceAccounts: { patterns: ['^svc_'], mode: 'exclude' }, - }, - }, - now: new Date('2026-05-11T12:00:00.000Z'), - }); - - expect(sqlAnalysis.analyzeBatch).toHaveBeenCalledTimes(1); - expect(sqlAnalysis.analyzeBatch).toHaveBeenCalledWith( - [ - { - id: 'orders-by-status', - sql: 'select o.status, count(*) from public.orders o join public.customers c on c.id = o.customer_id where o.created_at >= $1 group by o.status', - }, - { id: 'bad-parse', sql: 'select broken from' }, - ], - 'postgres', - ); - - expect(await readdir(join(stagedDir, 'tables'))).toEqual(['public.customers.json', 'public.orders.json']); - - const manifest = await readJson>(stagedDir, 'manifest.json'); - expect(manifest).toMatchObject({ - source: 'historic-sql', - connectionId: 'warehouse', - dialect: 'postgres', - snapshotRowCount: 3, - touchedTableCount: 2, - parseFailures: 1, - warnings: ['parse_failed:bad-parse'], - probeWarnings: ['pg_stat_statements.track is none; aggregation still proceeds'], - staleArchiveAfterDays: 90, - }); - - const orders = await readJson>(stagedDir, 'tables/public.orders.json'); - expect(orders).toMatchObject({ - table: 'public.orders', - stats: { - executionsBucket: '10-100', - distinctUsersBucket: '2-5', - errorRateBucket: 'none', - p95RuntimeBucket: '<100ms', - recencyBucket: 'current', - }, - columnsByClause: { - select: [['status', 'high']], - where: [['created_at', 'high']], - join: [['customer_id', 'high']], - groupBy: [['status', 'high']], - }, - observedJoins: [{ withTable: 'public.customers', on: ['customer_id'], freq: 'high' }], - topTemplates: [ - { - id: 'orders-by-status', - topUsers: [{ user: 'analyst' }], - }, - ], - }); - expect(orders.topTemplates[0].canonicalSql).toContain('group by o.status'); - - const patterns = await readJson>(stagedDir, 'patterns-input.json'); - expect(patterns.templates).toEqual([ - { - id: 'orders-by-status', - canonicalSql: expect.stringContaining('public.orders'), - tablesTouched: ['public.customers', 'public.orders'], - executionsBucket: '10-100', - distinctUsersBucket: '2-5', - dialect: 'postgres', - }, - ]); - }); - - it('redacts configured SQL substrings in staged artifacts while analyzing original SQL', async () => { - const stagedDir = await tempDir(); - const originalSql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret - const reader: HistoricSqlReader = { - async probe() { - return { warnings: [], info: [] }; - }, - async *fetchAggregated() { - yield aggregate({ - templateId: 'api-events-with-secret', - canonicalSql: originalSql, - stats: { - executions: 15, - distinctUsers: 2, - firstSeen: '2026-05-01T00:00:00.000Z', - lastSeen: '2026-05-11T00:00:00.000Z', - p50RuntimeMs: 12, - p95RuntimeMs: 25, - errorRate: 0, - rowsProduced: 15, - }, - }); - }, - }; - const sqlAnalysis: SqlAnalysisPort = { - analyzeForFingerprint: vi.fn(), - analyzeBatch: vi.fn(async () => new Map([ - [ - 'api-events-with-secret', - { - tablesTouched: ['public.api_events'], - columnsByClause: { - select: [], - where: ['api_key', 'note'], - join: [], - groupBy: [], - }, - }, - ], - ])), - validateReadOnly: vi.fn(async () => ({ ok: true })), - }; - - await stageHistoricSqlAggregatedSnapshot({ - stagedDir, - connectionId: 'warehouse', - queryClient: {}, - reader, - sqlAnalysis, - pullConfig: { - dialect: 'postgres', - redactionPatterns: ['sk_live_[A-Za-z0-9]+', '(?i)secret_token_[a-z0-9]+'], - }, - now: new Date('2026-05-11T12:00:00.000Z'), - }); - - expect(sqlAnalysis.analyzeBatch).toHaveBeenCalledWith( - [{ id: 'api-events-with-secret', sql: originalSql }], - 'postgres', - ); - - const tableJson = await readFile(join(stagedDir, 'tables/public.api_events.json'), 'utf-8'); - const patternsJson = await readFile(join(stagedDir, 'patterns-input.json'), 'utf-8'); - expect(tableJson).not.toContain('sk_live_abc123'); - expect(tableJson).not.toContain('Secret_Token_9f'); - expect(patternsJson).not.toContain('sk_live_abc123'); - expect(patternsJson).not.toContain('Secret_Token_9f'); - expect(tableJson).toContain('[REDACTED]'); - expect(patternsJson).toContain('[REDACTED]'); - }); - - it('limits staged table artifacts to configured enabled tables', async () => { - const stagedDir = await tempDir(); - const reader: HistoricSqlReader = { - async probe() { - return { warnings: [], info: [] }; - }, - async *fetchAggregated() { - yield aggregate({ - templateId: 'selected-qualified', - canonicalSql: 'select count(*) from orbit_analytics.int_active_contract_arr', - }); - yield aggregate({ - templateId: 'selected-unqualified', - canonicalSql: 'select count(*) from int_customer_health_signals', - }); - yield aggregate({ - templateId: 'unselected', - canonicalSql: 'select count(*) from orbit_raw.accounts', - }); - }, - }; - const sqlAnalysis: SqlAnalysisPort = { - analyzeForFingerprint: vi.fn(), - analyzeBatch: vi.fn(async () => new Map([ - [ - 'selected-qualified', - { - tablesTouched: ['orbit_analytics.int_active_contract_arr'], - columnsByClause: { select: [], where: [], join: [], groupBy: [] }, - }, - ], - [ - 'selected-unqualified', - { - tablesTouched: ['int_customer_health_signals'], - columnsByClause: { select: [], where: [], join: [], groupBy: [] }, - }, - ], - [ - 'unselected', - { - tablesTouched: ['orbit_raw.accounts'], - columnsByClause: { select: [], where: [], join: [], groupBy: [] }, - }, - ], - ])), - validateReadOnly: vi.fn(async () => ({ ok: true })), - }; - - await stageHistoricSqlAggregatedSnapshot({ - stagedDir, - connectionId: 'warehouse', - queryClient: {}, - reader, - sqlAnalysis, - pullConfig: { - dialect: 'postgres', - enabledTables: [ - 'orbit_analytics.int_active_contract_arr', - 'orbit_analytics.int_customer_health_signals', - ], - }, - now: new Date('2026-05-11T12:00:00.000Z'), - }); - - expect(await readdir(join(stagedDir, 'tables'))).toEqual([ - 'int_customer_health_signals.json', - 'orbit_analytics.int_active_contract_arr.json', - ]); - const manifest = await readJson>(stagedDir, 'manifest.json'); - expect(manifest.touchedTableCount).toBe(2); - const patterns = await readJson>(stagedDir, 'patterns-input.json'); - expect(patterns.templates.map((entry: any) => entry.id)).toEqual(['selected-qualified', 'selected-unqualified']); - }); - - it('preserves full patterns audit input and writes bounded cross-table pattern shards', async () => { - const stagedDir = await tempDir(); - const largeSql = `select * from public.orders o join public.customers c on c.id = o.customer_id where payload = '${'x'.repeat(8000)}'`; - const reader: HistoricSqlReader = { - async probe() { - return { warnings: [], info: [] }; - }, - async *fetchAggregated() { - yield aggregate({ - templateId: 'orders-customers-a', - canonicalSql: largeSql, - stats: { - executions: 25, - distinctUsers: 4, - firstSeen: '2026-05-01T00:00:00.000Z', - lastSeen: '2026-05-11T00:00:00.000Z', - p50RuntimeMs: 15, - p95RuntimeMs: 90, - errorRate: 0, - rowsProduced: 250, - }, - }); - yield aggregate({ - templateId: 'orders-customers-b', - canonicalSql: largeSql.replace('payload', 'payload_b'), - stats: { - executions: 22, - distinctUsers: 3, - firstSeen: '2026-05-01T00:00:00.000Z', - lastSeen: '2026-05-11T00:00:00.000Z', - p50RuntimeMs: 20, - p95RuntimeMs: 95, - errorRate: 0, - rowsProduced: 220, - }, - }); - yield aggregate({ - templateId: 'orders-single-table', - canonicalSql: 'select count(*) from public.orders', - stats: { - executions: 30, - distinctUsers: 2, - firstSeen: '2026-05-01T00:00:00.000Z', - lastSeen: '2026-05-11T00:00:00.000Z', - p50RuntimeMs: 10, - p95RuntimeMs: 20, - errorRate: 0, - rowsProduced: 30, - }, - }); - }, - }; - const sqlAnalysis: SqlAnalysisPort = { - analyzeForFingerprint: vi.fn(), - analyzeBatch: vi.fn(async () => new Map([ - [ - 'orders-customers-a', - { - tablesTouched: ['public.orders', 'public.customers'], - columnsByClause: { - select: [], - where: ['payload'], - join: ['customer_id', 'id'], - groupBy: [], - }, - }, - ], - [ - 'orders-customers-b', - { - tablesTouched: ['public.orders', 'public.customers'], - columnsByClause: { - select: [], - where: ['payload_b'], - join: ['customer_id', 'id'], - groupBy: [], - }, - }, - ], - [ - 'orders-single-table', - { - tablesTouched: ['public.orders'], - columnsByClause: { - select: [], - where: [], - join: [], - groupBy: [], - }, - }, - ], - ])), - validateReadOnly: vi.fn(async () => ({ ok: true })), - }; - - await stageHistoricSqlAggregatedSnapshot({ - stagedDir, - connectionId: 'warehouse', - queryClient: {}, - reader, - sqlAnalysis, - pullConfig: { dialect: 'postgres' }, - now: new Date('2026-05-11T12:00:00.000Z'), - }); - - const audit = await readJson>(stagedDir, 'patterns-input.json'); - expect(audit.templates.map((entry: any) => entry.id)).toEqual([ - 'orders-customers-a', - 'orders-customers-b', - 'orders-single-table', - ]); - - const firstShard = await readJson>(stagedDir, 'patterns-input/part-0001.json'); - expect(firstShard.templates.map((entry: any) => entry.id)).toEqual(['orders-customers-a', 'orders-customers-b']); - expect(firstShard.templates.some((entry: any) => entry.id === 'orders-single-table')).toBe(false); - - const manifest = await readJson>(stagedDir, 'manifest.json'); - expect(manifest.warnings).toEqual([]); - }); -}); diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.ts b/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.ts index 70997648..84ec75a7 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.ts @@ -1,6 +1,8 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js'; +import { tableRefKey, type KtxTableRefKey } from '../../../scan/table-ref.js'; +import type { KtxTableRef } from '../../../scan/types.js'; import { bucketDistinctUsers, bucketErrorRate, @@ -15,6 +17,11 @@ import { redactHistoricSqlText, type HistoricSqlRedactionPattern, } from './redaction.js'; +import { + includedQueryHistoryTableRefs, + isQueryHistoryScopeFloorDisabled, + shouldFailOpenQueryHistoryScope, +} from './scope-membership.js'; import { HISTORIC_SQL_SOURCE_KEY, aggregatedTemplateSchema, @@ -38,17 +45,13 @@ interface StageHistoricSqlAggregatedSnapshotInput { interface ParsedTemplate { template: AggregatedTemplate; - tablesTouched: string[]; - includedTables: string[]; + tablesTouched: KtxTableRef[]; + includedTables: KtxTableRef[]; columnsByClause: Record; } -interface EnabledTableFilter { - exact: Set; - uniqueUnqualified: Set; -} - interface TableAccumulator { + tableRef: KtxTableRef; table: string; executions: number; distinctUsers: number; @@ -79,8 +82,21 @@ function matchesAny(value: string | null, patterns: RegExp[]): boolean { return !!value && patterns.some((pattern) => pattern.test(value)); } +// ktx's own warehouse scan emits relationship- and column-profiling probes that land in +// pg_stat_statements (relationship-validation, relationship-composite-candidates, and each +// dialect's relationship value aggregation). They are ktx introspection, not genuine query +// usage, so they must not be mined back as query history. The markers are ktx-owned +// identifiers, stable across dialects. +function isKtxScanProbe(sql: string): boolean { + if (/\brelationship_profile_values\b/i.test(sql)) { + return true; + } + return /\bchild_values\b/i.test(sql) && /\bparent_values\b/i.test(sql); +} + function shouldDropBySql(sql: string, config: HistoricSqlUnifiedPullConfig): boolean { if (NOISE_PREFIX_RE.test(sql) || SYSTEM_TABLE_RE.test(sql)) return true; + if (isKtxScanProbe(sql)) return true; if (config.filters.dropTrivialProbes !== false && TRIVIAL_SQL_RE.test(sql)) return true; return false; } @@ -92,8 +108,7 @@ function shouldDropByUsers(template: AggregatedTemplate, config: HistoricSqlUnif const matchingExecutions = template.topUsers .filter((entry) => matchesAny(entry.user, patterns)) .reduce((sum, entry) => sum + entry.executions, 0); - const allExecutions = template.topUsers.reduce((sum, entry) => sum + entry.executions, 0); - const serviceOnly = allExecutions > 0 && matchingExecutions >= allExecutions; + const serviceOnly = template.stats.executions > 0 && matchingExecutions >= template.stats.executions; return service.mode === 'exclude' ? serviceOnly : !serviceOnly; } @@ -109,43 +124,8 @@ function shouldDropTemplate(template: AggregatedTemplate, config: HistoricSqlUni return false; } -function normalizeTableIdentifier(value: string): string { - return value.trim().toLowerCase(); -} - -function unqualifiedTableIdentifier(value: string): string { - const parts = normalizeTableIdentifier(value).split('.').filter(Boolean); - return parts.at(-1) ?? ''; -} - -function buildEnabledTableFilter(enabledTables: string[]): EnabledTableFilter | null { - if (enabledTables.length === 0) { - return null; - } - const exact = new Set(enabledTables.map(normalizeTableIdentifier).filter((value) => value.length > 0)); - const unqualifiedCounts = new Map(); - for (const table of exact) { - const unqualified = unqualifiedTableIdentifier(table); - if (unqualified.length > 0) { - unqualifiedCounts.set(unqualified, (unqualifiedCounts.get(unqualified) ?? 0) + 1); - } - } - return { - exact, - uniqueUnqualified: new Set( - [...unqualifiedCounts.entries()] - .filter(([, count]) => count === 1) - .map(([table]) => table), - ), - }; -} - -function isEnabledTable(table: string, filter: EnabledTableFilter | null): boolean { - if (!filter) { - return true; - } - const normalized = normalizeTableIdentifier(table); - return filter.exact.has(normalized) || filter.uniqueUnqualified.has(unqualifiedTableIdentifier(normalized)); +function displayTableRef(ref: KtxTableRef): string { + return [ref.catalog, ref.db, ref.name].filter((part): part is string => !!part && part.length > 0).join('.'); } function historicSqlWindowDays(config: HistoricSqlUnifiedPullConfig): number { @@ -180,9 +160,10 @@ function recordJoin(acc: TableAccumulator, otherTable: string, columns: string[] } } -function accumulatorFor(table: string): TableAccumulator { +function accumulatorFor(tableRef: KtxTableRef): TableAccumulator { return { - table, + tableRef, + table: displayTableRef(tableRef), executions: 0, distinctUsers: 0, errorRateNumerator: 0, @@ -212,8 +193,8 @@ function addTemplate(acc: TableAccumulator, parsed: ParsedTemplate): void { } } const joinColumns = parsed.columnsByClause.join ?? []; - for (const otherTable of parsed.tablesTouched.filter((table) => table !== acc.table)) { - recordJoin(acc, otherTable, joinColumns, executions); + for (const otherTable of parsed.tablesTouched.filter((table) => tableRefKey(table) !== tableRefKey(acc.tableRef))) { + recordJoin(acc, displayTableRef(otherTable), joinColumns, executions); } acc.topTemplates.push(parsed.template); } @@ -250,6 +231,7 @@ function toStagedTable(acc: TableAccumulator, now: Date): StagedTableInput { return { table: acc.table, + tableRef: acc.tableRef, stats: { executionsBucket: bucketExecutions(acc.executions), distinctUsersBucket: bucketDistinctUsers(acc.distinctUsers), @@ -269,7 +251,7 @@ function toPatternsInput(parsedTemplates: ParsedTemplate[]): StagedPatternsInput .map(({ template, tablesTouched }) => ({ id: template.templateId, canonicalSql: template.canonicalSql, - tablesTouched: [...tablesTouched].sort(), + tablesTouched: [...tablesTouched].sort((left, right) => tableRefKey(left).localeCompare(tableRefKey(right))), executionsBucket: bucketExecutions(template.stats.executions), distinctUsersBucket: bucketDistinctUsers(template.stats.distinctUsers), dialect: template.dialect, @@ -280,7 +262,6 @@ function toPatternsInput(parsedTemplates: ParsedTemplate[]): StagedPatternsInput export async function stageHistoricSqlAggregatedSnapshot(input: StageHistoricSqlAggregatedSnapshotInput): Promise { const config = historicSqlUnifiedPullConfigSchema.parse(input.pullConfig); - const enabledTableFilter = buildEnabledTableFilter(config.enabledTables); const redactors = compileHistoricSqlRedactionPatterns(config.redactionPatterns); const now = input.now ?? new Date(); const windowStart = new Date(now.getTime() - historicSqlWindowDays(config) * 24 * 60 * 60 * 1000); @@ -296,11 +277,25 @@ export async function stageHistoricSqlAggregatedSnapshot(input: StageHistoricSql } } - const analysis = await input.sqlAnalysis.analyzeBatch( - snapshot.map((template) => ({ id: template.templateId, sql: template.canonicalSql })), - config.dialect, - ); - const warnings: string[] = []; + const analysisItems = snapshot.map((template) => ({ id: template.templateId, sql: template.canonicalSql })); + const analysisOptions = + config.modeledTableCatalog.length > 0 ? { catalog: { tables: config.modeledTableCatalog } } : undefined; + const warnings: string[] = [ + ...config.scopeFloorWarnings, + ...(shouldFailOpenQueryHistoryScope(config) ? ['query_history_scope_floor_disabled:empty_modeled_scope'] : []), + ]; + let scopeDisabledByQualificationFailure = false; + let analysis: Awaited>; + try { + analysis = await input.sqlAnalysis.analyzeBatch(analysisItems, config.dialect, analysisOptions); + } catch (error) { + if (!analysisOptions || config.enabledTables.length > 0 || isQueryHistoryScopeFloorDisabled(config)) { + throw error; + } + warnings.push('query_history_scope_floor_disabled:catalog_qualification_failed'); + scopeDisabledByQualificationFailure = true; + analysis = await input.sqlAnalysis.analyzeBatch(analysisItems, config.dialect, undefined); + } const parsedTemplates: ParsedTemplate[] = []; for (const template of snapshot) { const parsed = analysis.get(template.templateId); @@ -308,8 +303,12 @@ export async function stageHistoricSqlAggregatedSnapshot(input: StageHistoricSql warnings.push(`parse_failed:${template.templateId}`); continue; } - const tablesTouched = [...new Set(parsed.tablesTouched)].filter((table) => table.length > 0).sort(); - const includedTables = tablesTouched.filter((table) => isEnabledTable(table, enabledTableFilter)); + const tablesTouched = [...new Map(parsed.tablesTouched.map((ref) => [tableRefKey(ref), ref])).values()] + .filter((ref) => ref.name.length > 0) + .sort((left, right) => tableRefKey(left).localeCompare(tableRefKey(right))); + const includedTables = scopeDisabledByQualificationFailure + ? [...tablesTouched] + : includedQueryHistoryTableRefs(tablesTouched, config); if (includedTables.length === 0) { continue; } @@ -323,22 +322,23 @@ export async function stageHistoricSqlAggregatedSnapshot(input: StageHistoricSql }); } - const byTable = new Map(); + const byTable = new Map(); for (const parsed of parsedTemplates) { - for (const table of parsed.includedTables) { - const acc = byTable.get(table) ?? accumulatorFor(table); + for (const tableRef of parsed.includedTables) { + const key = tableRefKey(tableRef); + const acc = byTable.get(key) ?? accumulatorFor(tableRef); addTemplate(acc, parsed); - byTable.set(table, acc); + byTable.set(key, acc); } } await mkdir(input.stagedDir, { recursive: true }); - for (const [table, acc] of [...byTable.entries()].sort(([left], [right]) => left.localeCompare(right))) { - await writeJson(input.stagedDir, `tables/${table}.json`, toStagedTable(acc, now)); + for (const [, acc] of [...byTable.entries()].sort((left, right) => left[0].localeCompare(right[0]))) { + await writeJson(input.stagedDir, `tables/${acc.table}.json`, toStagedTable(acc, now)); } const patternsInput = toPatternsInput(parsedTemplates); const patternInputSplit = splitHistoricSqlPatternInputs(patternsInput); - const allWarnings = [...warnings, ...patternInputSplit.warnings]; + const allWarnings = [...new Set([...warnings, ...patternInputSplit.warnings])]; await writeJson(input.stagedDir, 'patterns-input.json', patternInputSplit.auditInput); for (const shard of patternInputSplit.shards) { await writeJson(input.stagedDir, shard.path, shard.input); diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/types.ts b/packages/cli/src/context/ingest/adapters/historic-sql/types.ts index 1d256b13..aca50c4e 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/types.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/types.ts @@ -8,9 +8,22 @@ export type HistoricSqlDialect = z.infer; const filterModeSchema = z.enum(['exclude', 'include', 'mark-only']); +const ktxTableRefSchema = z.object({ + catalog: z.string().nullable(), + db: z.string().nullable(), + name: z.string().min(1), +}).strict(); + +const ktxTableRefWithColumnsSchema = ktxTableRefSchema.extend({ + columns: z.array(z.string().min(1)).optional(), +}).strict(); + const historicSqlCommonPullConfigSchema = z.object({ minExecutions: z.number().int().nonnegative().default(5), - enabledTables: z.array(z.string().min(1)).default([]), + enabledTables: z.array(ktxTableRefSchema).default([]), + enabledSchemas: z.array(z.string().min(1)).default([]), + modeledTableCatalog: z.array(ktxTableRefWithColumnsSchema).default([]), + scopeFloorWarnings: z.array(z.string()).default([]), filters: z.object({ serviceAccounts: z.object({ patterns: z.array(z.string()).default([]), @@ -68,6 +81,7 @@ export type AggregatedTemplate = z.infer; export const stagedTableInputSchema = z.object({ table: z.string().min(1), + tableRef: ktxTableRefSchema, stats: z.object({ executionsBucket: z.string(), distinctUsersBucket: z.string(), @@ -93,7 +107,7 @@ export const stagedPatternsInputSchema = z.object({ templates: z.array(z.object({ id: z.string(), canonicalSql: z.string(), - tablesTouched: z.array(z.string()), + tablesTouched: z.array(ktxTableRefSchema), executionsBucket: z.string(), distinctUsersBucket: z.string(), dialect: historicSqlDialectSchema, diff --git a/packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.ts b/packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.ts index f71e332d..03e5953d 100644 --- a/packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.ts +++ b/packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.ts @@ -151,7 +151,7 @@ function optionalString(value: unknown): string | undefined { function normalizeDriver(driver: unknown): string { const normalized = String(driver ?? '').trim().toLowerCase(); - return normalized === 'postgresql' ? 'postgres' : normalized; + return normalized; } function requirePostgresConnection( diff --git a/packages/cli/src/context/ingest/adapters/live-database/stage.ts b/packages/cli/src/context/ingest/adapters/live-database/stage.ts index ba925986..5dd21afd 100644 --- a/packages/cli/src/context/ingest/adapters/live-database/stage.ts +++ b/packages/cli/src/context/ingest/adapters/live-database/stage.ts @@ -7,6 +7,8 @@ import type { KtxSchemaSnapshot, KtxSchemaTable, KtxTableRef } from '../../../sc export const LIVE_DATABASE_META_FILE = 'connection.json'; export const LIVE_DATABASE_FOREIGN_KEYS_FILE = 'foreign-keys.json'; +/** @internal */ +export const LIVE_DATABASE_WARNINGS_FILE = 'warnings.json'; const LIVE_DATABASE_TABLES_DIR = 'tables'; interface LiveDatabaseTableFile { @@ -89,6 +91,13 @@ function foreignKeyIndex(snapshot: KtxSchemaSnapshot): ForeignKeyIndexEntry[] { return entries; } +function warningArtifact(snapshot: KtxSchemaSnapshot): { warnings: KtxSchemaSnapshot['warnings'] } { + const redacted = redactKtxSensitiveMetadata({ warnings: snapshot.warnings ?? [] }); + return { + warnings: Array.isArray(redacted.warnings) ? (redacted.warnings as KtxSchemaSnapshot['warnings']) : [], + }; +} + export async function writeLiveDatabaseSnapshot(stagedDir: string, snapshot: KtxSchemaSnapshot): Promise { await mkdir(join(stagedDir, LIVE_DATABASE_TABLES_DIR), { recursive: true }); const sortedTables = [...snapshot.tables].sort((a, b) => tableSortKey(a).localeCompare(tableSortKey(b))); @@ -105,6 +114,7 @@ export async function writeLiveDatabaseSnapshot(stagedDir: string, snapshot: Ktx join(stagedDir, LIVE_DATABASE_FOREIGN_KEYS_FILE), stableJson({ foreignKeys: foreignKeyIndex(snapshot) }), ); + await writeFile(join(stagedDir, LIVE_DATABASE_WARNINGS_FILE), stableJson(warningArtifact(snapshot))); for (const table of sortedTables) { await writeFile(join(stagedDir, liveDatabaseTablePath(table)), stableJson(table)); } diff --git a/packages/cli/src/context/ingest/adapters/looker/mapping.ts b/packages/cli/src/context/ingest/adapters/looker/mapping.ts index cbd80e80..4da6344d 100644 --- a/packages/cli/src/context/ingest/adapters/looker/mapping.ts +++ b/packages/cli/src/context/ingest/adapters/looker/mapping.ts @@ -7,12 +7,9 @@ const LOOKER_DIALECT_TO_CONNECTION_TYPE = { bigquery_standard_sql: 'BIGQUERY', snowflake: 'SNOWFLAKE', postgres: 'POSTGRESQL', - postgresql: 'POSTGRESQL', mysql: 'MYSQL', sqlite: 'SQLITE', sqlserver: 'SQLSERVER', - mssql: 'SQLSERVER', - tsql: 'SQLSERVER', clickhouse: 'CLICKHOUSE', } as const; diff --git a/packages/cli/src/context/ingest/context-candidates/curator-pagination.service.ts b/packages/cli/src/context/ingest/context-candidates/curator-pagination.service.ts index 348544ca..7848fab7 100644 --- a/packages/cli/src/context/ingest/context-candidates/curator-pagination.service.ts +++ b/packages/cli/src/context/ingest/context-candidates/curator-pagination.service.ts @@ -40,6 +40,7 @@ export interface CuratorPaginationInput { buildToolSet: (passNumber: number) => KtxRuntimeToolSet; getReconciliationActions: () => MemoryAction[]; onStepFinish?: (info: { passNumber: number; stepIndex: number; stepBudget: number }) => void; + abortSignal?: AbortSignal; } interface CuratorPaginationResult extends ReconciliationOutcome { @@ -243,6 +244,7 @@ export class CuratorPaginationService implements CuratorPaginationPort { sourceKey: params.input.sourceKey, jobId: params.input.jobId, forceRun: params.forceRun, + abortSignal: params.input.abortSignal, onStepFinish: params.input.onStepFinish ? ({ stepIndex, stepBudget }) => params.input.onStepFinish?.({ passNumber: params.passNumber, stepIndex, stepBudget }) diff --git a/packages/cli/src/context/ingest/final-gate-repair.ts b/packages/cli/src/context/ingest/final-gate-repair.ts index 1c373aa6..f32178d8 100644 --- a/packages/cli/src/context/ingest/final-gate-repair.ts +++ b/packages/cli/src/context/ingest/final-gate-repair.ts @@ -21,6 +21,7 @@ export interface RepairFinalGateFailureInput { repairKind: FinalGateRepairKind; maxAttempts?: number; stepBudget?: number; + abortSignal?: AbortSignal; } const readRepairFileSchema = z.object({ @@ -200,6 +201,7 @@ export async function repairFinalGateFailure( jobId: input.trace.context.jobId, repairKind: input.repairKind, }, + abortSignal: input.abortSignal, }), ); diff --git a/packages/cli/src/context/ingest/historic-sql-probes.ts b/packages/cli/src/context/ingest/historic-sql-probes.ts new file mode 100644 index 00000000..07204f3a --- /dev/null +++ b/packages/cli/src/context/ingest/historic-sql-probes.ts @@ -0,0 +1,141 @@ +import type { KtxProjectConnectionConfig } from '../project/config.js'; +import { queryHistoryDialectForConnection } from './adapters/historic-sql/connection-dialect.js'; +import type { HistoricSqlDialect } from './adapters/historic-sql/types.js'; + +export interface HistoricSqlFixAdvice { + failHeadline: string; + remediation: string; +} + +export interface HistoricSqlSuccessDetail { + detail: string; + warnings: string[]; +} + +export interface HistoricSqlProbeInput { + projectDir: string; + connectionId: string; + connection: KtxProjectConnectionConfig; + env?: NodeJS.ProcessEnv; +} + +export interface HistoricSqlProbeRunner { + readonly dialect: HistoricSqlDialect; + readonly catalogName: string; + run(input: HistoricSqlProbeInput): Promise; + formatSuccessDetail(result: unknown): HistoricSqlSuccessDetail; + fixAdvice(error: unknown): HistoricSqlFixAdvice; +} + +/** @internal */ +export interface HistoricSqlProbeRunnerFactoryEntry { + readonly catalogName: string; + load(): Promise; +} + +export type HistoricSqlProbeOutcome = + | { + ok: true; + dialect: HistoricSqlDialect; + runner: HistoricSqlProbeRunner; + result: unknown; + } + | { + ok: false; + dialect: HistoricSqlDialect; + runner: HistoricSqlProbeRunner; + error: unknown; + }; + +export type HistoricSqlReadinessProbe = ( + input: HistoricSqlProbeInput, +) => Promise; + +export interface HistoricSqlProbeRegistryDeps { + factories?: Record; + cache?: Map; +} + +const defaultHistoricSqlProbeRunnerFactories: Record< + HistoricSqlDialect, + HistoricSqlProbeRunnerFactoryEntry +> = { + postgres: { + catalogName: 'pg_stat_statements', + load: async () => { + const { PostgresPgssProbeRunner } = await import( + './historic-sql-probes/postgres-runner.js' + ); + return new PostgresPgssProbeRunner(); + }, + }, + snowflake: { + catalogName: 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY', + load: async () => { + const { SnowflakeAccountUsageProbeRunner } = await import( + './historic-sql-probes/snowflake-runner.js' + ); + return new SnowflakeAccountUsageProbeRunner(); + }, + }, + bigquery: { + catalogName: 'INFORMATION_SCHEMA.JOBS_BY_PROJECT', + load: async () => { + const { BigQueryJobsByProjectProbeRunner } = await import( + './historic-sql-probes/bigquery-runner.js' + ); + return new BigQueryJobsByProjectProbeRunner(); + }, + }, +}; + +const DEFAULT_RUNNER_CACHE = new Map(); + +function registryDeps(input: HistoricSqlProbeRegistryDeps) { + return { + factories: input.factories ?? defaultHistoricSqlProbeRunnerFactories, + cache: input.cache ?? DEFAULT_RUNNER_CACHE, + }; +} + +export function historicSqlProbeCatalogName( + dialect: HistoricSqlDialect, + deps: HistoricSqlProbeRegistryDeps = {}, +): string { + return registryDeps(deps).factories[dialect].catalogName; +} + +async function loadHistoricSqlProbeRunner( + dialect: HistoricSqlDialect, + deps: HistoricSqlProbeRegistryDeps = {}, +): Promise { + const { factories, cache } = registryDeps(deps); + const cached = cache.get(dialect); + if (cached) { + return cached; + } + const runner = await factories[dialect].load(); + cache.set(dialect, runner); + return runner; +} + +export async function runHistoricSqlReadinessProbe( + input: HistoricSqlProbeInput, + deps: HistoricSqlProbeRegistryDeps = {}, +): Promise { + const dialect = queryHistoryDialectForConnection(input.connection); + if (!dialect) { + return null; + } + const runner = await loadHistoricSqlProbeRunner(dialect, deps); + try { + return { + ok: true, + dialect, + runner, + result: await runner.run(input), + }; + } catch (error) { + return { ok: false, dialect, runner, error }; + } +} diff --git a/packages/cli/src/context/ingest/historic-sql-probes/bigquery-runner.ts b/packages/cli/src/context/ingest/historic-sql-probes/bigquery-runner.ts new file mode 100644 index 00000000..09ad65d5 --- /dev/null +++ b/packages/cli/src/context/ingest/historic-sql-probes/bigquery-runner.ts @@ -0,0 +1,160 @@ +import { HistoricSqlGrantsMissingError } from '../adapters/historic-sql/errors.js'; +import { BigQueryHistoricSqlQueryHistoryReader } from '../adapters/historic-sql/bigquery-query-history-reader.js'; +import { + type HistoricSqlFixAdvice, + type HistoricSqlProbeInput, + type HistoricSqlProbeRunner, + type HistoricSqlSuccessDetail, +} from '../historic-sql-probes.js'; +import { resolveKtxConfigReference } from '../../core/config-reference.js'; +import { + isKtxBigQueryConnectionConfig, + KtxBigQueryScanConnector, + type KtxBigQueryConnectionConfig, +} from '../../../connectors/bigquery/connector.js'; + +interface GenericProbeResult { + warnings: string[]; + info?: string[]; +} + +interface ClientHandle { + client: unknown; + cleanup(): Promise; +} + +interface BigQueryJobsByProjectProbeRunnerOptions { + createReader?: (options: { projectId: string; region: string }) => { + probe(client: unknown): Promise; + }; + createClient?: ( + input: HistoricSqlProbeInput & { connection: KtxBigQueryConnectionConfig }, + ) => ClientHandle; + resolveReference?: (value: string | undefined, env: NodeJS.ProcessEnv) => string | undefined; +} + +function bigQueryProjectId( + connectionId: string, + connection: KtxBigQueryConnectionConfig, + env: NodeJS.ProcessEnv, + resolveReference: (value: string | undefined, env: NodeJS.ProcessEnv) => string | undefined, +): string { + const rawCredentials = + typeof connection.credentials_json === 'string' ? connection.credentials_json : ''; + const resolvedCredentials = resolveReference(rawCredentials, env); + if (!resolvedCredentials) { + throw new Error(`Query history BigQuery connection ${connectionId} requires credentials_json`); + } + const parsed = JSON.parse(resolvedCredentials) as { project_id?: unknown }; + if (typeof parsed.project_id !== 'string' || parsed.project_id.trim().length === 0) { + throw new Error( + `Query history BigQuery connection ${connectionId} requires credentials_json.project_id`, + ); + } + return parsed.project_id; +} + +function bigQueryRegion(connection: KtxBigQueryConnectionConfig): string { + return typeof connection.location === 'string' && connection.location.trim().length > 0 + ? connection.location.trim() + : 'us'; +} + +function infoSuffix(info: readonly string[] | undefined): string { + return info && info.length > 0 ? `; ${info.join('; ')}` : ''; +} + +export class BigQueryJobsByProjectProbeRunner implements HistoricSqlProbeRunner { + readonly dialect = 'bigquery' as const; + readonly catalogName = 'INFORMATION_SCHEMA.JOBS_BY_PROJECT'; + + private readonly createReader: (options: { projectId: string; region: string }) => { + probe(client: unknown): Promise; + }; + private readonly createClient: ( + input: HistoricSqlProbeInput & { connection: KtxBigQueryConnectionConfig }, + ) => ClientHandle; + private readonly resolveReference: ( + value: string | undefined, + env: NodeJS.ProcessEnv, + ) => string | undefined; + + constructor(options: BigQueryJobsByProjectProbeRunnerOptions = {}) { + this.createReader = + options.createReader ?? + ((readerOptions) => new BigQueryHistoricSqlQueryHistoryReader(readerOptions)); + this.createClient = + options.createClient ?? + ((input) => { + const connector = new KtxBigQueryScanConnector({ + connectionId: input.connectionId, + connection: input.connection, + env: input.env, + }); + return { + client: { + async executeQuery(sql: string) { + const result = await connector.executeReadOnly( + { connectionId: input.connectionId, sql }, + {} as never, + ); + return { + headers: result.headers, + rows: result.rows, + totalRows: result.totalRows, + }; + }, + }, + cleanup: () => connector.cleanup(), + }; + }); + this.resolveReference = options.resolveReference ?? resolveKtxConfigReference; + } + + async run(input: HistoricSqlProbeInput): Promise { + const inputDriver = input.connection.driver ?? 'unknown'; + if (!isKtxBigQueryConnectionConfig(input.connection)) { + throw new Error(`Native BigQuery connector cannot run driver "${inputDriver}"`); + } + const projectId = bigQueryProjectId( + input.connectionId, + input.connection, + input.env ?? process.env, + this.resolveReference, + ); + const reader = this.createReader({ + projectId, + region: bigQueryRegion(input.connection), + }); + const handle = this.createClient({ + ...input, + connection: input.connection, + }); + try { + return await reader.probe(handle.client); + } finally { + await handle.cleanup(); + } + } + + formatSuccessDetail(result: unknown): HistoricSqlSuccessDetail { + const probeResult = result as GenericProbeResult; + return { + detail: `${this.catalogName} ready${infoSuffix(probeResult.info)}`, + warnings: probeResult.warnings, + }; + } + + fixAdvice(error: unknown): HistoricSqlFixAdvice { + if (error instanceof HistoricSqlGrantsMissingError) { + return { + failHeadline: 'BigQuery principal cannot read INFORMATION_SCHEMA.JOBS_BY_PROJECT', + remediation: error.remediation, + }; + } + return { + failHeadline: `${this.catalogName} readiness check failed`, + remediation: error instanceof Error ? error.message : String(error), + }; + } +} diff --git a/packages/cli/src/context/ingest/historic-sql-probes/postgres-runner.ts b/packages/cli/src/context/ingest/historic-sql-probes/postgres-runner.ts new file mode 100644 index 00000000..7ebf9721 --- /dev/null +++ b/packages/cli/src/context/ingest/historic-sql-probes/postgres-runner.ts @@ -0,0 +1,111 @@ +import { + HistoricSqlExtensionMissingError, + HistoricSqlGrantsMissingError, + HistoricSqlVersionUnsupportedError, +} from '../adapters/historic-sql/errors.js'; +import { PostgresPgssReader } from '../adapters/historic-sql/postgres-pgss-reader.js'; +import type { PostgresPgssProbeResult } from '../adapters/historic-sql/types.js'; +import { + type HistoricSqlFixAdvice, + type HistoricSqlProbeInput, + type HistoricSqlProbeRunner, + type HistoricSqlSuccessDetail, +} from '../historic-sql-probes.js'; +import { + isKtxPostgresConnectionConfig, + type KtxPostgresConnectionConfig, +} from '../../../connectors/postgres/connector.js'; +import { KtxPostgresHistoricSqlQueryClient } from '../../../connectors/postgres/historic-sql-query-client.js'; + +interface ClientHandle { + client: unknown; + cleanup(): Promise; +} + +interface PostgresPgssProbeRunnerOptions { + reader?: { probe(client: unknown): Promise }; + createClient?: ( + input: HistoricSqlProbeInput & { connection: KtxPostgresConnectionConfig }, + ) => ClientHandle; +} + +function genericAdvice(error: unknown, catalogName: string): HistoricSqlFixAdvice { + return { + failHeadline: `${catalogName} readiness check failed`, + remediation: error instanceof Error ? error.message : String(error), + }; +} + +function infoSuffix(info: readonly string[] | undefined): string { + return info && info.length > 0 ? `; ${info.join('; ')}` : ''; +} + +export class PostgresPgssProbeRunner implements HistoricSqlProbeRunner { + readonly dialect = 'postgres' as const; + readonly catalogName = 'pg_stat_statements'; + + private readonly reader: { probe(client: unknown): Promise }; + private readonly createClient: ( + input: HistoricSqlProbeInput & { connection: KtxPostgresConnectionConfig }, + ) => ClientHandle; + + constructor(options: PostgresPgssProbeRunnerOptions = {}) { + this.reader = options.reader ?? new PostgresPgssReader(); + this.createClient = + options.createClient ?? + ((input) => { + const client = new KtxPostgresHistoricSqlQueryClient({ + connectionId: input.connectionId, + connection: input.connection, + env: input.env, + }); + return { client, cleanup: () => client.cleanup() }; + }); + } + + async run(input: HistoricSqlProbeInput): Promise { + const inputDriver = input.connection.driver ?? 'unknown'; + if (!isKtxPostgresConnectionConfig(input.connection)) { + throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`); + } + const handle = this.createClient({ + ...input, + connection: input.connection, + }); + try { + return await this.reader.probe(handle.client); + } finally { + await handle.cleanup(); + } + } + + formatSuccessDetail(result: unknown): HistoricSqlSuccessDetail { + const pgssResult = result as PostgresPgssProbeResult; + return { + detail: `pg_stat_statements ready (${pgssResult.pgServerVersion})${infoSuffix(pgssResult.info)}`, + warnings: pgssResult.warnings, + }; + } + + fixAdvice(error: unknown): HistoricSqlFixAdvice { + if (error instanceof HistoricSqlExtensionMissingError) { + return { + failHeadline: 'pg_stat_statements extension is missing', + remediation: error.remediation, + }; + } + if (error instanceof HistoricSqlGrantsMissingError) { + return { + failHeadline: 'Postgres connection role lacks pg_read_all_stats', + remediation: error.remediation, + }; + } + if (error instanceof HistoricSqlVersionUnsupportedError) { + return { + failHeadline: 'Postgres version too old', + remediation: 'Use PostgreSQL 14 or newer, or disable query history for this connection', + }; + } + return genericAdvice(error, this.catalogName); + } +} diff --git a/packages/cli/src/context/ingest/historic-sql-probes/snowflake-runner.ts b/packages/cli/src/context/ingest/historic-sql-probes/snowflake-runner.ts new file mode 100644 index 00000000..415b46d6 --- /dev/null +++ b/packages/cli/src/context/ingest/historic-sql-probes/snowflake-runner.ts @@ -0,0 +1,96 @@ +import { HistoricSqlGrantsMissingError } from '../adapters/historic-sql/errors.js'; +import { SnowflakeHistoricSqlQueryHistoryReader } from '../adapters/historic-sql/snowflake-query-history-reader.js'; +import { + type HistoricSqlFixAdvice, + type HistoricSqlProbeInput, + type HistoricSqlProbeRunner, + type HistoricSqlSuccessDetail, +} from '../historic-sql-probes.js'; +import { + isKtxSnowflakeConnectionConfig, + type KtxSnowflakeConnectionConfig, +} from '../../../connectors/snowflake/connector.js'; +import { KtxSnowflakeHistoricSqlQueryClient } from '../../../connectors/snowflake/historic-sql-query-client.js'; + +interface GenericProbeResult { + warnings: string[]; + info?: string[]; +} + +interface ClientHandle { + client: unknown; + cleanup(): Promise; +} + +interface SnowflakeAccountUsageProbeRunnerOptions { + reader?: { probe(client: unknown): Promise }; + createClient?: ( + input: HistoricSqlProbeInput & { connection: KtxSnowflakeConnectionConfig }, + ) => ClientHandle; +} + +function infoSuffix(info: readonly string[] | undefined): string { + return info && info.length > 0 ? `; ${info.join('; ')}` : ''; +} + +export class SnowflakeAccountUsageProbeRunner implements HistoricSqlProbeRunner { + readonly dialect = 'snowflake' as const; + readonly catalogName = 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY'; + + private readonly reader: { probe(client: unknown): Promise }; + private readonly createClient: ( + input: HistoricSqlProbeInput & { connection: KtxSnowflakeConnectionConfig }, + ) => ClientHandle; + + constructor(options: SnowflakeAccountUsageProbeRunnerOptions = {}) { + this.reader = options.reader ?? new SnowflakeHistoricSqlQueryHistoryReader(); + this.createClient = + options.createClient ?? + ((input) => { + const client = new KtxSnowflakeHistoricSqlQueryClient({ + connectionId: input.connectionId, + connection: input.connection, + projectDir: input.projectDir, + env: input.env, + }); + return { client, cleanup: () => client.cleanup() }; + }); + } + + async run(input: HistoricSqlProbeInput): Promise { + const inputDriver = input.connection.driver ?? 'unknown'; + if (!isKtxSnowflakeConnectionConfig(input.connection)) { + throw new Error(`Native Snowflake connector cannot run driver "${inputDriver}"`); + } + const handle = this.createClient({ + ...input, + connection: input.connection, + }); + try { + return await this.reader.probe(handle.client); + } finally { + await handle.cleanup(); + } + } + + formatSuccessDetail(result: unknown): HistoricSqlSuccessDetail { + const probeResult = result as GenericProbeResult; + return { + detail: `${this.catalogName} ready${infoSuffix(probeResult.info)}`, + warnings: probeResult.warnings, + }; + } + + fixAdvice(error: unknown): HistoricSqlFixAdvice { + if (error instanceof HistoricSqlGrantsMissingError) { + return { + failHeadline: 'Snowflake role cannot read SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY', + remediation: error.remediation, + }; + } + return { + failHeadline: `${this.catalogName} readiness check failed`, + remediation: error instanceof Error ? error.message : String(error), + }; + } +} diff --git a/packages/cli/src/context/ingest/ingest-bundle.runner.ts b/packages/cli/src/context/ingest/ingest-bundle.runner.ts index 510e88d0..a242d58a 100644 --- a/packages/cli/src/context/ingest/ingest-bundle.runner.ts +++ b/packages/cli/src/context/ingest/ingest-bundle.runner.ts @@ -3,6 +3,7 @@ import { dirname, join } from 'node:path'; import pLimit from 'p-limit'; import { z } from 'zod'; import { type KtxLogger, noopLogger } from '../../context/core/config.js'; +import type { RateLimitWaitState } from '../../context/llm/rate-limit-governor.js'; import { createRuntimeToolDescriptorFromAiTool } from '../../context/llm/runtime-tools.js'; import type { KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; import type { CaptureSession, MemoryAction } from '../../context/memory/types.js'; @@ -25,6 +26,7 @@ import { deriveFinalizationWikiPageKeys, } from './finalization-scope.js'; import { FileIngestTraceWriter, ingestTracePathForJob, type IngestTraceWriter, traceTimed } from './ingest-trace.js'; +import { formatIngestProfile, formatIngestProfileJson, readIngestProfile, resolveIngestProfileMode } from './ingest-profile.js'; import { integrateWorkUnitPatch } from './isolated-diff/patch-integrator.js'; import { resolveTextualConflict } from './isolated-diff/textual-conflict-resolver.js'; import { runIsolatedWorkUnit } from './isolated-diff/work-unit-executor.js'; @@ -69,7 +71,7 @@ import { createEvictionListTool } from './tools/eviction-list.tool.js'; import { createReadRawSpanTool } from './tools/read-raw-span.tool.js'; import { createStageDiffTool } from './tools/stage-diff.tool.js'; import { createStageListTool } from './tools/stage-list.tool.js'; -import { type ToolCallLogEntry, wrapToolsWithLogger } from './tools/tool-call-logger.js'; +import { flushToolCallLogs, type ToolCallLogEntry, wrapToolsWithLogger } from './tools/tool-call-logger.js'; import { createMutableToolTranscriptSummary, recordToolTranscriptEntry, @@ -218,6 +220,10 @@ export class IngestBundleRunner { } async run(job: IngestBundleJob, ctx?: IngestJobContext): Promise { + const unsubscribeRateLimitGovernor = this.subscribeRateLimitGovernor({ + trace: this.createTrace(job), + memoryFlow: ctx?.memoryFlow, + }); const key = job.connectionId; const previous = this.chainByConnection.get(key); if (previous) { @@ -239,6 +245,103 @@ export class IngestBundleRunner { } catch (error) { ctx?.memoryFlow?.finish('error', [sanitizeMemoryFlowError(error)]); throw error; + } finally { + unsubscribeRateLimitGovernor(); + await this.maybeEmitIngestProfile(job.jobId); + } + } + + private formatRateLimitWait( + state: Extract, + ): string { + const seconds = Math.ceil(state.remainingMs / 1_000); + const minutes = Math.floor(seconds / 60); + const remainder = seconds % 60; + const duration = minutes > 0 ? `${minutes}m${String(remainder).padStart(2, '0')}s` : `${seconds}s`; + const type = state.rateLimitType ? ` ${state.rateLimitType}` : ''; + return `Rate-limited (${state.provider}${type}); resuming in ${duration}; Ctrl+C to stop`; + } + + private subscribeRateLimitGovernor(input: { + trace: IngestTraceWriter; + memoryFlow?: MemoryFlowEventSink; + }): () => void { + const governor = this.deps.settings.rateLimitGovernor; + if (!governor) { + return () => undefined; + } + return governor.subscribe((state: RateLimitWaitState) => { + if (state.kind === 'rate_limit_observed') { + void input.trace.event('info', 'rate_limit', 'rate_limit_observed', { ...state }); + return; + } + if (state.kind === 'concurrency_adjusted') { + void input.trace.event('info', 'rate_limit', 'concurrency_adjusted', { ...state }); + return; + } + void input.trace.event('info', 'rate_limit', state.kind, { ...state }); + if (state.kind === 'wait_tick' || state.kind === 'wait_started') { + input.memoryFlow?.emit({ + type: 'rate_limit_wait', + provider: state.provider, + ...(state.rateLimitType ? { rateLimitType: state.rateLimitType } : {}), + resumeAtMs: state.resumeAtMs, + remainingMs: state.remainingMs, + }); + input.memoryFlow?.emit({ + type: 'stage_progress', + stage: 'integration', + percent: 50, + message: this.formatRateLimitWait(state), + transient: true, + }); + } + }); + } + + private async withRateLimitWorkSlot(abortSignal: AbortSignal | undefined, fn: () => Promise): Promise { + const governor = this.deps.settings.rateLimitGovernor; + if (!governor) { + return fn(); + } + const release = await governor.acquireWorkSlot(abortSignal); + try { + return await fn(); + } finally { + release(); + } + } + + /** + * When profiling is enabled — via the `KTX_PROFILE_INGEST` env var or the + * `ingest.profile` config setting — read the job's trace + tool transcripts + * and print a rolled-up timing breakdown to stderr. `json` emits the raw + * structured profile for coding agents; `table` emits a human summary. + * Best-effort: profiling never affects the run outcome. + */ + private async maybeEmitIngestProfile(jobId: string): Promise { + const mode = resolveIngestProfileMode(this.deps.settings.profileIngest); + if (mode === 'off') { + return; + } + try { + // Tool transcripts are appended fire-and-forget; flush them so per-work-unit + // toolMs (and the derived model-vs-tool split) is complete before we read. + await flushToolCallLogs(); + const storage = this.deps.storage as typeof this.deps.storage & { + resolveTracePath?: (jobId: string) => string; + }; + const profile = await readIngestProfile(jobId, { + tracePath: storage.resolveTracePath?.(jobId) ?? ingestTracePathForJob(this.deps.storage.homeDir, jobId), + transcriptDir: this.deps.storage.resolveTranscriptDir(jobId), + }); + process.stderr.write(`\n${mode === 'json' ? formatIngestProfileJson(profile) : formatIngestProfile(profile)}`); + } catch (error) { + this.logger.warn( + `[ingest-bundle] ingest profile unavailable for job=${jobId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); } } @@ -841,6 +944,7 @@ export class IngestBundleRunner { includeContextEvidenceTools: boolean; currentTableExists(tableRef: string): Promise; memoryFlow?: MemoryFlowEventSink; + abortSignal?: AbortSignal; wuSkillNames: string[]; onStepFinish?: (info: { stepIndex: number; stepBudget: number }) => void; }): Promise { @@ -993,6 +1097,7 @@ export class IngestBundleRunner { jobId: input.job.jobId, toolFailureCount: (unitKey) => input.transcriptSummaries.get(unitKey)?.fatalErrorCount ?? 0, onStepFinish: input.onStepFinish, + abortSignal: input.abortSignal, }, input.wu, ); @@ -1100,8 +1205,15 @@ export class IngestBundleRunner { const scopeDescriptor = adapter.describeScope ? await adapter.describeScope(stagedDir) : null; - const sessionWorktree = await this.deps.lockingService.withLock('config:repo', () => - this.deps.sessionWorktreeService.create(job.jobId, baseSha), + const sessionWorktree = await traceTimed( + trace, + 'worktree', + 'session_worktree_created', + { jobId: job.jobId }, + () => + this.deps.lockingService.withLock('config:repo', () => + this.deps.sessionWorktreeService.create(job.jobId, baseSha), + ), ); let cleanupOutcome: 'success' | 'crash' | 'conflict' = 'crash'; @@ -1272,26 +1384,34 @@ export class IngestBundleRunner { sourceContextReport = chunk.contextReport; parseArtifacts = chunk.parseArtifacts; reconcileNotes = chunk.reconcileNotes ?? []; + const pageTriage = this.deps.pageTriage; + const triageRunId = runRow.id; triageResult = - contextReport && adapter.triageSupported && this.deps.pageTriage - ? await this.deps.pageTriage.triageRun({ - stagedDir, - runId: runRow.id, - connectionId: job.connectionId, - sourceKey: job.sourceKey, - syncId, - jobId: job.jobId, - diffSet, - adapter, - }) + contextReport && adapter.triageSupported && pageTriage + ? await traceTimed(runTrace, 'triage', 'page_triage', { sourceKey: job.sourceKey }, () => + pageTriage.triageRun({ + stagedDir, + runId: triageRunId, + connectionId: job.connectionId, + sourceKey: job.sourceKey, + syncId, + jobId: job.jobId, + diffSet, + adapter, + }), + ) : null; workUnits = this.filterWorkUnitsForTriage(workUnits, triageResult); - if (adapter.clusterWorkUnits && workUnits.length > 0) { - workUnits = await adapter.clusterWorkUnits({ - workUnits, - stagedDir, - embedding: this.deps.embedding, - }); + const clusterWorkUnits = adapter.clusterWorkUnits; + if (clusterWorkUnits && workUnits.length > 0) { + const preClusterCount = workUnits.length; + workUnits = await traceTimed( + runTrace, + 'clustering', + 'cluster_work_units', + { workUnitCount: preClusterCount }, + () => clusterWorkUnits({ workUnits, stagedDir, embedding: this.deps.embedding }), + ); } await stage2?.updateProgress(1.0, `Planned ${workUnits.length} update${workUnits.length === 1 ? '' : 's'}`); } @@ -1326,7 +1446,13 @@ export class IngestBundleRunner { }); // Build shared per-job context. - const [wikiIndex, slIndex] = await Promise.all([this.buildWikiIndex(), this.buildSlIndex(slConnectionIds)]); + const [wikiIndex, slIndex] = await traceTimed( + runTrace, + 'index_build', + 'build_indexes', + { connectionCount: slConnectionIds.length }, + () => Promise.all([this.buildWikiIndex(), this.buildSlIndex(slConnectionIds)]), + ); const baseFraming = await this.deps.promptService.loadPrompt('memory_agent_bundle_ingest_work_unit'); const wuSkillNames = Array.from( @@ -1467,7 +1593,8 @@ export class IngestBundleRunner { try { await Promise.all( workUnits.map((wu, index) => - limitWorkUnit(async () => { + limitWorkUnit(() => + this.withRateLimitWorkSlot(ctx?.abortSignal, async () => { const outcome = await runIsolatedWorkUnit({ unitIndex: index, ingestionBaseSha, @@ -1475,6 +1602,7 @@ export class IngestBundleRunner { patchDir, trace: runTrace, workUnit: wu, + abortSignal: ctx?.abortSignal, afterSuccess: (child) => copyTransientIngestEvidence(child.workdir, sessionWorktree.workdir), run: async (child) => { const scopedWikiService = this.deps.wikiService.forWorktree(child.workdir); @@ -1508,6 +1636,7 @@ export class IngestBundleRunner { includeContextEvidenceTools: adapter.evidenceIndexing === 'documents' && !!contextReport, currentTableExists: (tableRef) => this.tableRefExistsInSemanticLayer(scopedSemanticLayerService, slConnectionIds, tableRef), + abortSignal: ctx?.abortSignal, memoryFlow, wuSkillNames, onStepFinish: ({ stepIndex, stepBudget }) => { @@ -1537,7 +1666,8 @@ export class IngestBundleRunner { completedWorkUnits / workUnits.length, `${completedWorkUnits} of ${workUnits.length} work units complete`, ); - }), + }), + ), ), ); } catch (error) { @@ -1636,6 +1766,7 @@ export class IngestBundleRunner { reason: context.reason, maxAttempts: 1, stepBudget: 12, + abortSignal: ctx?.abortSignal, }); emitStageProgress( 'integration', @@ -1657,6 +1788,7 @@ export class IngestBundleRunner { repairKind: 'patch_semantic_gate', maxAttempts: 1, stepBudget: 16, + abortSignal: ctx?.abortSignal, }); emitStageProgress( 'integration', @@ -1881,6 +2013,8 @@ export class IngestBundleRunner { let curatorWarnings: string[] = []; let reconcileOutcome: Awaited>; + const reconcileStartedAt = Date.now(); + const reconcileMode = contextReport && this.deps.curatorPagination ? 'curator' : 'single'; if (contextReport && this.deps.curatorPagination) { const curatorOutcome = await this.deps.curatorPagination.reconcile({ runId: runRow.id, @@ -1934,6 +2068,7 @@ export class IngestBundleRunner { ); } : undefined, + abortSignal: ctx?.abortSignal, }); curatorReport = curatorOutcome.report; curatorWarnings = curatorOutcome.warnings; @@ -1979,6 +2114,7 @@ export class IngestBundleRunner { sourceKey: job.sourceKey, jobId: job.jobId, force: !!overrideReport, + abortSignal: ctx?.abortSignal, onStepFinish: stage4 ? ({ stepIndex, stepBudget }) => { emitStageProgress('reconciliation', 85, `Reconciling results: step ${stepIndex}/${stepBudget}`, { @@ -1989,6 +2125,33 @@ export class IngestBundleRunner { : undefined, }); } + await runTrace.event( + 'debug', + 'reconciliation', + 'reconciliation_executed', + { + mode: reconcileMode, + skipped: reconcileOutcome.skipped, + ...(reconcileOutcome.stopReason ? { stopReason: reconcileOutcome.stopReason } : {}), + ...(reconcileOutcome.metrics + ? { + agentLoopMs: reconcileOutcome.metrics.totalMs, + stepCount: reconcileOutcome.metrics.stepCount, + ...(reconcileOutcome.metrics.usage.inputTokens !== undefined + ? { inputTokens: reconcileOutcome.metrics.usage.inputTokens } + : {}), + ...(reconcileOutcome.metrics.usage.outputTokens !== undefined + ? { outputTokens: reconcileOutcome.metrics.usage.outputTokens } + : {}), + ...(reconcileOutcome.metrics.usage.totalTokens !== undefined + ? { totalTokens: reconcileOutcome.metrics.usage.totalTokens } + : {}), + } + : {}), + }, + undefined, + Date.now() - reconcileStartedAt, + ); latestReconciliationSkipped = reconcileOutcome.skipped; const danglingReconcileWikiRefs = await findDanglingWikiRefsForActions({ @@ -2036,6 +2199,7 @@ export class IngestBundleRunner { activePhase = 'finalization'; if (adapter.finalize) { const stageFinalization = ctx?.startPhase(0.04); + const finalizationStartedAt = Date.now(); emitStageProgress('finalization', 87, 'Running deterministic finalization'); await stageFinalization?.updateProgress(0.0, 'Running deterministic finalization'); await runTrace.event('debug', 'finalization', 'finalization_started', { sourceKey: job.sourceKey }); @@ -2215,14 +2379,21 @@ export class IngestBundleRunner { latestFinalizationOutcome = finalizationOutcome; emitStageProgress('finalization', 88, 'Deterministic finalization complete'); await stageFinalization?.updateProgress(1.0, 'Deterministic finalization complete'); - await runTrace.event('debug', 'finalization', 'finalization_committed', { - sourceKey: job.sourceKey, - commitSha: finalizationSha, - touchedPaths: finalizationTouchedPaths, - touchedSources: finalizationTouchedSources, - changedWikiPageKeys: finalizationChangedWikiPageKeys, - warnings: result.warnings, - }); + await runTrace.event( + 'debug', + 'finalization', + 'finalization_committed', + { + sourceKey: job.sourceKey, + commitSha: finalizationSha, + touchedPaths: finalizationTouchedPaths, + touchedSources: finalizationTouchedSources, + changedWikiPageKeys: finalizationChangedWikiPageKeys, + warnings: result.warnings, + }, + undefined, + Date.now() - finalizationStartedAt, + ); } else { await runTrace.event('debug', 'finalization', 'finalization_skipped', { sourceKey: job.sourceKey }); } @@ -2376,6 +2547,7 @@ export class IngestBundleRunner { repairKind: 'final_artifact_gate', maxAttempts: 1, stepBudget: 16, + abortSignal: ctx?.abortSignal, }); isolatedDiffSummary.gateRepairAttempts += gateRepair.attempts; @@ -2504,6 +2676,7 @@ export class IngestBundleRunner { const stage6 = ctx?.startPhase(0.04); emitStageProgress('save', 91, 'Saving changes'); await stage6?.updateProgress(0.0, 'Saving changes'); + const squashStartedAt = Date.now(); try { await sessionWorktree.git.assertWorktreeClean(); } catch (error) { @@ -2527,10 +2700,17 @@ export class IngestBundleRunner { throw new Error(`squash merge conflict: ${mergeResult.conflictPaths.join(', ')}`); } const commitSha = mergeResult.touchedPaths.length === 0 ? null : mergeResult.squashSha; - await runTrace.event('debug', 'squash', 'squash_finished', { - commitSha, - touchedPaths: mergeResult.touchedPaths, - }); + await runTrace.event( + 'debug', + 'squash', + 'squash_finished', + { + commitSha, + touchedPaths: mergeResult.touchedPaths, + }, + undefined, + Date.now() - squashStartedAt, + ); const memoryFlowSavedActions = stageIndex.workUnits .flatMap((wu) => wu.actions) .concat(reconcileActions) @@ -2547,6 +2727,7 @@ export class IngestBundleRunner { // transaction. If this throws, the run fails and no partial index state // survives (thanks to the transactional upsert in applyDiffTransactional). if (commitSha) { + const indexSyncStartedAt = Date.now(); // Multi-file squash → omit path so the handler diffs the whole commit // (a comma-joined pathspec would match nothing and the job would no-op). const pathFilter = mergeResult.touchedPaths.length === 1 ? mergeResult.touchedPaths[0] : ''; @@ -2571,6 +2752,14 @@ export class IngestBundleRunner { ); } } + await runTrace.event( + 'debug', + 'index_sync', + 'post_squash_index_sync_finished', + { connectionCount: touchedConnections.length }, + undefined, + Date.now() - indexSyncStartedAt, + ); } const stage5 = ctx?.startPhase(0.04); diff --git a/packages/cli/src/context/ingest/ingest-profile.ts b/packages/cli/src/context/ingest/ingest-profile.ts new file mode 100644 index 00000000..435bc7a2 --- /dev/null +++ b/packages/cli/src/context/ingest/ingest-profile.ts @@ -0,0 +1,437 @@ +import { readdir, readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { z } from 'zod'; + +export interface IngestProfilePaths { + tracePath: string; + transcriptDir: string; +} + +/** + * Post-processor over the ingest trace (`/ingest-traces//trace.jsonl`) + * and per-work-unit tool transcripts. Turns the durations recorded during a run + * into a rolled-up "where did the time go" view. Gated for display by + * `KTX_PROFILE_INGEST`; the durations themselves are always written to the trace. + */ + +const traceEventSchema = z + .object({ + at: z.string().optional(), + phase: z.string(), + event: z.string(), + durationMs: z.number().optional(), + data: z.record(z.string(), z.unknown()).optional(), + }) + .loose(); + +/** @internal */ +export type ProfiledTraceEvent = z.infer; + +export interface IngestProfile { + jobId: string; + totalWallMs?: number; + phases: Array<{ + phase: string; + totalMs: number; + /** Number of timed (durationMs-bearing) events that contributed to this phase. */ + count: number; + }>; + workUnits: Array<{ + unitKey: string; + status?: string; + /** Wall-clock for the whole work-unit run (agent loop + validation + git). */ + totalMs?: number; + /** Pure `generateText` agent-loop time reported by the runtime. */ + agentLoopMs?: number; + /** Summed tool-execution time from the work-unit transcript. */ + toolMs?: number; + /** Derived model "thinking" time = agentLoopMs - toolMs (clamped at 0). */ + modelMs?: number; + /** Worktree create time. */ + createMs?: number; + /** Worktree teardown time. */ + cleanupMs?: number; + stepCount?: number; + totalTokens?: number; + }>; + workUnitCount: number; + failedWorkUnitCount: number; + /** + * Plain-language diagnosis plus the raw numbers behind it, so a reader (human + * or coding agent) gets the conclusion without re-deriving it from the tables. + */ + summary: { + /** One-sentence conclusion, e.g. which phase dominated and whether work was model- or tool-bound. */ + headline: string; + dominantPhase?: { phase: string; totalMs: number; pctOfWall?: number }; + /** Aggregate across all work units, in milliseconds. */ + workUnits?: { + count: number; + failed: number; + agentLoopMs: number; + modelMs: number; + toolMs: number; + /** Percent of agent-loop time spent in model generation vs tool execution. */ + modelPct?: number; + }; + }; +} + +type IngestWorkUnitTiming = IngestProfile['workUnits'][number]; + +function asNumber(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + +function asString(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} + +/** @internal */ +export function parseTraceEvents(traceText: string): ProfiledTraceEvent[] { + const events: ProfiledTraceEvent[] = []; + for (const line of traceText.split('\n')) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + let json: unknown; + try { + json = JSON.parse(trimmed); + } catch { + continue; + } + const parsed = traceEventSchema.safeParse(json); + if (parsed.success) { + events.push(parsed.data); + } + } + return events; +} + +/** @internal */ +export function aggregateIngestProfile(input: { + jobId: string; + events: ProfiledTraceEvent[]; + toolMsByUnit: Record; +}): IngestProfile { + const { jobId, events, toolMsByUnit } = input; + + const phaseTotals = new Map(); + const workUnits = new Map(); + + const wu = (unitKey: string): IngestWorkUnitTiming => { + let existing = workUnits.get(unitKey); + if (!existing) { + existing = { unitKey }; + workUnits.set(unitKey, existing); + } + return existing; + }; + + let minAt = Number.POSITIVE_INFINITY; + let maxAt = Number.NEGATIVE_INFINITY; + + for (const event of events) { + const at = event.at ? Date.parse(event.at) : Number.NaN; + if (!Number.isNaN(at)) { + minAt = Math.min(minAt, at); + maxAt = Math.max(maxAt, at); + } + + if (event.durationMs !== undefined) { + const bucket = phaseTotals.get(event.phase) ?? { totalMs: 0, count: 0 }; + bucket.totalMs += event.durationMs; + bucket.count += 1; + phaseTotals.set(event.phase, bucket); + } + + const data = event.data ?? {}; + const unitKey = asString(data.unitKey); + if (unitKey) { + const entry = wu(unitKey); + if (event.event === 'work_unit_executed') { + entry.totalMs = event.durationMs; + entry.agentLoopMs = asNumber(data.agentLoopMs); + entry.stepCount = asNumber(data.stepCount); + entry.totalTokens = asNumber(data.totalTokens); + entry.status = asString(data.status) ?? entry.status; + } else if (event.event === 'work_unit_child_created') { + entry.createMs = event.durationMs; + } else if (event.event === 'work_unit_child_cleanup') { + entry.cleanupMs = event.durationMs; + } else if (event.event === 'work_unit_failed_before_patch') { + entry.status = entry.status ?? 'failed'; + } + } + } + + for (const [unitKey, entry] of workUnits) { + const toolMs = toolMsByUnit[unitKey]; + if (toolMs !== undefined) { + entry.toolMs = toolMs; + if (entry.agentLoopMs !== undefined) { + entry.modelMs = Math.max(0, entry.agentLoopMs - toolMs); + } + } else if (entry.agentLoopMs !== undefined) { + entry.modelMs = entry.agentLoopMs; + } + } + + const phases = [...phaseTotals.entries()] + .map(([phase, { totalMs, count }]) => ({ phase, totalMs, count })) + .sort((a, b) => b.totalMs - a.totalMs); + + const workUnitList = [...workUnits.values()].sort((a, b) => (b.totalMs ?? 0) - (a.totalMs ?? 0)); + const totalWallMs = Number.isFinite(minAt) && Number.isFinite(maxAt) && maxAt >= minAt ? maxAt - minAt : undefined; + const failedWorkUnitCount = workUnitList.filter((entry) => entry.status === 'failed').length; + + return { + jobId, + ...(totalWallMs !== undefined ? { totalWallMs } : {}), + phases, + workUnits: workUnitList, + workUnitCount: workUnitList.length, + failedWorkUnitCount, + summary: buildSummary(phases, workUnitList, failedWorkUnitCount, totalWallMs), + }; +} + +function buildSummary( + phases: IngestProfile['phases'], + workUnits: IngestWorkUnitTiming[], + failed: number, + totalWallMs: number | undefined, +): IngestProfile['summary'] { + const dominant = phases[0]; + const dominantPhase = dominant + ? { + phase: dominant.phase, + totalMs: dominant.totalMs, + ...(totalWallMs && totalWallMs > 0 + ? { pctOfWall: Math.round((dominant.totalMs / totalWallMs) * 100) } + : {}), + } + : undefined; + + const agentLoopMs = workUnits.reduce((sum, wu) => sum + (wu.agentLoopMs ?? 0), 0); + const toolMs = workUnits.reduce((sum, wu) => sum + (wu.toolMs ?? 0), 0); + const modelMs = workUnits.reduce((sum, wu) => sum + (wu.modelMs ?? 0), 0); + const workUnitAggregate = + workUnits.length > 0 + ? { + count: workUnits.length, + failed, + agentLoopMs, + modelMs, + toolMs, + ...(agentLoopMs > 0 ? { modelPct: Math.round((modelMs / agentLoopMs) * 100) } : {}), + } + : undefined; + + const parts: string[] = []; + if (dominantPhase) { + const pct = dominantPhase.pctOfWall !== undefined ? `, ${dominantPhase.pctOfWall}% of wall time` : ''; + parts.push(`Slowest phase: ${dominantPhase.phase} (${formatMs(dominantPhase.totalMs)}${pct})`); + } + if (workUnitAggregate) { + const split = + workUnitAggregate.modelPct !== undefined + ? `, ~${workUnitAggregate.modelPct}% model generation vs ~${100 - workUnitAggregate.modelPct}% tools` + : ''; + parts.push( + `${workUnitAggregate.count} work unit${workUnitAggregate.count === 1 ? '' : 's'}${ + failed > 0 ? ` (${failed} failed)` : '' + }${split}`, + ); + } + const headline = parts.length > 0 ? parts.join('. ') + '.' : 'No timed phases recorded.'; + + return { + headline, + ...(dominantPhase ? { dominantPhase } : {}), + ...(workUnitAggregate ? { workUnits: workUnitAggregate } : {}), + }; +} + +/** Read the trace and tool transcripts for a job and aggregate them into a profile. */ +export async function readIngestProfile( + jobId: string, + paths: IngestProfilePaths, +): Promise { + const traceText = await readFile(paths.tracePath, 'utf-8'); + const events = parseTraceEvents(traceText); + const toolMsByUnit = await readToolMsByUnit(paths.transcriptDir); + return aggregateIngestProfile({ jobId, events, toolMsByUnit }); +} + +async function listTranscriptFiles(dir: string): Promise { + // Work-unit keys can contain slashes (e.g. "cards/marketing"), so the runner + // writes nested transcript files (".../cards/marketing.jsonl"). Walk + // recursively and bucket by the `wuKey` field inside each entry rather than + // by file name. + const entries = await readdir(dir, { withFileTypes: true }).catch(() => null); + if (!entries) { + return []; + } + const files: string[] = []; + for (const entry of entries) { + const full = join(dir, entry.name); + if (entry.isDirectory()) { + files.push(...(await listTranscriptFiles(full))); + } else if (entry.isFile() && entry.name.endsWith('.jsonl')) { + files.push(full); + } + } + return files; +} + +async function readToolMsByUnit(transcriptDir: string): Promise> { + const toolMs: Record = {}; + for (const file of await listTranscriptFiles(transcriptDir)) { + let text: string; + try { + text = await readFile(file, 'utf-8'); + } catch { + continue; + } + for (const line of text.split('\n')) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + try { + const entry = JSON.parse(trimmed) as { wuKey?: unknown; durationMs?: unknown }; + const wuKey = asString(entry.wuKey); + const ms = asNumber(entry.durationMs); + if (wuKey && ms !== undefined) { + toolMs[wuKey] = (toolMs[wuKey] ?? 0) + ms; + } + } catch { + // skip malformed line + } + } + } + return toolMs; +} + +function formatMs(ms: number | undefined): string { + if (ms === undefined) { + return '—'; + } + if (ms < 1000) { + return `${Math.round(ms)}ms`; + } + const seconds = ms / 1000; + if (seconds < 60) { + return `${seconds.toFixed(1)}s`; + } + const minutes = Math.floor(seconds / 60); + const rem = Math.round(seconds - minutes * 60); + return `${minutes}m ${String(rem).padStart(2, '0')}s`; +} + +function formatTokens(tokens: number | undefined): string { + if (tokens === undefined) { + return '—'; + } + if (tokens < 1000) { + return String(tokens); + } + return `${(tokens / 1000).toFixed(1)}k`; +} + +function pad(value: string, width: number): string { + return value.length >= width ? value : value + ' '.repeat(width - value.length); +} + +function padStart(value: string, width: number): string { + return value.length >= width ? value : ' '.repeat(width - value.length) + value; +} + +/** Render a human-readable profile table for stderr / the admin command. */ +export function formatIngestProfile(profile: IngestProfile, options: { topWorkUnits?: number } = {}): string { + const topWorkUnits = options.topWorkUnits ?? 10; + const lines: string[] = []; + lines.push(`ktx ingest profile — job ${profile.jobId}`); + if (profile.totalWallMs !== undefined) { + lines.push(` total wall time: ${formatMs(profile.totalWallMs)}`); + } + lines.push(` ${profile.summary.headline}`); + + const wall = profile.totalWallMs; + lines.push(''); + lines.push(' Phase breakdown (by total duration):'); + if (profile.phases.length === 0) { + lines.push(' (no timed phases recorded)'); + } + for (const phase of profile.phases) { + const pct = wall && wall > 0 ? `(${((phase.totalMs / wall) * 100).toFixed(1)}%)` : ''; + lines.push( + ` ${pad(phase.phase, 22)}${padStart(formatMs(phase.totalMs), 9)} ${padStart(pct, 8)} ${padStart( + String(phase.count), + 4, + )} event${phase.count === 1 ? '' : 's'}`, + ); + } + + if (profile.workUnits.length > 0) { + lines.push(''); + lines.push(` Work units (top ${Math.min(topWorkUnits, profile.workUnits.length)} slowest):`); + lines.push( + ` ${pad('unitKey', 30)}${padStart('total', 9)}${padStart('model', 9)}${padStart('tool', 9)}${padStart( + 'steps', + 8, + )}${padStart('tokens', 9)} status`, + ); + for (const entry of profile.workUnits.slice(0, topWorkUnits)) { + const steps = entry.stepCount !== undefined ? String(entry.stepCount) : '—'; + lines.push( + ` ${pad(entry.unitKey.slice(0, 30), 30)}${padStart(formatMs(entry.totalMs), 9)}${padStart( + formatMs(entry.modelMs), + 9, + )}${padStart(formatMs(entry.toolMs), 9)}${padStart(steps, 8)}${padStart( + formatTokens(entry.totalTokens), + 9, + )} ${entry.status ?? '—'}`, + ); + } + lines.push( + ` (${profile.workUnitCount} work unit${profile.workUnitCount === 1 ? '' : 's'} total; ${ + profile.failedWorkUnitCount + } failed)`, + ); + } + + return `${lines.join('\n')}\n`; +} + +/** + * Machine-readable rendering for coding agents: the full structured profile + * (raw milliseconds and token counts, stable keys) as a single JSON object + * under a stable marker line so it is easy to locate and parse in stderr. + */ +export function formatIngestProfileJson(profile: IngestProfile): string { + return `ktx ingest profile (json)\n${JSON.stringify(profile, null, 2)}\n`; +} + +export type IngestProfileMode = 'off' | 'table' | 'json'; + +/** + * Resolve how (and whether) to emit the ingest profile, from the + * `ingest.profile` config value and the `KTX_PROFILE_INGEST` env var. Either + * source may request `json` (raw, agent-friendly) or a human `table`; `json` + * wins if either asks for it. + */ +export function resolveIngestProfileMode( + configValue: boolean | 'json' | undefined, + env: NodeJS.ProcessEnv = process.env, +): IngestProfileMode { + const envValue = env.KTX_PROFILE_INGEST; + if (configValue === 'json' || envValue === 'json') { + return 'json'; + } + const wantsTable = + configValue === true || envValue === '1' || envValue === 'true' || envValue === 'table'; + return wantsTable ? 'table' : 'off'; +} diff --git a/packages/cli/src/context/ingest/isolated-diff/patch-integrator.ts b/packages/cli/src/context/ingest/isolated-diff/patch-integrator.ts index 869c019e..1e2f0cee 100644 --- a/packages/cli/src/context/ingest/isolated-diff/patch-integrator.ts +++ b/packages/cli/src/context/ingest/isolated-diff/patch-integrator.ts @@ -155,18 +155,103 @@ export async function integrateWorkUnitPatch(input: IntegrateWorkUnitPatchInput) }, ); } catch (semanticError) { - if (preApplyHead) { - await input.integrationGit.resetHardTo(preApplyHead); - } + const reason = errorMessage(semanticError); await input.trace.event('error', 'integration', 'patch_semantic_conflict_after_textual_resolution', { unitKey: input.unitKey, patchPath: input.patchPath, touchedPaths: textualResolution.changedPaths, - reason: errorMessage(semanticError), + reason, }); + + // A textual conflict and a semantic-gate failure can co-occur: the resolver + // reconciles the text but can leave wiki sl_refs pointing at measures the + // merged source no longer defines. Recover via the same gate repair the + // clean-apply branch uses, instead of hard-failing the whole job. + if (input.repairGateFailure) { + const gateRepair = await input.repairGateFailure({ + unitKey: input.unitKey, + patchPath: input.patchPath, + touchedPaths: textualResolution.changedPaths, + reason, + }); + + if (gateRepair.status !== 'failed') { + // The resolver wrote its merge to the worktree (unstaged); the repair + // edited a subset on top. Commit the union so neither is dropped. + const resolvedAndRepairedPaths = [ + ...new Set([...textualResolution.changedPaths, ...gateRepair.changedPaths]), + ].sort(); + try { + await traceTimed( + input.trace, + 'integration', + 'semantic_gate_after_gate_repair', + { unitKey: input.unitKey, touchedPaths: gateRepair.changedPaths }, + async () => { + await input.validateAppliedTree(gateRepair.changedPaths); + }, + ); + + const commit = await input.integrationGit.commitFiles( + resolvedAndRepairedPaths, + `ingest: resolve WorkUnit ${input.unitKey} conflict`, + input.author.name, + input.author.email, + ); + if (commit.created) { + await input.trace.event('debug', 'integration', 'patch_accepted_after_textual_resolution', { + unitKey: input.unitKey, + commitSha: commit.commitHash, + touchedPaths: resolvedAndRepairedPaths, + attempts: textualResolution.attempts, + gateRepairAttempts: gateRepair.attempts, + }); + return { + status: 'accepted', + commitSha: commit.commitHash, + touchedPaths: resolvedAndRepairedPaths, + textualResolution, + gateRepair, + }; + } + } catch (repairValidationError) { + if (preApplyHead) { + await input.integrationGit.resetHardTo(preApplyHead); + } + await input.trace.event('error', 'integration', 'patch_semantic_conflict_after_textual_resolution', { + unitKey: input.unitKey, + patchPath: input.patchPath, + touchedPaths: gateRepair.changedPaths, + reason: errorMessage(repairValidationError), + }); + return { + status: 'semantic_conflict', + reason: errorMessage(repairValidationError), + touchedPaths: gateRepair.changedPaths, + textualResolution, + gateRepair, + }; + } + } + + if (preApplyHead) { + await input.integrationGit.resetHardTo(preApplyHead); + } + return { + status: 'semantic_conflict', + reason: gateRepair.status === 'failed' ? gateRepair.reason : reason, + touchedPaths: textualResolution.changedPaths, + textualResolution, + gateRepair, + }; + } + + if (preApplyHead) { + await input.integrationGit.resetHardTo(preApplyHead); + } return { status: 'semantic_conflict', - reason: errorMessage(semanticError), + reason, touchedPaths: textualResolution.changedPaths, textualResolution, }; diff --git a/packages/cli/src/context/ingest/isolated-diff/textual-conflict-resolver.ts b/packages/cli/src/context/ingest/isolated-diff/textual-conflict-resolver.ts index 5ae551d1..c4a00448 100644 --- a/packages/cli/src/context/ingest/isolated-diff/textual-conflict-resolver.ts +++ b/packages/cli/src/context/ingest/isolated-diff/textual-conflict-resolver.ts @@ -19,6 +19,7 @@ export interface ResolveTextualConflictInput { reason: string; maxAttempts?: number; stepBudget?: number; + abortSignal?: AbortSignal; } const readIntegrationFileSchema = z.object({ @@ -208,6 +209,7 @@ export async function resolveTextualConflict( jobId: input.trace.context.jobId, unitKey: input.unitKey, }, + abortSignal: input.abortSignal, }), ); diff --git a/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.ts b/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.ts index 81e6edfa..5ab52102 100644 --- a/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.ts +++ b/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.ts @@ -14,6 +14,7 @@ export interface RunIsolatedWorkUnitInput { patchDir: string; trace: IngestTraceWriter; workUnit: WorkUnit; + abortSignal?: AbortSignal; run(child: IngestSessionWorktree): Promise; afterSuccess?(child: IngestSessionWorktree): Promise; } @@ -26,16 +27,52 @@ function patchFileName(unitIndex: number, unitKey: string): string { export async function runIsolatedWorkUnit(input: RunIsolatedWorkUnitInput): Promise { const sessionKey = `${input.trace.context.jobId}-${input.workUnit.unitKey}`; let cleanupOutcome: SessionOutcome = 'crash'; + const createStartedAt = Date.now(); const child = await input.sessionWorktreeService.create(sessionKey, input.ingestionBaseSha); - await input.trace.event('debug', 'work_unit', 'work_unit_child_created', { - unitKey: input.workUnit.unitKey, - unitIndex: input.unitIndex, - worktreePath: child.workdir, - baseSha: input.ingestionBaseSha, - }); + await input.trace.event( + 'debug', + 'work_unit', + 'work_unit_child_created', + { + unitKey: input.workUnit.unitKey, + unitIndex: input.unitIndex, + worktreePath: child.workdir, + baseSha: input.ingestionBaseSha, + }, + undefined, + Date.now() - createStartedAt, + ); try { + const runStartedAt = Date.now(); const outcome = await input.run(child); + await input.trace.event( + 'debug', + 'work_unit', + 'work_unit_executed', + { + unitKey: input.workUnit.unitKey, + unitIndex: input.unitIndex, + status: outcome.status, + ...(outcome.metrics + ? { + agentLoopMs: outcome.metrics.totalMs, + stepCount: outcome.metrics.stepCount, + ...(outcome.metrics.usage.inputTokens !== undefined + ? { inputTokens: outcome.metrics.usage.inputTokens } + : {}), + ...(outcome.metrics.usage.outputTokens !== undefined + ? { outputTokens: outcome.metrics.usage.outputTokens } + : {}), + ...(outcome.metrics.usage.totalTokens !== undefined + ? { totalTokens: outcome.metrics.usage.totalTokens } + : {}), + } + : {}), + }, + undefined, + Date.now() - runStartedAt, + ); if (outcome.status !== 'success') { cleanupOutcome = 'success'; await input.trace.event('error', 'work_unit', 'work_unit_failed_before_patch', { @@ -75,11 +112,19 @@ export async function runIsolatedWorkUnit(input: RunIsolatedWorkUnitInput): Prom cleanupOutcome = 'success'; throw error; } finally { + const cleanupStartedAt = Date.now(); await input.sessionWorktreeService.cleanup(child, cleanupOutcome); - await input.trace.event('trace', 'work_unit', 'work_unit_child_cleanup', { - unitKey: input.workUnit.unitKey, - outcome: cleanupOutcome, - worktreePath: child.workdir, - }); + await input.trace.event( + 'trace', + 'work_unit', + 'work_unit_child_cleanup', + { + unitKey: input.workUnit.unitKey, + outcome: cleanupOutcome, + worktreePath: child.workdir, + }, + undefined, + Date.now() - cleanupStartedAt, + ); } } diff --git a/packages/cli/src/context/ingest/local-adapters.ts b/packages/cli/src/context/ingest/local-adapters.ts index e7c99146..3cd8a998 100644 --- a/packages/cli/src/context/ingest/local-adapters.ts +++ b/packages/cli/src/context/ingest/local-adapters.ts @@ -9,6 +9,7 @@ import { DbtSourceAdapter } from './adapters/dbt/dbt.adapter.js'; import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js'; import { HistoricSqlSourceAdapter } from './adapters/historic-sql/historic-sql.adapter.js'; import { PostgresPgssReader } from './adapters/historic-sql/postgres-pgss-reader.js'; +import { resolveQueryHistoryScopeFloor } from './adapters/historic-sql/scope-floor.js'; import { HISTORIC_SQL_SOURCE_KEY, historicSqlUnifiedPullConfigSchema, @@ -168,7 +169,6 @@ function isRecord(value: unknown): value is Record { const historicSqlDialectByDriver = new Map([ ['postgres', 'postgres'], - ['postgresql', 'postgres'], ['bigquery', 'bigquery'], ['snowflake', 'snowflake'], ]); @@ -180,12 +180,39 @@ function queryHistoryRecord(connection: unknown): Record | null return queryHistory; } -function queryHistoryPullConfig(connection: unknown): Record | null { +async function queryHistoryPullConfig( + project: KtxLocalProject, + connectionId: string, + connection: unknown, +): Promise | null> { const queryHistory = queryHistoryRecord(connection); if (queryHistory?.enabled !== true || !isRecord(connection)) return null; - const dialect = historicSqlDialectByDriver.get(String(connection.driver ?? '').toLowerCase()); + const driver = String(connection.driver ?? '').toLowerCase(); + const dialect = historicSqlDialectByDriver.get(driver); if (!dialect) return null; - return { ...queryHistory, dialect }; + const scopeFloor = await resolveQueryHistoryScopeFloor({ + projectDir: project.projectDir, + connectionId, + driver, + connection, + storedQueryHistory: queryHistory, + }); + const { + enabled: _enabled, + dialect: _dialect, + enabledTables: _enabledTables, + enabledSchemas: _enabledSchemas, + scopeFloorWarnings: _scopeFloorWarnings, + ...stored + } = queryHistory; + return { + ...stored, + dialect, + ...(scopeFloor.enabledTables.length > 0 ? { enabledTables: scopeFloor.enabledTables } : {}), + ...(scopeFloor.enabledSchemas.length > 0 ? { enabledSchemas: scopeFloor.enabledSchemas } : {}), + ...(scopeFloor.modeledTableCatalog.length > 0 ? { modeledTableCatalog: scopeFloor.modeledTableCatalog } : {}), + ...(scopeFloor.warnings.length > 0 ? { scopeFloorWarnings: scopeFloor.warnings } : {}), + }; } function stringField(value: unknown): string | null { @@ -246,7 +273,7 @@ export async function localPullConfigForAdapter( if (options.historicSqlPullConfigOverride) { return historicSqlUnifiedPullConfigSchema.parse(options.historicSqlPullConfigOverride); } - const queryHistory = queryHistoryPullConfig(connection); + const queryHistory = await queryHistoryPullConfig(project, connectionId, connection); if (!queryHistory) { throw new Error(`Connection "${connectionId}" does not have context.queryHistory.enabled: true`); } diff --git a/packages/cli/src/context/ingest/local-bundle-runtime.ts b/packages/cli/src/context/ingest/local-bundle-runtime.ts index 8b87d7be..e4c45b3f 100644 --- a/packages/cli/src/context/ingest/local-bundle-runtime.ts +++ b/packages/cli/src/context/ingest/local-bundle-runtime.ts @@ -12,6 +12,7 @@ import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic- import { createRuntimeToolDescriptorFromAiTool } from '../../context/llm/runtime-tools.js'; import { createLocalKtxLlmRuntimeFromConfig } from '../../context/llm/local-config.js'; import { KtxIngestEmbeddingPortAdapter } from '../../context/llm/embedding-port.js'; +import { createRateLimitGovernorConfig, RateLimitGovernor } from '../../context/llm/rate-limit-governor.js'; import { RuntimeAgentRunner, type AgentRunnerPort, type KtxLlmRuntimePort, type KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; import type { KtxEmbeddingProvider } from '../../llm/types.js'; import type { KtxLocalProject } from '../../context/project/project.js'; @@ -611,14 +612,15 @@ function nextLocalJobId(): string { function localIngestLlmProviderGuardMessage(projectDir: string): string { return [ - 'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, or claude-code, or an injected agentRunner.', - 'Configure a local Claude Code session or API-backed LLM, then rerun ingest:', + 'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, claude-code, or codex, or an injected agentRunner.', + 'Configure a local Claude Code/Codex session or API-backed LLM, then rerun ingest:', ` ktx setup --project-dir ${projectDir} --llm-backend claude-code --no-input`, + ` ktx setup --project-dir ${projectDir} --llm-backend codex --llm-model gpt-5.5 --no-input`, ` ktx setup --project-dir ${projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --llm-model claude-sonnet-4-6 --no-input`, ].join('\n'); } -function resolveAgentRunner(options: CreateLocalBundleIngestRuntimeOptions): { +function resolveAgentRunner(options: CreateLocalBundleIngestRuntimeOptions, rateLimitGovernor: RateLimitGovernor): { agentRunner: AgentRunnerPort; llmRuntime?: KtxLlmRuntimePort; } { @@ -627,6 +629,7 @@ function resolveAgentRunner(options: CreateLocalBundleIngestRuntimeOptions): { (options.createLlmRuntime ?? createLocalKtxLlmRuntimeFromConfig)(options.project.config.llm, { projectDir: options.project.projectDir, env: process.env, + rateLimitGovernor, }) ?? undefined; @@ -676,7 +679,13 @@ export function createLocalBundleIngestRuntime( const knowledgeIndex = new LocalKnowledgeIndex(options.project, embedding); const knowledgeEvents = new NoopKnowledgeEventPort(); const wikiService = new KnowledgeWikiService(rootFileStore, embedding, knowledgeIndex, options.project.git, logger); - const { agentRunner, llmRuntime } = resolveAgentRunner(options); + const rateLimitGovernor = new RateLimitGovernor( + createRateLimitGovernorConfig({ + ...options.project.config.ingest.rateLimit, + maxConcurrency: options.project.config.ingest.workUnits.maxConcurrency, + }), + ); + const { agentRunner, llmRuntime } = resolveAgentRunner(options, rateLimitGovernor); const promptService = new PromptService({ promptsDir, partials: [], logger }); const storage = new LocalIngestStorage(options.project); const registry = registerAdapters(options.adapters); @@ -716,6 +725,8 @@ export function createLocalBundleIngestRuntime( workUnitMaxConcurrency: options.project.config.ingest.workUnits.maxConcurrency, workUnitStepBudget: options.project.config.ingest.workUnits.stepBudget, workUnitFailureMode: options.project.config.ingest.workUnits.failureMode, + rateLimitGovernor, + profileIngest: options.project.config.ingest.profile, ingestTraceLevel: ingestTraceLevelFromEnv(), }, skillsRegistry: new SkillsRegistryService({ skillsDir, logger }), diff --git a/packages/cli/src/context/ingest/local-ingest.ts b/packages/cli/src/context/ingest/local-ingest.ts index 2832d9ff..1a219629 100644 --- a/packages/cli/src/context/ingest/local-ingest.ts +++ b/packages/cli/src/context/ingest/local-ingest.ts @@ -3,6 +3,7 @@ import { cp, mkdir, rm } from 'node:fs/promises'; import { isAbsolute, resolve } from 'node:path'; import type { KtxSqlQueryExecutorPort } from '../../context/connections/query-executor.js'; import type { KtxLogger } from '../../context/core/config.js'; +import { createAbortError, isAbortError } from '../../context/core/abort.js'; import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js'; import type { AgentRunnerPort, KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; import type { KtxLocalProject } from '../../context/project/project.js'; @@ -13,6 +14,7 @@ import { localPullConfigForAdapter, type DefaultLocalIngestAdaptersOptions } fro import { createLocalBundleIngestRuntime } from './local-bundle-runtime.js'; import type { MemoryFlowEventSink } from './memory-flow/types.js'; import { buildSyncId } from './raw-sources-paths.js'; +import { ingestReportOutcome } from './reports.js'; import type { IngestReportBody, IngestReportSnapshot } from './reports.js'; import { SqliteBundleIngestStore } from './sqlite-bundle-ingest-store.js'; import type { IngestBundleResult, IngestJobContext, IngestJobPhase, IngestTrigger, SourceAdapter } from './types.js'; @@ -35,6 +37,7 @@ export interface RunLocalIngestOptions { queryExecutor?: KtxSqlQueryExecutorPort; logger?: KtxLogger; embeddingProvider?: import('../../llm/types.js').KtxEmbeddingProvider | null; + abortSignal?: AbortSignal; } export interface LocalIngestResult { @@ -79,7 +82,7 @@ export interface LocalMetabaseFanoutProgress { metabaseDatabaseId: number; targetConnectionId: string; jobId: string; - status: 'done' | 'failed'; + status: 'done' | 'partial' | 'failed'; }): void; } @@ -122,10 +125,11 @@ function findAdapter(adapters: SourceAdapter[], source: string): SourceAdapter { return adapter; } -function localJobContext(jobId: string, memoryFlow?: MemoryFlowEventSink): IngestJobContext { +function localJobContext(jobId: string, memoryFlow?: MemoryFlowEventSink, abortSignal?: AbortSignal): IngestJobContext { return { jobId, ...(memoryFlow ? { memoryFlow } : {}), + ...(abortSignal ? { abortSignal } : {}), startPhase() { return new LocalIngestPhase(); }, @@ -157,6 +161,7 @@ async function runScheduledPullJob(options: { queryExecutor?: KtxSqlQueryExecutorPort; logger?: KtxLogger; embeddingProvider?: import('../../llm/types.js').KtxEmbeddingProvider | null; + abortSignal?: AbortSignal; }): Promise { const runtime = createLocalBundleIngestRuntime(options); const jobId = options.jobId ?? runtime.nextJobId(); @@ -168,7 +173,7 @@ async function runScheduledPullJob(options: { trigger: options.trigger ?? 'manual_resync', bundleRef: { kind: 'scheduled_pull', config: options.pullConfig }, }, - localJobContext(jobId, options.memoryFlow), + localJobContext(jobId, options.memoryFlow, options.abortSignal), ); const report = await runtime.store.findByJobId(jobId); if (!report) { @@ -211,6 +216,7 @@ export async function runLocalIngest(options: RunLocalIngestOptions): Promise child.report.body.failedWorkUnits.length === 0).length; - if (succeeded === children.length) { + const outcomes = children.map((child) => ingestReportOutcome(child.report)); + if (outcomes.every((outcome) => outcome === 'done')) { return 'all_succeeded'; } - if (succeeded === 0) { + if (outcomes.every((outcome) => outcome === 'error')) { return 'all_failed'; } return 'partial_failure'; @@ -336,7 +342,7 @@ export async function runLocalMetabaseIngest( options: RunLocalMetabaseIngestOptions, ): Promise { if ((options as RunLocalMetabaseIngestOptions & { sourceDir?: string }).sourceDir) { - throw new Error('source-dir uploads are not supported for the Metabase fan-out adapter'); + throw new Error('source-dir uploads are not supported for the Metabase fanout adapter'); } const metabaseConnectionId = safeSegment('metabase connection id', options.metabaseConnectionId); @@ -361,6 +367,9 @@ export async function runLocalMetabaseIngest( const children: LocalMetabaseFanoutChild[] = []; for (const childPlan of childPlans) { + if (options.abortSignal?.aborted) { + throw createAbortError(); + } const targetConnectionId = safeSegment('target connection id', childPlan.targetConnectionId); if (!options.project.config.connections[targetConnectionId]) { throw new Error(`Target connection "${targetConnectionId}" is not configured in ktx.yaml`); @@ -390,8 +399,12 @@ export async function runLocalMetabaseIngest( queryExecutor: options.queryExecutor, logger: options.logger, embeddingProvider: options.embeddingProvider, + abortSignal: options.abortSignal, }); } catch (error) { + if (isAbortError(error)) { + throw error; + } child = await recordLocalMetabaseChildFailure({ project: options.project, jobId: childJobId, @@ -401,12 +414,13 @@ export async function runLocalMetabaseIngest( error, }); } + const childOutcome = ingestReportOutcome(child.report); options.progress?.onMetabaseChildCompleted?.({ metabaseConnectionId, metabaseDatabaseId: childPlan.metabaseDatabaseId, targetConnectionId, jobId: child.report.jobId, - status: child.report.body.failedWorkUnits.length > 0 ? 'failed' : 'done', + status: childOutcome === 'error' ? 'failed' : childOutcome, }); children.push({ jobId: child.report.jobId, diff --git a/packages/cli/src/context/ingest/memory-flow/events.ts b/packages/cli/src/context/ingest/memory-flow/events.ts index 020ce5ae..92cebe0f 100644 --- a/packages/cli/src/context/ingest/memory-flow/events.ts +++ b/packages/cli/src/context/ingest/memory-flow/events.ts @@ -1,5 +1,6 @@ import type { MemoryAction } from '../../../context/memory/types.js'; import type { LocalIngestRunRecord } from '../local-stage-ingest.js'; +import { ingestReportOutcome } from '../reports.js'; import type { IngestReportSnapshot } from '../reports.js'; import type { MemoryFlowActionDetail, @@ -72,7 +73,7 @@ function fullModeMetadata(input: { } function reportStatus(report: IngestReportSnapshot): MemoryFlowReplayInput['status'] { - return report.body.failedWorkUnits.length > 0 ? 'error' : 'done'; + return ingestReportOutcome(report) === 'error' ? 'error' : 'done'; } function reportCreatedEvent(report: IngestReportSnapshot): MemoryFlowEvent { diff --git a/packages/cli/src/context/ingest/memory-flow/schema.ts b/packages/cli/src/context/ingest/memory-flow/schema.ts index 0268a53f..f448bbc8 100644 --- a/packages/cli/src/context/ingest/memory-flow/schema.ts +++ b/packages/cli/src/context/ingest/memory-flow/schema.ts @@ -70,6 +70,13 @@ const memoryFlowEventSchema = z.discriminatedUnion('type', [ message: z.string().min(1), transient: z.boolean().optional(), }), + eventSchema({ + type: z.literal('rate_limit_wait'), + provider: z.string(), + rateLimitType: z.string().optional(), + resumeAtMs: z.number().int().nonnegative(), + remainingMs: z.number().int().nonnegative(), + }), eventSchema({ type: z.literal('work_unit_started'), unitKey: z.string().min(1), diff --git a/packages/cli/src/context/ingest/memory-flow/types.ts b/packages/cli/src/context/ingest/memory-flow/types.ts index ab4619a6..72f1b6de 100644 --- a/packages/cli/src/context/ingest/memory-flow/types.ts +++ b/packages/cli/src/context/ingest/memory-flow/types.ts @@ -60,6 +60,13 @@ type MemoryFlowEventPayload = message: string; transient?: boolean; } + | { + type: 'rate_limit_wait'; + provider: string; + rateLimitType?: string; + resumeAtMs: number; + remainingMs: number; + } | { type: 'work_unit_started'; unitKey: string; diff --git a/packages/cli/src/context/ingest/ports.ts b/packages/cli/src/context/ingest/ports.ts index 76e9d765..7532919e 100644 --- a/packages/cli/src/context/ingest/ports.ts +++ b/packages/cli/src/context/ingest/ports.ts @@ -5,6 +5,7 @@ import type { KtxFileStorePort } from '../../context/core/file-store.js'; import type { KtxLogger } from '../../context/core/config.js'; import type { SessionOutcome } from '../../context/core/session-worktree.service.js'; import type { AgentRunnerPort, KtxLlmRuntimePort, KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; +import type { RateLimitGovernor } from '../llm/rate-limit-governor.js'; import type { MemoryAction, MemoryKnowledgeSlRefsPort } from '../../context/memory/types.js'; import type { PromptService } from '../../context/prompts/prompt.service.js'; import type { SkillsRegistryService } from '../../context/skills/skills-registry.service.js'; @@ -144,6 +145,9 @@ interface IngestSettingsPort { workUnitMaxConcurrency?: number; workUnitStepBudget?: number; workUnitFailureMode?: 'abort' | 'continue'; + rateLimitGovernor?: RateLimitGovernor; + /** Print a timing breakdown to stderr at the end of each run (config-driven; see also KTX_PROFILE_INGEST). `'json'` emits the raw structured profile. */ + profileIngest?: boolean | 'json'; ingestTraceLevel?: IngestTraceLevel; } @@ -321,6 +325,7 @@ export interface CuratorPaginationPort { buildToolSet: (passNumber: number) => KtxRuntimeToolSet; getReconciliationActions: () => MemoryAction[]; onStepFinish?: (info: { passNumber: number; stepIndex: number; stepBudget: number }) => void; + abortSignal?: AbortSignal; }): Promise; } diff --git a/packages/cli/src/context/ingest/reports.ts b/packages/cli/src/context/ingest/reports.ts index ea02a31a..09f92170 100644 --- a/packages/cli/src/context/ingest/reports.ts +++ b/packages/cli/src/context/ingest/reports.ts @@ -146,6 +146,20 @@ export function savedMemoryCountsForReport(report: IngestReportSnapshot): Ingest }; } +/** @internal */ +export type IngestReportOutcome = 'done' | 'partial' | 'error'; + +export function ingestReportOutcome(report: IngestReportSnapshot): IngestReportOutcome { + if (report.body.status === 'failed') { + return 'error'; + } + if (report.body.failedWorkUnits.length === 0) { + return 'done'; + } + const { wikiCount, slCount } = savedMemoryCountsForReport(report); + return wikiCount + slCount > 0 ? 'partial' : 'error'; +} + export function buildStageIndexFromReportBody(jobId: string, connectionId: string, body: IngestReportBody): StageIndex { return { jobId, diff --git a/packages/cli/src/context/ingest/stages/stage-3-work-units.ts b/packages/cli/src/context/ingest/stages/stage-3-work-units.ts index 96c0e65c..a7387c8a 100644 --- a/packages/cli/src/context/ingest/stages/stage-3-work-units.ts +++ b/packages/cli/src/context/ingest/stages/stage-3-work-units.ts @@ -1,5 +1,6 @@ import type { KtxModelRole } from '../../../llm/types.js'; -import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; +import { isAbortError } from '../../core/abort.js'; +import type { AgentRunnerPort, KtxRuntimeToolSet, RunLoopMetrics } from '../../../context/llm/runtime-port.js'; import type { CaptureSession, MemoryAction } from '../../../context/memory/types.js'; import { listTouchedSlSources, type TouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; import type { WorkUnit } from '../types.js'; @@ -28,6 +29,7 @@ export interface WorkUnitExecutionDeps { connectionId: string; jobId: string; onStepFinish?: (info: { stepIndex: number; stepBudget: number }) => void; + abortSignal?: AbortSignal; toolFailureCount?: (unitKey: string) => number; } @@ -44,6 +46,8 @@ export interface WorkUnitOutcome { patchPath?: string; patchTouchedPaths?: string[]; childWorktreePath?: string; + /** Timing and token metrics for the work-unit agent loop, used for ingest profiling. */ + metrics?: RunLoopMetrics; } export async function executeWorkUnit(deps: WorkUnitExecutionDeps, wu: WorkUnit): Promise { @@ -104,8 +108,12 @@ export async function executeWorkUnit(deps: WorkUnitExecutionDeps, wu: WorkUnit) jobId: deps.jobId, }, onStepFinish: deps.onStepFinish, + abortSignal: deps.abortSignal, }); } catch (error) { + if (isAbortError(error)) { + throw error; + } return failWithResetFromCurrentHead(error instanceof Error ? error.message : String(error)); } @@ -125,6 +133,7 @@ export async function executeWorkUnit(deps: WorkUnitExecutionDeps, wu: WorkUnit) touchedSlSources: [], slDisallowed: wu.slDisallowed, slDisallowedReason: wu.slDisallowedReason, + ...(runResult.metrics ? { metrics: runResult.metrics } : {}), }; }; @@ -162,5 +171,6 @@ export async function executeWorkUnit(deps: WorkUnitExecutionDeps, wu: WorkUnit) touchedSlSources: touched, slDisallowed: wu.slDisallowed, slDisallowedReason: wu.slDisallowedReason, + ...(runResult.metrics ? { metrics: runResult.metrics } : {}), }; } diff --git a/packages/cli/src/context/ingest/stages/stage-4-reconciliation.ts b/packages/cli/src/context/ingest/stages/stage-4-reconciliation.ts index 61fc1efe..c78e1b48 100644 --- a/packages/cli/src/context/ingest/stages/stage-4-reconciliation.ts +++ b/packages/cli/src/context/ingest/stages/stage-4-reconciliation.ts @@ -1,4 +1,4 @@ -import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; +import type { AgentRunnerPort, KtxRuntimeToolSet, RunLoopMetrics } from '../../../context/llm/runtime-port.js'; import type { KtxModelRole } from '../../../llm/types.js'; import type { EvictionUnit } from '../types.js'; import type { StageIndex } from './stage-index.types.js'; @@ -16,6 +16,7 @@ export interface ReconciliationContext { jobId: string; force?: boolean; onStepFinish?: (info: { stepIndex: number; stepBudget: number }) => void; + abortSignal?: AbortSignal; forceRun?: boolean; } @@ -23,6 +24,7 @@ export interface ReconciliationOutcome { skipped: boolean; stopReason?: 'budget' | 'natural' | 'error'; error?: Error; + metrics?: RunLoopMetrics; } export async function runReconciliationStage4(ctx: ReconciliationContext): Promise { @@ -39,6 +41,7 @@ export async function runReconciliationStage4(ctx: ReconciliationContext): Promi stepBudget: ctx.stepBudget, telemetryTags: { operationName: 'ingest-bundle-reconcile', source: ctx.sourceKey, jobId: ctx.jobId }, onStepFinish: ctx.onStepFinish, + abortSignal: ctx.abortSignal, }); - return { skipped: false, stopReason: run.stopReason, error: run.error }; + return { skipped: false, stopReason: run.stopReason, error: run.error, ...(run.metrics ? { metrics: run.metrics } : {}) }; } diff --git a/packages/cli/src/context/ingest/tools/tool-call-logger.ts b/packages/cli/src/context/ingest/tools/tool-call-logger.ts index 61d020c6..abdfdf25 100644 --- a/packages/cli/src/context/ingest/tools/tool-call-logger.ts +++ b/packages/cli/src/context/ingest/tools/tool-call-logger.ts @@ -81,8 +81,13 @@ export function wrapToolsWithLogger( return wrapped as T; } +// Fire-and-forget appends are intentional (the agent hot path must never block +// or fail on logging), but readers like the ingest profiler need to know when +// the writes have settled. Track in-flight appends so a consumer can flush. +const pendingWrites = new Set>(); + function appendEntry(path: string, entry: ToolCallLogEntry): void { - void (async () => { + const write = (async () => { try { await mkdir(dirname(path), { recursive: true }); await appendFile(path, `${safeStringify(entry)}\n`, 'utf-8'); @@ -90,6 +95,37 @@ function appendEntry(path: string, entry: ToolCallLogEntry): void { // best-effort } })(); + pendingWrites.add(write); + void write.finally(() => pendingWrites.delete(write)); +} + +/** + * Await all in-flight tool-call log writes (best-effort, bounded by `timeoutMs` + * so it can never hang a caller). Lets readers such as the ingest profiler see + * complete transcripts despite the fire-and-forget append design. + */ +export async function flushToolCallLogs(timeoutMs = 5000): Promise { + const pending = [...pendingWrites]; + if (pending.length === 0) { + return; + } + const settled = Promise.allSettled(pending).then(() => undefined); + if (timeoutMs <= 0) { + await settled; + return; + } + let timer: ReturnType | undefined; + const timeout = new Promise((resolve) => { + timer = setTimeout(resolve, timeoutMs); + timer.unref?.(); + }); + try { + await Promise.race([settled, timeout]); + } finally { + if (timer) { + clearTimeout(timer); + } + } } function safeStringify(v: unknown): string { diff --git a/packages/cli/src/context/ingest/types.ts b/packages/cli/src/context/ingest/types.ts index 337885af..925f3d82 100644 --- a/packages/cli/src/context/ingest/types.ts +++ b/packages/cli/src/context/ingest/types.ts @@ -220,5 +220,6 @@ export interface IngestJobPhase { export interface IngestJobContext { jobId: string; memoryFlow?: MemoryFlowEventSink; + abortSignal?: AbortSignal; startPhase(weight: number): IngestJobPhase; } diff --git a/packages/cli/src/context/llm/ai-sdk-runtime.ts b/packages/cli/src/context/llm/ai-sdk-runtime.ts index 33a55c11..d5a60c7b 100644 --- a/packages/cli/src/context/llm/ai-sdk-runtime.ts +++ b/packages/cli/src/context/llm/ai-sdk-runtime.ts @@ -3,12 +3,15 @@ import type { KtxLlmProvider } from '../../llm/types.js'; import { generateText, Output, stepCountIs, type FlexibleSchema, type TelemetrySettings, type ToolSet } from 'ai'; import type { z } from 'zod'; import { noopLogger, type KtxLogger } from '../../context/core/config.js'; +import { isAbortError } from '../core/abort.js'; import { summarizeKtxLlmDebugRequest, type KtxLlmDebugRequestRecorder } from './debug-request-recorder.js'; +import type { RateLimitGovernor, RateLimitProvider, RateLimitSignal } from './rate-limit-governor.js'; import { createAiSdkToolSet } from './runtime-tools.js'; import type { KtxGenerateObjectInput, KtxGenerateTextInput, KtxLlmRuntimePort, + LlmTokenUsage, RunLoopParams, RunLoopResult, } from './runtime-port.js'; @@ -17,17 +20,151 @@ interface AgentTelemetryPort { createTelemetry(tags: Record): TelemetrySettings; } +interface MaybeUsage { + inputTokens?: number; + outputTokens?: number; + totalTokens?: number; +} + +function toLlmTokenUsage(usage: MaybeUsage | undefined): LlmTokenUsage { + if (!usage) { + return {}; + } + return { + ...(usage.inputTokens !== undefined ? { inputTokens: usage.inputTokens } : {}), + ...(usage.outputTokens !== undefined ? { outputTokens: usage.outputTokens } : {}), + ...(usage.totalTokens !== undefined ? { totalTokens: usage.totalTokens } : {}), + }; +} + export interface AiSdkKtxLlmRuntimeDeps { llmProvider: KtxLlmProvider; telemetry?: AgentTelemetryPort; logger?: KtxLogger; debugRequestRecorder?: KtxLlmDebugRequestRecorder; + rateLimitGovernor?: Pick; } function hasTools(tools: Record): boolean { return Object.keys(tools).length > 0; } +function modelProviderName(model: unknown): RateLimitProvider { + const provider = (model as { provider?: string }).provider ?? ''; + return provider.includes('vertex') || provider.includes('google') ? 'vertex' : 'anthropic-api'; +} + +interface HeaderLimitPair { + limit: string; + remaining: string; + rateLimitType: string; +} + +const RATE_LIMIT_HEADER_PAIRS: HeaderLimitPair[] = [ + { + limit: 'anthropic-ratelimit-requests-limit', + remaining: 'anthropic-ratelimit-requests-remaining', + rateLimitType: 'rpm', + }, + { + limit: 'anthropic-ratelimit-tokens-limit', + remaining: 'anthropic-ratelimit-tokens-remaining', + rateLimitType: 'tpm', + }, + { + limit: 'anthropic-ratelimit-input-tokens-limit', + remaining: 'anthropic-ratelimit-input-tokens-remaining', + rateLimitType: 'itpm', + }, + { + limit: 'anthropic-ratelimit-output-tokens-limit', + remaining: 'anthropic-ratelimit-output-tokens-remaining', + rateLimitType: 'otpm', + }, + { + limit: 'x-ratelimit-limit-requests', + remaining: 'x-ratelimit-remaining-requests', + rateLimitType: 'rpm', + }, + { + limit: 'x-ratelimit-limit-tokens', + remaining: 'x-ratelimit-remaining-tokens', + rateLimitType: 'tpm', + }, +]; + +function normalizeHeaders(headers: unknown): Record { + if (!headers || typeof headers !== 'object') { + return {}; + } + const get = (headers as { get?: unknown }).get; + if (typeof get === 'function') { + const out: Record = {}; + for (const pair of RATE_LIMIT_HEADER_PAIRS) { + const limit = get.call(headers, pair.limit); + const remaining = get.call(headers, pair.remaining); + if (typeof limit === 'string') out[pair.limit] = limit; + if (typeof remaining === 'string') out[pair.remaining] = remaining; + } + return out; + } + return Object.fromEntries( + Object.entries(headers as Record) + .filter((entry): entry is [string, string | number] => typeof entry[1] === 'string' || typeof entry[1] === 'number') + .map(([key, value]) => [key.toLowerCase(), String(value)]), + ); +} + +function numericHeader(headers: Record, key: string): number | undefined { + const value = Number(headers[key]); + return Number.isFinite(value) && value >= 0 ? value : undefined; +} + +function utilizationForPair(headers: Record, pair: HeaderLimitPair): number | undefined { + const limit = numericHeader(headers, pair.limit); + const remaining = numericHeader(headers, pair.remaining); + if (limit === undefined || remaining === undefined || limit <= 0) { + return undefined; + } + return 1 - Math.min(limit, remaining) / limit; +} + +function aiSdkHeaderRateLimitSignal(provider: RateLimitProvider, result: unknown): RateLimitSignal | undefined { + const headers = normalizeHeaders((result as { response?: { headers?: unknown } }).response?.headers); + let best: { utilization: number; rateLimitType: string } | undefined; + for (const pair of RATE_LIMIT_HEADER_PAIRS) { + const utilization = utilizationForPair(headers, pair); + if (utilization === undefined) { + continue; + } + if (!best || utilization > best.utilization) { + best = { utilization, rateLimitType: pair.rateLimitType }; + } + } + if (!best) { + return undefined; + } + return { + provider, + status: 'allowed', + rateLimitType: best.rateLimitType, + utilization: Number(best.utilization.toFixed(4)), + }; +} + +function retryAfterMs(error: unknown): number | undefined { + const value = (error as { retryAfter?: unknown }).retryAfter; + if (typeof value === 'number' && Number.isFinite(value) && value > 0) { + return value < 1_000 ? value * 1_000 : value; + } + return undefined; +} + +function isAiSdkRateLimitError(error: unknown): boolean { + const record = error as { name?: string; statusCode?: number; status?: number }; + return record.name === 'TooManyRequestsError' || record.statusCode === 429 || record.status === 429; +} + export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort { private readonly logger: KtxLogger; @@ -35,6 +172,41 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort { this.logger = deps.logger ?? noopLogger; } + private async generateTextWithRateLimitRetry( + provider: RateLimitProvider, + abortSignal: AbortSignal | undefined, + run: () => Promise, + ): Promise { + // maxRetryAttempts() returns 1 when no governor is present or pacing is + // disabled, so a 429 throws immediately instead of hammering the provider + // with no backoff; the AI SDK's own maxRetries still handles transient 429s. + const maxAttempts = this.deps.rateLimitGovernor?.maxRetryAttempts() ?? 1; + let attempt = 0; + while (true) { + await this.deps.rateLimitGovernor?.waitForReady(abortSignal); + try { + const result = await run(); + const signal = aiSdkHeaderRateLimitSignal(provider, result); + if (signal) { + this.deps.rateLimitGovernor?.report(signal); + } + return result; + } catch (error) { + if (isAbortError(error) || !isAiSdkRateLimitError(error) || attempt >= maxAttempts - 1) { + throw error; + } + attempt += 1; + const retryAfter = retryAfterMs(error); + this.deps.rateLimitGovernor?.report({ + provider, + status: 'rejected', + rateLimitType: 'http_429', + ...(retryAfter !== undefined ? { retryAfterMs: retryAfter } : {}), + }); + } + } + } + async generateText(input: KtxGenerateTextInput): Promise { const model = this.deps.llmProvider.getModel(input.role); if ((model as { provider?: string }).provider === 'deterministic') { @@ -48,12 +220,14 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort { model, }); const split = splitKtxSystemMessages(built.messages); - const result = await generateText({ + const startedAt = Date.now(); + const request = { model, temperature: input.temperature ?? 0, ...(split.system ? { system: split.system } : {}), messages: split.messages, tools: built.tools as ToolSet, + ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}), ...(hasTools(tools) ? { experimental_repairToolCall: this.deps.llmProvider.repairToolCallHandler({ @@ -61,7 +235,9 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort { }), } : {}), - }); + }; + const result = await this.generateTextWithRateLimitRetry(modelProviderName(model), input.abortSignal, () => generateText(request)); + input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: toLlmTokenUsage(result.totalUsage ?? result.usage) }); if (typeof result.text !== 'string') { throw new Error('KTX LLM text generation returned no text'); } @@ -80,12 +256,14 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort { model, }); const split = splitKtxSystemMessages(built.messages); - const result = await generateText({ + const startedAt = Date.now(); + const request = { model, temperature: input.temperature ?? 0, ...(split.system ? { system: split.system } : {}), messages: split.messages, tools: built.tools as ToolSet, + ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}), ...(hasTools(tools) ? { experimental_repairToolCall: this.deps.llmProvider.repairToolCallHandler({ @@ -94,7 +272,9 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort { } : {}), output: Output.object({ schema: input.schema as unknown as FlexibleSchema }), - }); + }; + const result = await this.generateTextWithRateLimitRetry(modelProviderName(model), input.abortSignal, () => generateText(request)); + input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: toLlmTokenUsage(result.totalUsage ?? result.usage) }); if (result.output == null) { throw new Error('KTX LLM object generation returned no output'); } @@ -103,6 +283,8 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort { async runAgentLoop(params: RunLoopParams): Promise { let stepIndex = 0; + const startedAt = Date.now(); + const stepBoundariesMs: number[] = []; try { const model = this.deps.llmProvider.getModel(params.modelRole); const tools = createAiSdkToolSet(params.toolSet); @@ -128,7 +310,7 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort { }), ); - await generateText({ + const request = { model, temperature: 0, stopWhen: stepCountIs(params.stepBudget), @@ -139,8 +321,10 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort { ...(promptMessages.system ? { system: promptMessages.system } : {}), messages: promptMessages.messages, tools: built.tools as ToolSet, + ...(params.abortSignal ? { abortSignal: params.abortSignal } : {}), onStepFinish: async () => { stepIndex += 1; + stepBoundariesMs.push(Date.now() - startedAt); if (!params.onStepFinish) { return; } @@ -154,12 +338,28 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort { ); } }, - }); - return { stopReason: 'natural' }; + }; + const result = await this.generateTextWithRateLimitRetry(modelProviderName(model), params.abortSignal, () => generateText(request)); + return { + stopReason: 'natural', + metrics: { + totalMs: Date.now() - startedAt, + stepCount: stepIndex, + stepBoundariesMs, + usage: toLlmTokenUsage(result.totalUsage ?? result.usage), + }, + }; } catch (error) { + if (isAbortError(error)) { + throw error; + } const err = error instanceof Error ? error : new Error(String(error)); this.logger.warn(`[agent-runner] loop failed: ${err.message}`); - return { stopReason: 'error', error: err }; + return { + stopReason: 'error', + error: err, + metrics: { totalMs: Date.now() - startedAt, stepCount: stepIndex, stepBoundariesMs, usage: {} }, + }; } } } diff --git a/packages/cli/src/context/llm/claude-code-runtime.ts b/packages/cli/src/context/llm/claude-code-runtime.ts index 0eb3eadb..26bd0529 100644 --- a/packages/cli/src/context/llm/claude-code-runtime.ts +++ b/packages/cli/src/context/llm/claude-code-runtime.ts @@ -7,20 +7,46 @@ import { } from '@anthropic-ai/claude-agent-sdk'; import { z } from 'zod'; import { noopLogger, type KtxLogger } from '../../context/core/config.js'; +import { createAbortError, isAbortError, throwIfAborted } from '../core/abort.js'; import { createKtxClaudeCodeEnv } from './claude-code-env.js'; import { resolveClaudeCodeModel } from './claude-code-models.js'; +import type { RateLimitGovernor, RateLimitSignal } from './rate-limit-governor.js'; import { createClaudeSdkTools, mcpToolIds } from './runtime-tools.js'; import type { KtxGenerateObjectInput, KtxGenerateTextInput, KtxLlmRuntimePort, KtxRuntimeToolSet, + LlmTokenUsage, RunLoopParams, RunLoopResult, RunLoopStopReason, } from './runtime-port.js'; -type QueryFn = (params: Parameters[0]) => AsyncIterable; +type QueryResult = AsyncIterable & { + interrupt?: () => void | Promise; +}; + +type QueryFn = (params: Parameters[0]) => QueryResult; + +interface ClaudeQueryOutcome { + result: SDKResultMessage; + rejectedRateLimitSignal?: RateLimitSignal; +} + +function claudeTokenUsage(result: SDKResultMessage): LlmTokenUsage { + const usage = (result as { usage?: { input_tokens?: number; output_tokens?: number } }).usage; + if (!usage) { + return {}; + } + const { input_tokens: inputTokens, output_tokens: outputTokens } = usage; + const totalTokens = inputTokens !== undefined && outputTokens !== undefined ? inputTokens + outputTokens : undefined; + return { + ...(inputTokens !== undefined ? { inputTokens } : {}), + ...(outputTokens !== undefined ? { outputTokens } : {}), + ...(totalTokens !== undefined ? { totalTokens } : {}), + }; +} export interface ClaudeCodeKtxLlmRuntimeDeps { projectDir: string; @@ -28,6 +54,7 @@ export interface ClaudeCodeKtxLlmRuntimeDeps { query?: QueryFn; env?: NodeJS.ProcessEnv; logger?: KtxLogger; + rateLimitGovernor?: Pick; } const BUILTIN_TOOLS = [ @@ -58,6 +85,22 @@ function isResult(message: SDKMessage): message is SDKResultMessage { return message.type === 'result'; } +// Skip emissions the SDK does not count toward `num_turns`: `pause_turn` continuations and +// errored partials (e.g. `max_output_tokens`) it retries internally. Without this, the +// runtime's step counter outruns `maxTurns` and the HUD renders e.g. `step 69/40`. +function countsAsAssistantTurn(message: SDKMessage): boolean { + if (message.type !== 'assistant' || message.parent_tool_use_id !== null) { + return false; + } + if (message.error !== undefined) { + return false; + } + if (message.message.stop_reason === 'pause_turn') { + return false; + } + return true; +} + function resultError(result: SDKResultMessage): Error | undefined { if (result.subtype === 'success') { return undefined; @@ -126,6 +169,74 @@ function expectedMcpServerNames(tools: KtxRuntimeToolSet | undefined): Set 0 ? new Set([KTX_MCP_SERVER_NAME]) : new Set(); } +const CLAUDE_RATE_LIMIT_ERROR_MARKERS = /\b429\b|rate limit|too many requests|quota exceeded|overloaded|max_retries/i; + +function normalizeClaudeResetAtMs(value: unknown): number | undefined { + if (typeof value === 'number' && Number.isFinite(value) && value > 0) { + return Math.round(value < 10_000_000_000 ? value * 1_000 : value); + } + if (typeof value === 'string') { + const numeric = Number(value); + if (Number.isFinite(numeric) && numeric > 0) { + return normalizeClaudeResetAtMs(numeric); + } + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : undefined; + } + return undefined; +} + +function isClaudeRateLimitResult(result: SDKResultMessage, rejectedSignal: RateLimitSignal | undefined): boolean { + const error = resultError(result); + if (!error) { + return false; + } + if (rejectedSignal?.status === 'rejected') { + return true; + } + const resultDetails = result as { + stop_reason?: unknown; + terminal_reason?: unknown; + errors?: unknown[]; + }; + const details = [ + error.message, + resultDetails.stop_reason, + resultDetails.terminal_reason, + ...(resultDetails.errors ?? []), + ] + .filter((value): value is string => typeof value === 'string' && value.length > 0) + .join('\n'); + return CLAUDE_RATE_LIMIT_ERROR_MARKERS.test(details); +} + +function claudeRateLimitSignal(message: SDKMessage): RateLimitSignal | null { + const record = message as unknown as Record; + if (record.type === 'rate_limit_event') { + const info = record.rate_limit_info as Record | undefined; + if (!info) return null; + const rawStatus = typeof info.status === 'string' ? info.status : 'allowed'; + const resetAtMs = normalizeClaudeResetAtMs(info.resetsAt); + return { + provider: 'claude-subscription', + status: rawStatus === 'rejected' ? 'rejected' : rawStatus === 'allowed_warning' ? 'warning' : 'allowed', + ...(resetAtMs !== undefined ? { resetAtMs } : {}), + ...(typeof info.rateLimitType === 'string' ? { rateLimitType: info.rateLimitType } : {}), + ...(typeof info.utilization === 'number' ? { utilization: info.utilization } : {}), + }; + } + if (record.subtype === 'api_retry' || record.type === 'api_retry') { + const retryDelayMs = typeof record.retry_delay_ms === 'number' ? record.retry_delay_ms : undefined; + return { + provider: 'claude-subscription', + status: 'warning', + ...(retryDelayMs !== undefined ? { retryAfterMs: retryDelayMs } : {}), + rateLimitType: 'api_retry', + }; + } + return null; +} + function managedMcpSettings(serverNames: string[]): NonNullable { return { allowManagedMcpServersOnly: true, @@ -186,21 +297,63 @@ async function collectResult(params: { allowedToolIds: Set; expectedMcpServerNames: Set; onAssistantTurn?: () => Promise; -}): Promise { + rateLimitGovernor?: Pick; + abortSignal?: AbortSignal; +}): Promise { let result: SDKResultMessage | undefined; - for await (const message of params.query({ prompt: params.prompt, options: params.options })) { - assertInitIsolation(message, params.allowedToolIds, params.expectedMcpServerNames); - if (message.type === 'assistant' && message.parent_tool_use_id === null) { - await params.onAssistantTurn?.(); - } - if (isResult(message)) { - result = message; + let rejectedRateLimitSignal: RateLimitSignal | undefined; + throwIfAborted(params.abortSignal); + await params.rateLimitGovernor?.waitForReady(params.abortSignal); + throwIfAborted(params.abortSignal); + const queryResult = params.query({ prompt: params.prompt, options: params.options }); + const onAbort = () => { + void Promise.resolve(queryResult.interrupt?.()).catch(() => undefined); + }; + params.abortSignal?.addEventListener('abort', onAbort, { once: true }); + try { + for await (const message of queryResult) { + throwIfAborted(params.abortSignal); + const rateLimitSignal = claudeRateLimitSignal(message); + if (rateLimitSignal) { + if (rateLimitSignal.status === 'rejected') { + rejectedRateLimitSignal = rateLimitSignal; + } + params.rateLimitGovernor?.report(rateLimitSignal); + } + assertInitIsolation(message, params.allowedToolIds, params.expectedMcpServerNames); + if (countsAsAssistantTurn(message)) { + await params.onAssistantTurn?.(); + } + if (isResult(message)) { + result = message; + } } + } finally { + params.abortSignal?.removeEventListener('abort', onAbort); + } + if (params.abortSignal?.aborted) { + throw createAbortError(); } if (!result) { throw new Error('Claude Code query returned no result message'); } - return result; + return { + result, + ...(rejectedRateLimitSignal ? { rejectedRateLimitSignal } : {}), + }; +} + +async function collectResultWithRateLimitRetry(params: Parameters[0]): Promise { + // maxRetryAttempts() returns 1 when no governor is present or pacing is + // disabled, so a rate-limited result surfaces without an extra query; the + // Claude Code SDK applies its own backoff for transient rejections. + const maxAttempts = params.rateLimitGovernor?.maxRetryAttempts() ?? 1; + for (let attempt = 0; ; attempt += 1) { + const outcome = await collectResult(params); + if (!isClaudeRateLimitResult(outcome.result, outcome.rejectedRateLimitSignal) || attempt >= maxAttempts - 1) { + return outcome.result; + } + } } export class ClaudeCodeKtxLlmRuntime implements KtxLlmRuntimePort { @@ -220,13 +373,17 @@ export class ClaudeCodeKtxLlmRuntime implements KtxLlmRuntimePort { maxTurns: 1, tools: input.tools, }); - const result = await collectResult({ + const startedAt = Date.now(); + const result = await collectResultWithRateLimitRetry({ query: this.runQuery, prompt: [input.system, input.prompt].filter(Boolean).join('\n\n'), options, allowedToolIds: new Set(mcpToolIds(input.tools ?? {})), expectedMcpServerNames: expectedMcpServerNames(input.tools), + rateLimitGovernor: this.deps.rateLimitGovernor, + abortSignal: input.abortSignal, }); + input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: claudeTokenUsage(result) }); const error = resultError(result); if (error) { throw error; @@ -255,13 +412,17 @@ export class ClaudeCodeKtxLlmRuntime implements KtxLlmRuntimePort { }), outputFormat: { type: 'json_schema' as const, schema: jsonSchema(input.schema as z.ZodType) }, }; - const result = await collectResult({ + const startedAt = Date.now(); + const result = await collectResultWithRateLimitRetry({ query: this.runQuery, prompt: [input.system, input.prompt].filter(Boolean).join('\n\n'), options, allowedToolIds: new Set([...mcpToolIds(input.tools ?? {}), STRUCTURED_OUTPUT_TOOL_NAME]), expectedMcpServerNames: expectedMcpServerNames(input.tools), + rateLimitGovernor: this.deps.rateLimitGovernor, + abortSignal: input.abortSignal, }); + input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: claudeTokenUsage(result) }); const error = resultError(result); if (error) { throw error; @@ -274,6 +435,8 @@ export class ClaudeCodeKtxLlmRuntime implements KtxLlmRuntimePort { async runAgentLoop(params: RunLoopParams): Promise { let stepIndex = 0; + const startedAt = Date.now(); + const stepBoundariesMs: number[] = []; try { const options = baseOptions({ projectDir: this.deps.projectDir, @@ -282,14 +445,17 @@ export class ClaudeCodeKtxLlmRuntime implements KtxLlmRuntimePort { maxTurns: params.stepBudget, tools: params.toolSet, }); - const result = await collectResult({ + const result = await collectResultWithRateLimitRetry({ query: this.runQuery, prompt: params.userPrompt, options: { ...options, systemPrompt: params.systemPrompt }, allowedToolIds: new Set(mcpToolIds(params.toolSet)), expectedMcpServerNames: expectedMcpServerNames(params.toolSet), + rateLimitGovernor: this.deps.rateLimitGovernor, + abortSignal: params.abortSignal, onAssistantTurn: async () => { stepIndex += 1; + stepBoundariesMs.push(Date.now() - startedAt); if (!params.onStepFinish) { return; } @@ -306,10 +472,26 @@ export class ClaudeCodeKtxLlmRuntime implements KtxLlmRuntimePort { }); const stopReason = mapClaudeCodeStopReason(result); const error = resultError(result); - return { stopReason, ...(stopReason === 'error' && error ? { error } : {}) }; + return { + stopReason, + ...(stopReason === 'error' && error ? { error } : {}), + metrics: { + totalMs: Date.now() - startedAt, + stepCount: stepIndex, + stepBoundariesMs, + usage: claudeTokenUsage(result), + }, + }; } catch (error) { + if (isAbortError(error)) { + throw error; + } const err = error instanceof Error ? error : new Error(String(error)); - return { stopReason: 'error', error: err }; + return { + stopReason: 'error', + error: err, + metrics: { totalMs: Date.now() - startedAt, stepCount: stepIndex, stepBoundariesMs, usage: {} }, + }; } } } @@ -337,7 +519,7 @@ export async function runClaudeCodeAuthProbe(input: { env: input.env, maxTurns: 1, }); - const result = await collectResult({ + const result = await collectResultWithRateLimitRetry({ query: input.query ?? defaultQuery, prompt: 'Reply with exactly: ok', options, diff --git a/packages/cli/src/context/llm/codex-exec-events.ts b/packages/cli/src/context/llm/codex-exec-events.ts new file mode 100644 index 00000000..86e13694 --- /dev/null +++ b/packages/cli/src/context/llm/codex-exec-events.ts @@ -0,0 +1,194 @@ +import type { LlmTokenUsage, RunLoopStopReason } from './runtime-port.js'; + +export interface CodexExecEventSummary { + finalText: string; + stopReason: RunLoopStopReason; + usage: LlmTokenUsage; + stepCount: number; + stepBoundariesMs: number[]; + toolCallCount: number; + toolFailures: string[]; + error?: Error; +} + +interface CodexEventParseOptions { + startedAt?: number; + now?: () => number; +} + +function record(value: unknown): Record | undefined { + return value && typeof value === 'object' ? (value as Record) : undefined; +} + +/** + * Codex thread items that represent a discrete agent action consuming one loop + * step. The step budget caps the total number of these regardless of which + * capability the agent reaches for, so built-in `command_execution` (and any + * file/web action the public Codex surface still exposes) count alongside our + * own `mcp_tool_call` items rather than only the MCP ones. + */ +const AGENT_STEP_ITEM_TYPES = new Set(['command_execution', 'mcp_tool_call', 'file_change', 'web_search']); + +export function isCompletedAgentStep(event: unknown): boolean { + const eventRecord = record(event); + if (eventRecord?.type !== 'item.completed') { + return false; + } + const itemType = record(eventRecord.item)?.type; + return typeof itemType === 'string' && AGENT_STEP_ITEM_TYPES.has(itemType); +} + +function text(value: unknown): string | undefined { + return typeof value === 'string' && value.trim().length > 0 ? value : undefined; +} + +function numberValue(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + +function usageFrom(value: unknown): LlmTokenUsage { + const usage = record(value); + if (!usage) { + return {}; + } + const inputTokens = numberValue(usage.input_tokens ?? usage.inputTokens); + const outputTokens = numberValue(usage.output_tokens ?? usage.outputTokens); + const explicitTotalTokens = numberValue(usage.total_tokens ?? usage.totalTokens); + const totalTokens = + explicitTotalTokens ?? + (inputTokens !== undefined && outputTokens !== undefined ? inputTokens + outputTokens : undefined); + return { + ...(inputTokens !== undefined ? { inputTokens } : {}), + ...(outputTokens !== undefined ? { outputTokens } : {}), + ...(totalTokens !== undefined ? { totalTokens } : {}), + }; +} + +function stopReasonFrom(value: unknown): RunLoopStopReason { + const reason = text(value)?.toLowerCase(); + if (reason && /(budget|max_turn|max-turn|limit)/.test(reason)) { + return 'budget'; + } + return 'natural'; +} + +function errorMessageFrom(value: unknown): string { + if (value instanceof Error) { + return value.message; + } + const asRecord = record(value); + const message = text(asRecord?.message); + return message ?? text(value) ?? 'Codex turn failed'; +} + +/** + * Codex serializes API failures as a JSON envelope inside the event message + * (e.g. `{"type":"error","status":400,"error":{"message":"…"}}`). Surface the + * human-readable inner message so callers don't leak raw JSON; pass plain + * strings through unchanged. + */ +function unwrapCodexApiErrorMessage(raw: string): string { + const trimmed = raw.trim(); + if (!trimmed.startsWith('{')) { + return raw; + } + try { + const parsed = record(JSON.parse(trimmed)); + return text(record(parsed?.error)?.message) ?? text(parsed?.message) ?? raw; + } catch { + return raw; + } +} + +/** @internal */ +export function parseCodexExecEventLine(line: string): unknown { + try { + return JSON.parse(line) as unknown; + } catch (error) { + throw new Error(`Codex JSONL event stream was malformed: ${error instanceof Error ? error.message : String(error)}`); + } +} + +export function summarizeCodexExecEvents( + events: Iterable, + options: CodexEventParseOptions = {}, +): CodexExecEventSummary { + const startedAt = options.startedAt ?? Date.now(); + const now = options.now ?? Date.now; + let finalText = ''; + let stopReason: RunLoopStopReason = 'natural'; + let usage: LlmTokenUsage = {}; + let turnCount = 0; + let completedStepCount = 0; + const stepBoundariesMs: number[] = []; + let toolCallCount = 0; + const toolFailures: string[] = []; + let error: Error | undefined; + + for (const event of events) { + const eventRecord = record(event); + const eventType = text(eventRecord?.type); + if (!eventRecord || !eventType) { + continue; + } + + if (eventType === 'turn.started') { + turnCount += 1; + continue; + } + + const item = record(eventRecord.item); + const itemType = text(item?.type); + + if (eventType === 'item.started' && itemType === 'mcp_tool_call') { + toolCallCount += 1; + continue; + } + + if (isCompletedAgentStep(event)) { + completedStepCount += 1; + stepBoundariesMs.push(now() - startedAt); + // Only MCP tool calls fail the loop: a non-zero `command_execution` exit + // is normal agent exploration, not a runtime error. `status` is the + // authoritative signal (the SDK always sets it); the SDK also serializes + // `error: null` on successful calls, so an explicit-null `error` must NOT + // be read as a failure — only a populated error object counts. + if (itemType === 'mcp_tool_call' && (item?.status === 'failed' || (item?.error !== undefined && item?.error !== null))) { + const name = text(item?.name) ?? text(item?.tool) ?? text(item?.tool_name) ?? 'unknown'; + toolFailures.push(`${name}: ${errorMessageFrom(item?.error)}`); + } + continue; + } + + if (eventType === 'item.completed' && itemType === 'agent_message') { + finalText = text(item?.text) ?? finalText; + continue; + } + + if (eventType === 'turn.completed') { + usage = usageFrom(eventRecord.usage); + if (completedStepCount === 0) { + stepBoundariesMs.push(now() - startedAt); + } + stopReason = stopReasonFrom(eventRecord.reason ?? eventRecord.stop_reason ?? eventRecord.terminal_reason); + continue; + } + + if (eventType === 'turn.failed' || eventType === 'error') { + stopReason = 'error'; + error = new Error(unwrapCodexApiErrorMessage(errorMessageFrom(eventRecord.error ?? eventRecord.message))); + continue; + } + } + + return { + finalText, + stopReason, + usage, + stepCount: completedStepCount > 0 ? completedStepCount : turnCount, + stepBoundariesMs, + toolCallCount, + toolFailures, + ...(error ? { error } : {}), + }; +} diff --git a/packages/cli/src/context/llm/codex-isolation.ts b/packages/cli/src/context/llm/codex-isolation.ts new file mode 100644 index 00000000..d54ac1f8 --- /dev/null +++ b/packages/cli/src/context/llm/codex-isolation.ts @@ -0,0 +1,9 @@ +export const CODEX_ISOLATION_WARNING = + 'Codex backend isolation is limited by the public Codex SDK/CLI surface: ktx restricts the runtime MCP server to the current ktx tool set, disables Codex web search, asks for a read-only sandbox, and sets approval_policy=never, but Codex may still load user Codex config and built-in command execution or read-only file capabilities.'; + +export const CODEX_ISOLATION_WARNING_FIX = + 'Use llm.provider.backend: claude-code when you need stricter Claude-Code-style runtime tool isolation, or remove host Codex MCP/tool config before running untrusted prompts through the codex backend.'; + +export function formatCodexIsolationWarning(): string { + return `${CODEX_ISOLATION_WARNING} ${CODEX_ISOLATION_WARNING_FIX}`; +} diff --git a/packages/cli/src/context/llm/codex-mcp-runtime-server.ts b/packages/cli/src/context/llm/codex-mcp-runtime-server.ts new file mode 100644 index 00000000..eacf28f9 --- /dev/null +++ b/packages/cli/src/context/llm/codex-mcp-runtime-server.ts @@ -0,0 +1,87 @@ +import { randomBytes } from 'node:crypto'; +import type { Server } from 'node:http'; +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import type { KtxMcpServerLike } from '../mcp/types.js'; +import { runKtxMcpHttpServer, type KtxMcpHttpServerHandle } from '../../mcp-http-server.js'; +import type { KtxRuntimeToolSet } from './runtime-port.js'; +import { normalizeKtxRuntimeToolOutput } from './runtime-tools.js'; + +/** @internal */ +export interface CreateCodexRuntimeMcpServerInput { + server?: KtxMcpServerLike; + toolSet: KtxRuntimeToolSet; +} + +export interface CodexRuntimeMcpServerHandle { + url: string; + bearerTokenEnvVar: 'KTX_CODEX_RUNTIME_MCP_TOKEN'; + bearerToken: string; + close(): Promise; +} + +type RunServer = typeof runKtxMcpHttpServer; + +export interface StartCodexRuntimeMcpServerInput { + projectDir: string; + toolSet: KtxRuntimeToolSet; + runServer?: RunServer; +} + +/** @internal */ +export function createCodexRuntimeMcpServer(input: CreateCodexRuntimeMcpServerInput): KtxMcpServerLike { + const server = + input.server ?? + (new McpServer({ + name: 'ktx-runtime', + version: '0.0.0', + }) as KtxMcpServerLike); + + for (const descriptor of Object.values(input.toolSet)) { + server.registerTool( + descriptor.name, + { + description: descriptor.description, + inputSchema: descriptor.inputSchema.shape, + }, + async (toolInput) => { + const normalized = normalizeKtxRuntimeToolOutput(await descriptor.execute(toolInput)); + return { + content: [{ type: 'text', text: normalized.markdown }], + ...(normalized.structured !== undefined && normalized.structured !== null && typeof normalized.structured === 'object' + ? { structuredContent: normalized.structured as object } + : {}), + }; + }, + ); + } + + return server; +} + +function serverPort(server: Server, fallback: number): number { + const address = server.address(); + return typeof address === 'object' && address ? address.port : fallback; +} + +export async function startCodexRuntimeMcpServer( + input: StartCodexRuntimeMcpServerInput, +): Promise { + const bearerToken = randomBytes(32).toString('hex'); + const runServer = input.runServer ?? runKtxMcpHttpServer; + const handle = (await runServer({ + projectDir: input.projectDir, + host: '127.0.0.1', + port: 0, + token: bearerToken, + allowedHosts: ['127.0.0.1', 'localhost'], + allowedOrigins: [], + createMcpServer: () => createCodexRuntimeMcpServer({ toolSet: input.toolSet }) as McpServer, + })) as KtxMcpHttpServerHandle; + const port = serverPort(handle.server, 0); + return { + url: `http://127.0.0.1:${port}/mcp`, + bearerTokenEnvVar: 'KTX_CODEX_RUNTIME_MCP_TOKEN', + bearerToken, + close: () => handle.close(), + }; +} diff --git a/packages/cli/src/context/llm/codex-models.ts b/packages/cli/src/context/llm/codex-models.ts new file mode 100644 index 00000000..1a8b9b9d --- /dev/null +++ b/packages/cli/src/context/llm/codex-models.ts @@ -0,0 +1,20 @@ +export const DEFAULT_CODEX_MODEL = 'gpt-5.5'; + +const CODEX_MODEL_ALIASES: Record = { + codex: DEFAULT_CODEX_MODEL, + default: DEFAULT_CODEX_MODEL, +}; + +const EXPLICIT_CODEX_MODEL_ID = /^(?:gpt|codex)-[a-z0-9][a-z0-9._-]*$/i; + +export function resolveCodexModel(model: string): string { + const normalized = model.trim(); + const alias = CODEX_MODEL_ALIASES[normalized]; + if (alias) { + return alias; + } + if (EXPLICIT_CODEX_MODEL_ID.test(normalized)) { + return normalized; + } + throw new Error(`Unsupported Codex model "${model}". Use codex, default, or a gpt-* / codex-* model id.`); +} diff --git a/packages/cli/src/context/llm/codex-runtime-config.ts b/packages/cli/src/context/llm/codex-runtime-config.ts new file mode 100644 index 00000000..74de9efe --- /dev/null +++ b/packages/cli/src/context/llm/codex-runtime-config.ts @@ -0,0 +1,38 @@ +interface CodexRuntimeMcpConfig { + url: string; + bearerTokenEnvVar: string; + bearerToken: string; + toolNames: string[]; +} + +export interface BuildCodexRuntimeConfigInput { + model: string; + mcp?: CodexRuntimeMcpConfig; +} + +export interface CodexRuntimeConfig { + configOverrides: Record; + env: Record; +} + +export function buildCodexRuntimeConfig(input: BuildCodexRuntimeConfigInput): CodexRuntimeConfig { + const configOverrides: Record = { + history: { persistence: 'none' }, + }; + const env: Record = {}; + + if (input.mcp) { + configOverrides.mcp_servers = { + ktx: { + url: input.mcp.url, + bearer_token_env_var: input.mcp.bearerTokenEnvVar, + enabled_tools: input.mcp.toolNames, + default_tools_approval_mode: 'approve', + required: true, + }, + }; + env[input.mcp.bearerTokenEnvVar] = input.mcp.bearerToken; + } + + return { configOverrides, env }; +} diff --git a/packages/cli/src/context/llm/codex-runtime.ts b/packages/cli/src/context/llm/codex-runtime.ts new file mode 100644 index 00000000..2958b3f8 --- /dev/null +++ b/packages/cli/src/context/llm/codex-runtime.ts @@ -0,0 +1,445 @@ +import { z } from 'zod'; +import { noopLogger, type KtxLogger } from '../core/config.js'; +import { isAbortError, linkAbortSignal } from '../core/abort.js'; +import { isCompletedAgentStep, summarizeCodexExecEvents, type CodexExecEventSummary } from './codex-exec-events.js'; +import { + startCodexRuntimeMcpServer, + type CodexRuntimeMcpServerHandle, +} from './codex-mcp-runtime-server.js'; +import { resolveCodexModel } from './codex-models.js'; +import { buildCodexRuntimeConfig } from './codex-runtime-config.js'; +import { CodexSdkCliRunner, type CodexSdkRunner } from './codex-sdk-runner.js'; +import type { RateLimitGovernor } from './rate-limit-governor.js'; +import type { + KtxGenerateObjectInput, + KtxGenerateTextInput, + KtxLlmRuntimePort, + KtxRuntimeToolSet, + LlmTokenUsage, + RunLoopParams, + RunLoopResult, +} from './runtime-port.js'; + +export interface CodexKtxLlmRuntimeDeps { + projectDir: string; + modelSlots: { default: string } & Partial>; + runner?: CodexSdkRunner; + startMcpServer?: (input: { projectDir: string; toolSet: KtxRuntimeToolSet }) => Promise; + logger?: KtxLogger; + rateLimitGovernor?: Pick; +} + +function modelForRole(modelSlots: CodexKtxLlmRuntimeDeps['modelSlots'], role: string): string { + return resolveCodexModel(modelSlots[role] ?? modelSlots.default); +} + +function promptWithSystem(system: string | undefined, prompt: string): string { + return [system, prompt].filter(Boolean).join('\n\n'); +} + +interface CollectCodexEventsOptions { + stepBudget?: number; + abortController?: AbortController; + onStep?: (stepIndex: number) => void | Promise; +} + +interface CollectCodexEventsResult { + events: unknown[]; + budgetExceeded: boolean; + streamError?: Error; +} + +function eventRecord(value: unknown): Record | undefined { + return value && typeof value === 'object' ? (value as Record) : undefined; +} + +function isTurnCompleted(event: unknown): boolean { + return eventRecord(event)?.type === 'turn.completed'; +} + +/** + * Drains the Codex stream once, emitting a step as each agent action completes + * so callers see live progress and the step budget is enforced mid-run. Every + * completed agent-action item counts (see {@link isCompletedAgentStep}), so + * built-in `command_execution` steps decrement the budget the same as + * `mcp_tool_call`s. A turn that produced no actions still counts as one step, + * matching the metrics summary and the AI SDK backend. + */ +async function collectEvents( + events: AsyncIterable, + options: CollectCodexEventsOptions = {}, +): Promise { + const collected: unknown[] = []; + let completedSteps = 0; + let sawActionStep = false; + let budgetExceeded = false; + let streamError: Error | undefined; + + // The SDK yields every stdout event, then throws on a non-zero codex exec + // exit. Catch that throw so the events already collected (which carry the + // real `turn.failed`/`error` reason) survive for the summary; the masked + // exit message is kept only as a fallback when no error event was emitted. + try { + for await (const event of events) { + collected.push(event); + + const isActionStep = isCompletedAgentStep(event); + if (isActionStep) { + sawActionStep = true; + } else if (sawActionStep || !isTurnCompleted(event)) { + // Only fall back to counting a bare turn as a step when the turn produced + // no agent actions; a completed turn is terminal, so it never aborts. + continue; + } + + completedSteps += 1; + await options.onStep?.(completedSteps); + if (isActionStep && options.stepBudget !== undefined && completedSteps >= options.stepBudget) { + budgetExceeded = true; + options.abortController?.abort(); + break; + } + } + } catch (error) { + streamError = error instanceof Error ? error : new Error(String(error)); + } + + return { events: collected, budgetExceeded, ...(streamError ? { streamError } : {}) }; +} + +function metrics(summary: CodexExecEventSummary, startedAt: number): { totalMs: number; usage: LlmTokenUsage } { + return { totalMs: Date.now() - startedAt, usage: summary.usage }; +} + +function summaryError(summary: CodexExecEventSummary, streamError?: Error): Error | undefined { + // A `turn.failed`/`error` event carries the real reason; prefer it over the + // SDK's generic non-zero-exit throw. Fall back to the stream error only when + // no event explained the failure (e.g. spawn failure or auth before a turn). + if (summary.error) { + return summary.error; + } + if (summary.toolFailures.length > 0) { + return new Error(`Codex runtime tool call failed: ${summary.toolFailures.join('; ')}`); + } + return streamError; +} + +function assertSuccessfulText(summary: CodexExecEventSummary, streamError?: Error): string { + const error = summaryError(summary, streamError); + if (error) { + throw error; + } + if (!summary.finalText.trim()) { + throw new Error('Codex completed without an agent message'); + } + return summary.finalText; +} + +function parseStructuredOutput>(schema: TSchema, text: string): TOutput { + try { + return schema.parse(JSON.parse(text)); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Codex structured output failed validation: ${message}`); + } +} + +async function mcpForTools(input: { + projectDir: string; + toolSet?: KtxRuntimeToolSet; + startMcpServer: CodexKtxLlmRuntimeDeps['startMcpServer']; +}): Promise { + if (!input.toolSet || Object.keys(input.toolSet).length === 0) { + return undefined; + } + return (input.startMcpServer ?? startCodexRuntimeMcpServer)({ + projectDir: input.projectDir, + toolSet: input.toolSet, + }); +} + +function runtimeToolNames(toolSet: KtxRuntimeToolSet | undefined): string[] { + return Object.values(toolSet ?? {}).map((descriptor) => descriptor.name); +} + +const CODEX_RATE_LIMIT_MARKERS = /\b429\b|rate limit|too many requests|quota exceeded|temporarily overloaded/i; + +function isCodexRateLimitError(error: Error | undefined): boolean { + return !!error && CODEX_RATE_LIMIT_MARKERS.test(error.message); +} + +export class CodexKtxLlmRuntime implements KtxLlmRuntimePort { + private readonly runner: CodexSdkRunner; + private readonly logger: KtxLogger; + + constructor(private readonly deps: CodexKtxLlmRuntimeDeps) { + this.runner = deps.runner ?? new CodexSdkCliRunner(); + this.logger = deps.logger ?? noopLogger; + } + + private async runWithRateLimitRetry( + abortSignal: AbortSignal | undefined, + run: () => Promise, + getError: (result: T) => Error | undefined, + ): Promise { + // maxRetryAttempts() returns 1 when no governor is present or pacing is + // disabled, so an opaque rate-limit failure surfaces on the first attempt + // instead of being retried with no backoff. + const maxAttempts = this.deps.rateLimitGovernor?.maxRetryAttempts() ?? 1; + for (let attempt = 0; ; attempt += 1) { + await this.deps.rateLimitGovernor?.waitForReady(abortSignal); + const lastAttempt = attempt >= maxAttempts - 1; + try { + const result = await run(); + const error = getError(result); + if (!isCodexRateLimitError(error) || lastAttempt) { + return result; + } + } catch (error) { + if (isAbortError(error)) { + throw error; + } + const err = error instanceof Error ? error : new Error(String(error)); + if (!isCodexRateLimitError(err) || lastAttempt) { + throw error; + } + } + this.deps.rateLimitGovernor?.report({ provider: 'codex', status: 'rejected', rateLimitType: 'opaque' }); + } + } + + async generateText(input: KtxGenerateTextInput): Promise { + const startedAt = Date.now(); + const model = modelForRole(this.deps.modelSlots, input.role); + const mcp = await mcpForTools({ + projectDir: this.deps.projectDir, + toolSet: input.tools, + startMcpServer: this.deps.startMcpServer, + }); + try { + const config = buildCodexRuntimeConfig({ + model, + ...(mcp + ? { + mcp: { + url: mcp.url, + bearerTokenEnvVar: mcp.bearerTokenEnvVar, + bearerToken: mcp.bearerToken, + toolNames: runtimeToolNames(input.tools), + }, + } + : {}), + }); + const result = await this.runWithRateLimitRetry( + input.abortSignal, + async () => { + const collected = await collectEvents( + await this.runner.runStreamed({ + projectDir: this.deps.projectDir, + model, + prompt: promptWithSystem(input.system, input.prompt), + configOverrides: config.configOverrides, + env: config.env, + ...(input.abortSignal ? { signal: input.abortSignal } : {}), + }), + ); + const summary = summarizeCodexExecEvents(collected.events, { startedAt }); + return { collected, summary }; + }, + ({ collected, summary }) => summaryError(summary, collected.streamError), + ); + input.onMetrics?.(metrics(result.summary, startedAt)); + return assertSuccessfulText(result.summary, result.collected.streamError); + } finally { + await mcp?.close(); + } + } + + async generateObject>( + input: KtxGenerateObjectInput, + ): Promise { + const startedAt = Date.now(); + const model = modelForRole(this.deps.modelSlots, input.role); + const mcp = await mcpForTools({ + projectDir: this.deps.projectDir, + toolSet: input.tools, + startMcpServer: this.deps.startMcpServer, + }); + try { + const config = buildCodexRuntimeConfig({ + model, + ...(mcp + ? { + mcp: { + url: mcp.url, + bearerTokenEnvVar: mcp.bearerTokenEnvVar, + bearerToken: mcp.bearerToken, + toolNames: runtimeToolNames(input.tools), + }, + } + : {}), + }); + const result = await this.runWithRateLimitRetry( + input.abortSignal, + async () => { + const collected = await collectEvents( + await this.runner.runStreamed({ + projectDir: this.deps.projectDir, + model, + prompt: promptWithSystem(input.system, input.prompt), + configOverrides: config.configOverrides, + env: config.env, + outputSchema: z.toJSONSchema(input.schema, { target: 'draft-7' }) as Record, + ...(input.abortSignal ? { signal: input.abortSignal } : {}), + }), + ); + const summary = summarizeCodexExecEvents(collected.events, { startedAt }); + return { collected, summary }; + }, + ({ collected, summary }) => summaryError(summary, collected.streamError), + ); + input.onMetrics?.(metrics(result.summary, startedAt)); + return parseStructuredOutput(input.schema, assertSuccessfulText(result.summary, result.collected.streamError)); + } finally { + await mcp?.close(); + } + } + + async runAgentLoop(params: RunLoopParams): Promise { + const startedAt = Date.now(); + const model = modelForRole(this.deps.modelSlots, params.modelRole); + let mcp: CodexRuntimeMcpServerHandle | undefined; + try { + mcp = await mcpForTools({ + projectDir: this.deps.projectDir, + toolSet: params.toolSet, + startMcpServer: this.deps.startMcpServer, + }); + const config = buildCodexRuntimeConfig({ + model, + ...(mcp + ? { + mcp: { + url: mcp.url, + bearerTokenEnvVar: mcp.bearerTokenEnvVar, + bearerToken: mcp.bearerToken, + toolNames: runtimeToolNames(params.toolSet), + }, + } + : {}), + }); + const onStep = async (stepIndex: number): Promise => { + try { + await params.onStepFinish?.({ stepIndex, stepBudget: params.stepBudget }); + } catch (error) { + this.logger.warn( + `[codex-runner] onStepFinish callback threw; ignoring: ${error instanceof Error ? error.message : String(error)}`, + ); + } + }; + const result = await this.runWithRateLimitRetry( + params.abortSignal, + async () => { + const linked = linkAbortSignal(params.abortSignal); + const abortController = linked.controller; + try { + const collected = await collectEvents( + await this.runner.runStreamed({ + projectDir: this.deps.projectDir, + model, + prompt: promptWithSystem(params.systemPrompt, params.userPrompt), + configOverrides: config.configOverrides, + env: config.env, + signal: abortController.signal, + }), + { stepBudget: params.stepBudget, abortController, onStep }, + ); + const summary = summarizeCodexExecEvents(collected.events, { startedAt }); + return { collected, summary }; + } finally { + linked.dispose(); + } + }, + ({ collected, summary }) => summaryError(summary, collected.streamError), + ); + const error = summaryError(result.summary, result.collected.streamError); + if (isAbortError(error)) { + throw error; + } + const stopReason = result.collected.budgetExceeded ? 'budget' : error ? 'error' : result.summary.stopReason; + return { + stopReason, + ...(stopReason === 'error' && error ? { error } : {}), + metrics: { + totalMs: Date.now() - startedAt, + usage: result.summary.usage, + stepCount: result.summary.stepCount, + stepBoundariesMs: result.summary.stepBoundariesMs, + }, + }; + } catch (error) { + if (isAbortError(error)) { + throw error; + } + const err = error instanceof Error ? error : new Error(String(error)); + return { + stopReason: 'error', + error: err, + metrics: { totalMs: Date.now() - startedAt, usage: {}, stepCount: 0, stepBoundariesMs: [] }, + }; + } finally { + await mcp?.close(); + } + } +} + +// A rejected model is not an auth failure: Codex authenticated, connected, and +// the API refused the model id. These markers come from the API error envelope +// (e.g. "model is not supported", "invalid_request_error"). +const MODEL_UNAVAILABLE_MARKERS = + /\bnot supported\b|\bnot available\b|\bdoes not exist\b|invalid_request_error|\bunknown model\b|\bunsupported model\b/i; + +function describeCodexProbeFailure(model: string, message: string): { message: string; fix: string } { + if (MODEL_UNAVAILABLE_MARKERS.test(message)) { + const fix = `Run \`codex\` to see the models your account supports, then set llm.models.default in ktx.yaml (or rerun \`ktx setup\`).`; + return { + message: `Codex is authenticated, but the configured model "${model}" is not available for this Codex account. ${fix} Details: ${message}`, + fix, + }; + } + const fix = `Authenticate Codex locally with the Codex CLI, verify the Codex CLI is installed, then rerun setup or \`ktx status\`.`; + return { + message: `Codex authentication is not usable. ${fix} Details: ${message}`, + fix, + }; +} + +export async function runCodexAuthProbe(input: { + projectDir: string; + model: string; + runner?: CodexSdkRunner; +}): Promise<{ ok: true } | { ok: false; message: string; fix: string }> { + let model: string; + try { + model = resolveCodexModel(input.model); + } catch (error) { + return { + ok: false, + message: error instanceof Error ? error.message : String(error), + fix: 'Set llm.models.default in ktx.yaml to a supported codex model (codex, default, or a gpt-* / codex-* id), or rerun `ktx setup`.', + }; + } + + const runtime = new CodexKtxLlmRuntime({ + projectDir: input.projectDir, + modelSlots: { default: model }, + ...(input.runner ? { runner: input.runner } : {}), + }); + try { + await runtime.generateText({ role: 'default', prompt: 'Reply with exactly: ok' }); + return { ok: true }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { ok: false, ...describeCodexProbeFailure(model, message) }; + } +} diff --git a/packages/cli/src/context/llm/codex-sdk-runner.ts b/packages/cli/src/context/llm/codex-sdk-runner.ts new file mode 100644 index 00000000..58170b3a --- /dev/null +++ b/packages/cli/src/context/llm/codex-sdk-runner.ts @@ -0,0 +1,96 @@ +import { Codex, type CodexOptions, type ThreadOptions, type TurnOptions } from '@openai/codex-sdk'; + +export interface CodexSdkRunnerInput { + projectDir: string; + model: string; + prompt: string; + configOverrides?: Record; + env?: Record; + outputSchema?: Record; + signal?: AbortSignal; +} + +export interface CodexSdkRunner { + runStreamed(input: CodexSdkRunnerInput): Promise>; +} + +type CodexThread = { + runStreamed(input: string, turnOptions?: TurnOptions): Promise<{ events: AsyncIterable }>; +}; + +type CodexClient = { + startThread(options: ThreadOptions): CodexThread; +}; + +type CodexConstructor = new (options?: CodexOptions) => CodexClient; + +export interface CodexSdkCliRunnerOptions { + envBase?: NodeJS.ProcessEnv; + codexPathOverride?: string; +} + +const CODEX_ENV_ALLOWLIST = new Set([ + 'HOME', + 'USERPROFILE', + 'APPDATA', + 'LOCALAPPDATA', + 'XDG_CONFIG_HOME', + 'CODEX_HOME', + 'CODEX_API_KEY', + 'OPENAI_API_KEY', + 'PATH', + 'Path', + 'SYSTEMROOT', + 'COMSPEC', + 'TMPDIR', + 'TMP', + 'TEMP', + 'SSL_CERT_FILE', + 'SSL_CERT_DIR', + 'NODE_EXTRA_CA_CERTS', + 'HTTPS_PROXY', + 'HTTP_PROXY', + 'ALL_PROXY', + 'NO_PROXY', +]); + +function buildCodexSdkEnv(baseEnv: NodeJS.ProcessEnv, overrides: Record | undefined): Record { + const env: Record = {}; + for (const key of CODEX_ENV_ALLOWLIST) { + const value = baseEnv[key]; + if (typeof value === 'string') { + env[key] = value; + } + } + return { ...env, ...(overrides ?? {}) }; +} + +export class CodexSdkCliRunner implements CodexSdkRunner { + constructor(private readonly options: CodexSdkCliRunnerOptions = {}) {} + + async runStreamed(input: CodexSdkRunnerInput): Promise> { + const CodexClass = Codex as CodexConstructor; + const codex = new CodexClass({ + ...(input.configOverrides ? { config: input.configOverrides as CodexOptions['config'] } : {}), + env: buildCodexSdkEnv(this.options.envBase ?? process.env, input.env), + ...(this.options.codexPathOverride ? { codexPathOverride: this.options.codexPathOverride } : {}), + }); + const thread = codex.startThread({ + workingDirectory: input.projectDir, + skipGitRepoCheck: true, + model: input.model, + sandboxMode: 'read-only', + webSearchMode: 'disabled', + approvalPolicy: 'never', + }); + const turnOptions: TurnOptions = { + ...(input.outputSchema ? { outputSchema: input.outputSchema } : {}), + ...(input.signal ? { signal: input.signal } : {}), + }; + const streamed = await thread.runStreamed( + input.prompt, + Object.keys(turnOptions).length > 0 ? turnOptions : undefined, + ); + return streamed.events; + } +} diff --git a/packages/cli/src/context/llm/local-config.ts b/packages/cli/src/context/llm/local-config.ts index c64a85cf..4c2502d1 100644 --- a/packages/cli/src/context/llm/local-config.ts +++ b/packages/cli/src/context/llm/local-config.ts @@ -5,15 +5,29 @@ import { resolveKtxConfigReference } from '../core/config-reference.js'; import type { KtxProjectEmbeddingConfig, KtxProjectLlmConfig } from '../project/config.js'; import { AiSdkKtxLlmRuntime } from './ai-sdk-runtime.js'; import { ClaudeCodeKtxLlmRuntime } from './claude-code-runtime.js'; +import { CodexKtxLlmRuntime } from './codex-runtime.js'; +import type { RateLimitGovernor } from './rate-limit-governor.js'; import type { KtxLlmRuntimePort } from './runtime-port.js'; +type ClaudeCodeRuntimeDeps = ConstructorParameters[0] & { + rateLimitGovernor?: RateLimitGovernor; +}; +type CodexRuntimeDeps = ConstructorParameters[0] & { + rateLimitGovernor?: RateLimitGovernor; +}; +type AiSdkRuntimeDeps = ConstructorParameters[0] & { + rateLimitGovernor?: RateLimitGovernor; +}; + interface LocalConfigDeps { env?: NodeJS.ProcessEnv; projectDir?: string; + rateLimitGovernor?: RateLimitGovernor; createKtxLlmProvider?: typeof createKtxLlmProvider; createKtxEmbeddingProvider?: typeof createKtxEmbeddingProvider; - createClaudeCodeRuntime?: (deps: ConstructorParameters[0]) => KtxLlmRuntimePort; - createAiSdkRuntime?: (deps: { llmProvider: KtxLlmProvider }) => KtxLlmRuntimePort; + createClaudeCodeRuntime?: (deps: ClaudeCodeRuntimeDeps) => KtxLlmRuntimePort; + createCodexRuntime?: (deps: CodexRuntimeDeps) => KtxLlmRuntimePort; + createAiSdkRuntime?: (deps: AiSdkRuntimeDeps) => KtxLlmRuntimePort; } function resolveOptional(value: string | undefined, env: NodeJS.ProcessEnv): string | undefined { @@ -104,7 +118,7 @@ export function createLocalKtxLlmProviderFromConfig( deps: LocalConfigDeps = {}, ): KtxLlmProvider | null { const resolved = resolveLocalKtxLlmConfig(config, deps.env ?? process.env); - if (!resolved || resolved.backend === 'claude-code') { + if (!resolved || resolved.backend === 'claude-code' || resolved.backend === 'codex') { return null; } return (deps.createKtxLlmProvider ?? createKtxLlmProvider)(resolved); @@ -127,10 +141,25 @@ export function createLocalKtxLlmRuntimeFromConfig( projectDir, modelSlots: resolved.modelSlots, env: deps.env, + rateLimitGovernor: deps.rateLimitGovernor, + }); + } + if (resolved.backend === 'codex') { + const projectDir = deps.projectDir; + if (!projectDir) { + throw new Error('projectDir is required when creating the codex LLM runtime'); + } + return (deps.createCodexRuntime ?? ((runtimeDeps) => new CodexKtxLlmRuntime(runtimeDeps)))({ + projectDir, + modelSlots: resolved.modelSlots, + rateLimitGovernor: deps.rateLimitGovernor, }); } const llmProvider = (deps.createKtxLlmProvider ?? createKtxLlmProvider)(resolved); - return (deps.createAiSdkRuntime ?? ((runtimeDeps) => new AiSdkKtxLlmRuntime(runtimeDeps)))({ llmProvider }); + return (deps.createAiSdkRuntime ?? ((runtimeDeps) => new AiSdkKtxLlmRuntime(runtimeDeps)))({ + llmProvider, + rateLimitGovernor: deps.rateLimitGovernor, + }); } export function resolveLocalKtxEmbeddingConfig( diff --git a/packages/cli/src/context/llm/rate-limit-governor.ts b/packages/cli/src/context/llm/rate-limit-governor.ts new file mode 100644 index 00000000..909e4c44 --- /dev/null +++ b/packages/cli/src/context/llm/rate-limit-governor.ts @@ -0,0 +1,387 @@ +import { createAbortError, throwIfAborted } from '../core/abort.js'; + +export type RateLimitProvider = 'claude-subscription' | 'anthropic-api' | 'vertex' | 'codex'; +type RateLimitSignalStatus = 'allowed' | 'warning' | 'rejected'; + +export interface RateLimitSignal { + provider: RateLimitProvider; + status: RateLimitSignalStatus; + resetAtMs?: number; + retryAfterMs?: number; + utilization?: number; + rateLimitType?: string; +} + +export interface RateLimitRetryConfig { + maxAttempts: number; + baseDelayMs: number; + maxDelayMs: number; + jitter: boolean; +} + +export interface RateLimitGovernorConfig { + enabled: boolean; + maxConcurrency: number; + throttleThreshold: number; + minConcurrencyUnderPressure: number; + maxWaitMs?: number; + waitStateTickMs: number; + retry: RateLimitRetryConfig; +} + +export type RateLimitWaitState = + | { + kind: 'rate_limit_observed'; + provider: RateLimitProvider; + status: RateLimitSignalStatus; + rateLimitType?: string; + resetAtMs?: number; + retryAfterMs?: number; + utilization?: number; + } + | { + kind: 'concurrency_adjusted'; + provider: RateLimitProvider; + from: number; + to: number; + reason: string; + rateLimitType?: string; + utilization?: number; + } + | { + kind: 'wait_started' | 'wait_tick' | 'wait_finished'; + provider: RateLimitProvider; + rateLimitType?: string; + resumeAtMs: number; + remainingMs: number; + }; + +export interface RateLimitGovernorDeps { + now?: () => number; + sleep?: (ms: number, signal?: AbortSignal) => Promise; + random?: () => number; +} + +export type RateLimitRelease = () => void; +type Subscriber = (state: RateLimitWaitState) => void; + +const defaultSleep = (ms: number, signal?: AbortSignal): Promise => + new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(createAbortError()); + return; + } + const timeout = setTimeout(resolve, ms); + signal?.addEventListener( + 'abort', + () => { + clearTimeout(timeout); + reject(createAbortError()); + }, + { once: true }, + ); + }); + +export function createRateLimitGovernorConfig( + input: Partial & { retry?: Partial } = {}, +): RateLimitGovernorConfig { + return { + enabled: input.enabled ?? true, + maxConcurrency: input.maxConcurrency ?? 1, + throttleThreshold: input.throttleThreshold ?? 0.8, + minConcurrencyUnderPressure: input.minConcurrencyUnderPressure ?? 1, + ...(input.maxWaitMs !== undefined ? { maxWaitMs: input.maxWaitMs } : {}), + waitStateTickMs: input.waitStateTickMs ?? 1_000, + retry: { + maxAttempts: input.retry?.maxAttempts ?? 6, + baseDelayMs: input.retry?.baseDelayMs ?? 1_000, + maxDelayMs: input.retry?.maxDelayMs ?? 60_000, + jitter: input.retry?.jitter ?? true, + }, + }; +} + +export class RateLimitGovernor { + private readonly now: () => number; + private readonly sleep: (ms: number, signal?: AbortSignal) => Promise; + private readonly random: () => number; + private readonly subscribers = new Set(); + private waiters: Array<() => void> = []; + private active = 0; + private effectiveLimit: number; + private pausedUntilMs: number | null = null; + private pausedProvider: RateLimitProvider | null = null; + private pausedRateLimitType: string | undefined; + private pausedTickMs: number | null = null; + private opaqueAttempts = new Map(); + private pauseGeneration = 0; + private visibleWaitAbort: AbortController | null = null; + + constructor( + private readonly config: RateLimitGovernorConfig, + deps: RateLimitGovernorDeps = {}, + ) { + this.now = deps.now ?? Date.now; + this.sleep = deps.sleep ?? defaultSleep; + this.random = deps.random ?? Math.random; + this.effectiveLimit = Math.max(1, config.maxConcurrency); + } + + currentLimit(): number { + return this.config.enabled ? this.effectiveLimit : this.config.maxConcurrency; + } + + /** + * Total attempts a runtime should make for a single rate-limited LLM call, + * including the first try. Returns 1 (no outer retry) when pacing is disabled: + * the outer retry loop only exists to cooperate with this governor's pause, so + * without active pacing there is no backoff to apply and the backend's own + * retry handles transient rejections. + */ + maxRetryAttempts(): number { + return this.config.enabled ? Math.max(1, this.config.retry.maxAttempts) : 1; + } + + activeSlots(): number { + return this.active; + } + + subscribe(cb: Subscriber): () => void { + this.subscribers.add(cb); + if (this.pausedUntilMs !== null) { + this.startVisibleWaitTicker(); + } + return () => { + this.subscribers.delete(cb); + if (this.subscribers.size === 0) { + this.stopVisibleWaitTicker(); + this.wakeWaiters(); + } + }; + } + + report(signal: RateLimitSignal): void { + if (!this.config.enabled) { + return; + } + this.emit({ + kind: 'rate_limit_observed', + provider: signal.provider, + status: signal.status, + ...(signal.rateLimitType ? { rateLimitType: signal.rateLimitType } : {}), + ...(signal.resetAtMs !== undefined ? { resetAtMs: signal.resetAtMs } : {}), + ...(signal.retryAfterMs !== undefined ? { retryAfterMs: signal.retryAfterMs } : {}), + ...(signal.utilization !== undefined ? { utilization: signal.utilization } : {}), + }); + + if (signal.status === 'rejected') { + this.applyPause(signal); + return; + } + + if (signal.status === 'warning' || (signal.utilization ?? 0) >= this.config.throttleThreshold) { + this.adjustLimit(Math.max(1, this.config.minConcurrencyUnderPressure), signal, 'provider pressure'); + return; + } + + this.opaqueAttempts.delete(signal.provider); + if ((signal.utilization ?? 0) < this.config.throttleThreshold) { + this.adjustLimit(Math.max(1, this.config.maxConcurrency), signal, 'provider recovered'); + } + } + + async waitForReady(signal?: AbortSignal): Promise { + throwIfAborted(signal); + if (!this.config.enabled) { + return; + } + await this.waitForPause(signal); + throwIfAborted(signal); + } + + async acquireWorkSlot(signal?: AbortSignal): Promise { + throwIfAborted(signal); + if (!this.config.enabled) { + this.active += 1; + return () => { + this.active -= 1; + }; + } + + while (true) { + throwIfAborted(signal); + await this.waitForPause(signal); + throwIfAborted(signal); + if (this.active < this.effectiveLimit) { + this.active += 1; + let released = false; + return () => { + if (released) return; + released = true; + this.active -= 1; + this.wakeWaiters(); + }; + } + await this.waitForSlot(signal); + } + } + + private applyPause(signal: RateLimitSignal): void { + const resumeAtMs = this.resumeAtMsFor(signal); + const boundedResumeAtMs = + this.config.maxWaitMs === undefined ? resumeAtMs : Math.min(resumeAtMs, this.now() + this.config.maxWaitMs); + if (this.pausedUntilMs === null || boundedResumeAtMs > this.pausedUntilMs) { + this.pausedUntilMs = boundedResumeAtMs; + this.pausedProvider = signal.provider; + this.pausedRateLimitType = signal.rateLimitType; + this.pausedTickMs = signal.rateLimitType === 'opaque' ? Math.max(1, boundedResumeAtMs - this.now()) : null; + this.emitWait('wait_started'); + this.startVisibleWaitTicker(); + this.wakeWaiters(); + } + this.adjustLimit(Math.max(1, this.config.minConcurrencyUnderPressure), signal, 'provider rejected'); + } + + private resumeAtMsFor(signal: RateLimitSignal): number { + if (signal.resetAtMs !== undefined) { + return signal.resetAtMs; + } + if (signal.retryAfterMs !== undefined) { + return this.now() + signal.retryAfterMs; + } + const attempts = this.opaqueAttempts.get(signal.provider) ?? 0; + this.opaqueAttempts.set(signal.provider, Math.min(attempts + 1, this.config.retry.maxAttempts)); + const base = Math.min( + this.config.retry.maxDelayMs, + this.config.retry.baseDelayMs * 2 ** Math.min(attempts, this.config.retry.maxAttempts - 1), + ); + const jitterMultiplier = this.config.retry.jitter ? 0.75 + this.random() * 0.5 : 1; + return this.now() + Math.round(base * jitterMultiplier); + } + + private adjustLimit(to: number, signal: RateLimitSignal, reason: string): void { + const bounded = Math.max(1, Math.min(this.config.maxConcurrency, to)); + if (bounded === this.effectiveLimit) { + return; + } + const from = this.effectiveLimit; + this.effectiveLimit = bounded; + this.emit({ + kind: 'concurrency_adjusted', + provider: signal.provider, + from, + to: bounded, + reason, + ...(signal.rateLimitType ? { rateLimitType: signal.rateLimitType } : {}), + ...(signal.utilization !== undefined ? { utilization: signal.utilization } : {}), + }); + this.wakeWaiters(); + } + + private startVisibleWaitTicker(): void { + if (this.subscribers.size === 0 || this.pausedUntilMs === null) { + return; + } + this.stopVisibleWaitTicker(); + const generation = (this.pauseGeneration += 1); + const controller = new AbortController(); + this.visibleWaitAbort = controller; + void this.runVisibleWaitTicker(generation, controller.signal).catch(() => undefined); + } + + private stopVisibleWaitTicker(): void { + this.visibleWaitAbort?.abort(); + this.visibleWaitAbort = null; + } + + private async runVisibleWaitTicker(generation: number, signal: AbortSignal): Promise { + while (!signal.aborted && generation === this.pauseGeneration && this.pausedUntilMs !== null) { + const remainingMs = this.pausedUntilMs - this.now(); + if (remainingMs <= 0) { + this.finishPause(generation); + return; + } + this.emitWait('wait_tick'); + await this.sleep(Math.min(this.pausedTickMs ?? this.config.waitStateTickMs, remainingMs), signal); + } + } + + private finishPause(generation?: number): void { + if (generation !== undefined && generation !== this.pauseGeneration) { + return; + } + this.emitWait('wait_finished'); + this.pausedUntilMs = null; + this.pausedProvider = null; + this.pausedRateLimitType = undefined; + this.pausedTickMs = null; + this.stopVisibleWaitTicker(); + this.wakeWaiters(); + } + + private async waitForPause(signal?: AbortSignal): Promise { + throwIfAborted(signal); + while (this.pausedUntilMs !== null) { + const remainingMs = this.pausedUntilMs - this.now(); + if (remainingMs <= 0) { + this.finishPause(); + return; + } + if (this.visibleWaitAbort !== null) { + await this.waitForSlot(signal); + } else { + await this.sleep(Math.min(this.pausedTickMs ?? this.config.waitStateTickMs, remainingMs), signal); + } + throwIfAborted(signal); + } + } + + private waitForSlot(signal?: AbortSignal): Promise { + if (signal?.aborted) { + return Promise.reject(createAbortError()); + } + return new Promise((resolve, reject) => { + const wake = () => { + cleanup(); + resolve(); + }; + const onAbort = () => { + cleanup(); + reject(createAbortError()); + }; + const cleanup = () => { + this.waiters = this.waiters.filter((candidate) => candidate !== wake); + signal?.removeEventListener('abort', onAbort); + }; + this.waiters.push(wake); + signal?.addEventListener('abort', onAbort, { once: true }); + }); + } + + private wakeWaiters(): void { + const waiters = this.waiters; + this.waiters = []; + for (const waiter of waiters) { + waiter(); + } + } + + private emitWait(kind: Extract): void { + if (this.pausedUntilMs === null || this.pausedProvider === null) { + return; + } + this.emit({ + kind, + provider: this.pausedProvider, + ...(this.pausedRateLimitType ? { rateLimitType: this.pausedRateLimitType } : {}), + resumeAtMs: this.pausedUntilMs, + remainingMs: Math.max(0, this.pausedUntilMs - this.now()), + }); + } + + private emit(state: RateLimitWaitState): void { + for (const subscriber of this.subscribers) { + subscriber(state); + } + } +} diff --git a/packages/cli/src/context/llm/runtime-port.ts b/packages/cli/src/context/llm/runtime-port.ts index c1f5ca10..9fec6208 100644 --- a/packages/cli/src/context/llm/runtime-port.ts +++ b/packages/cli/src/context/llm/runtime-port.ts @@ -23,6 +23,24 @@ export interface RunLoopStepInfo { stepBudget: number; } +export interface LlmTokenUsage { + inputTokens?: number; + outputTokens?: number; + totalTokens?: number; +} + +/** Timing and token metrics for a multi-step agent loop, used for ingest profiling. */ +export interface RunLoopMetrics { + /** Wall-clock time around the whole `generateText` call, in milliseconds. */ + totalMs: number; + /** Aggregate token usage across all steps. */ + usage: LlmTokenUsage; + /** Number of agent steps (model round-trips) that actually ran. */ + stepCount: number; + /** Wall-clock offset (ms from loop start) at which each step finished. */ + stepBoundariesMs: number[]; +} + export interface RunLoopParams { modelRole: KtxModelRole; systemPrompt: string; @@ -31,11 +49,13 @@ export interface RunLoopParams { stepBudget: number; telemetryTags: Record; onStepFinish?: (info: RunLoopStepInfo) => void | Promise; + abortSignal?: AbortSignal; } export interface RunLoopResult { stopReason: RunLoopStopReason; error?: Error; + metrics?: RunLoopMetrics; } export interface KtxGenerateTextInput { @@ -44,6 +64,8 @@ export interface KtxGenerateTextInput { system?: string; tools?: KtxRuntimeToolSet; temperature?: number; + onMetrics?: (metrics: { totalMs: number; usage: LlmTokenUsage }) => void; + abortSignal?: AbortSignal; } export interface KtxGenerateObjectInput> { @@ -53,6 +75,8 @@ export interface KtxGenerateObjectInput void; + abortSignal?: AbortSignal; } export interface KtxLlmRuntimePort { diff --git a/packages/cli/src/context/mcp/context-tools.ts b/packages/cli/src/context/mcp/context-tools.ts index 963ab44f..2d07d121 100644 --- a/packages/cli/src/context/mcp/context-tools.ts +++ b/packages/cli/src/context/mcp/context-tools.ts @@ -3,15 +3,23 @@ import type { ToolAnnotations } from '@modelcontextprotocol/sdk/types.js'; import { z } from 'zod'; import type { KtxCliIo } from '../../cli-runtime.js'; import type { MemoryAgentInput } from '../../context/memory/types.js'; -import { emitTelemetryEvent, mcpTelemetrySampleRate, shouldEmitMcpTelemetry } from '../../telemetry/index.js'; +import { + emitTelemetryEvent, + mcpTelemetrySampleRate, + reportException, + shouldEmitMcpTelemetry, +} from '../../telemetry/index.js'; +import { collectTelemetryRedactionSecrets } from '../../telemetry/redaction-secrets.js'; import { scrubErrorClass } from '../../telemetry/scrubber.js'; import type { + KtxMcpClientInfo, KtxMcpContextPorts, KtxMcpProgressCallback, KtxMcpServerLike, KtxMcpToolHandlerContext, KtxMcpToolResult, KtxMcpUserContext, + KtxSemanticLayerQueryResponse, NonArrayObject, } from './types.js'; @@ -21,6 +29,7 @@ export interface RegisterKtxContextToolsDeps { userContext: KtxMcpUserContext; projectDir?: string; io?: KtxCliIo; + getClientInfo?: () => KtxMcpClientInfo | undefined; } const connectionIdSchema = z.string().min(1); @@ -54,13 +63,13 @@ const toolDescriptions = { 'Search KTX wiki pages for reusable business context. Example: wiki_search({ query: "revenue recognition", limit: 5 }).', wiki_read: 'Read a KTX wiki page by key returned from wiki_search. Example: wiki_read({ key: "global/revenue" }).', entity_details: - 'Read table and column metadata from the latest live-database scan snapshot. Example: entity_details({ connectionId: "warehouse", entities: [{ table: { schema: "public", table: "orders" }, columns: ["id"] }] }).', + 'Read table and column metadata from the latest live-database scan snapshot. Example: entity_details({ connectionId: "warehouse", entities: [{ table: { catalog: null, db: "public", name: "orders" }, columns: ["id"] }] }).', dictionary_search: 'Search profile-sampled warehouse values to locate likely source columns for business values. Example: dictionary_search({ values: ["Acme Corp"], connectionId: "warehouse" }).', sl_read_source: 'Read a semantic-layer YAML source by connection id and source name. Example: sl_read_source({ connectionId: "warehouse", sourceName: "orders" }).', sl_query: - 'Execute a semantic-layer query and return rows, headers, generated SQL, and plan details. Example: sl_query({ connectionId: "warehouse", measures: ["orders.order_count"], dimensions: [{ dimension: "orders.created_at", granularity: "month" }] }).', + 'Execute a semantic-layer query and return headers, rows, and total row count, plus correctness notes (e.g. compile-only or fan-out) when relevant. The generated SQL and full query plan are omitted by default; request them with include: ["sql"] and/or include: ["plan"]. Example: sl_query({ connectionId: "warehouse", measures: ["orders.order_count"], dimensions: [{ field: "orders.created_at", granularity: "month" }], include: ["sql"] }).', sql_execution: 'Execute one parser-validated read-only SQL query against a configured KTX connection. Example: sql_execution({ connectionId: "warehouse", sql: "select count(*) from public.orders", maxRows: 100 }).', memory_ingest: @@ -73,7 +82,7 @@ const connectionListSchema = z.object({}); const knowledgeSearchSchema = z.object({ query: z.string().min(1).describe('Natural-language wiki search query, e.g. "revenue recognition policy".'), - limit: z.number().int().min(1).max(50).default(10).describe('Maximum wiki pages to return. Defaults to 10.'), + limit: z.number().int().min(1).max(50).default(10).describe('Maximum wiki pages to return.'), }); const knowledgeReadSchema = z.object({ @@ -93,56 +102,24 @@ const slQueryMeasureSchema = z.union([ }), ]); -const slQueryDimensionSchema = z.preprocess( - (value) => { - if (typeof value === 'string') return { field: value }; - if (value && typeof value === 'object' && !Array.isArray(value)) { - const obj = { ...(value as Record) }; - if (!('field' in obj) && typeof obj.dimension === 'string') obj.field = obj.dimension; - return obj; - } - return value; - }, - z.object({ +const slQueryDimensionSchema = z.object({ field: z.string().min(1).describe('Dimension to group by, e.g. "orders.created_at" or "orders.status".'), granularity: z .string() .min(1) .optional() .describe('Time grain for time dimensions: day, week, month, quarter, or year.'), - }), -); + }); -const slQueryOrderBySchema = z.preprocess( - (value) => { - if (typeof value === 'string') { - return { field: value }; - } - if (value && typeof value === 'object' && !Array.isArray(value)) { - const obj = { ...(value as Record) }; - if (!('field' in obj) && typeof obj.id === 'string') { - obj.field = obj.id; - } - if (!('direction' in obj) && 'desc' in obj) { - obj.direction = obj.desc === true ? 'desc' : 'asc'; - } - return obj; - } - return value; - }, - z.object({ +const slQueryOrderBySchema = z.object({ field: z .string() .min(1) .describe( 'Field/measure/dimension id to order by, e.g. "orders.created_at", a dimension key like "mart_nrr_quarterly.quarter_label", or a measure alias.', ), - direction: z - .enum(['asc', 'desc']) - .default('asc') - .describe('Sort direction: "asc" or "desc". Defaults to "asc".'), - }), -); + direction: z.enum(['asc', 'desc']).default('asc').describe('Sort direction for this field.'), + }); const slQuerySchema = z.object({ connectionId: connectionIdSchema @@ -152,7 +129,7 @@ const slQuerySchema = z.object({ dimensions: z .array(slQueryDimensionSchema) .default([]) - .describe('Dimensions to group by. Strings and {dimension, granularity} are accepted.'), + .describe('Dimensions to group by. Use {field, granularity?} entries.'), filters: z .array(z.string().describe('Semantic-layer filter expression, e.g. "orders.status = paid".')) .default([]) @@ -164,28 +141,20 @@ const slQuerySchema = z.object({ order_by: z .array(slQueryOrderBySchema) .default([]) - .describe('Sort clauses. Strings and Cube-style {id, desc} are accepted.'), - limit: z.number().int().min(0).default(1000).describe('Maximum rows to return. Defaults to 1000.'), - include_empty: z.boolean().default(true).describe('Whether to include empty dimension groups. Defaults to true.'), + .describe('Sort clauses. Use {field, direction?} entries.'), + limit: z.number().int().min(0).default(1000).describe('Maximum rows to return.'), + include_empty: z.boolean().default(true).describe('Whether to include empty dimension groups.'), + include: z + .array(z.enum(['plan', 'sql'])) + .default([]) + .describe('Extra detail to attach to the response: "sql" for the generated SQL, "plan" for the full query plan.'), }); -const entityDetailsTableRefSchema = z.preprocess( - (value) => { - if (value && typeof value === 'object' && !Array.isArray(value)) { - const obj = { ...(value as Record) }; - if (!('db' in obj) && typeof obj.schema === 'string') obj.db = obj.schema; - if (!('name' in obj) && typeof obj.table === 'string') obj.name = obj.table; - if (!('catalog' in obj)) obj.catalog = null; - return obj; - } - return value; - }, - z.object({ +const entityDetailsTableRefSchema = z.object({ catalog: z.string().nullable().describe('Catalog/project/database. Use null when not applicable.'), db: z.string().nullable().describe('Schema/database/dataset. Use null when not applicable.'), name: z.string().min(1).describe('Table name.'), - }), -); + }); const entityDetailsSchema = z.object({ connectionId: connectionIdSchema.describe('Connection id whose latest scan snapshot should be read.'), @@ -194,7 +163,7 @@ const entityDetailsSchema = z.object({ z.object({ table: z .union([z.string().min(1), entityDetailsTableRefSchema]) - .describe('Table display string or object ref. {schema, table} is accepted as an alias for {db, name}.'), + .describe('Table display string or canonical object ref.'), columns: z .array(z.string().min(1).describe('Column name to inspect.')) .optional() @@ -225,13 +194,13 @@ const discoverDataSchema = z.object({ .optional() .describe('Optional connection id. Pass it when user intent pins a specific warehouse.'), kinds: z.array(discoverDataKindSchema.describe('Reference kind to include.')).optional().describe('Optional kind filter.'), - limit: z.number().int().min(1).max(50).default(15).optional().describe('Maximum refs to return. Defaults to 15.'), + limit: z.number().int().min(1).max(50).default(10).optional().describe('Maximum refs to return.'), }); const sqlExecutionSchema = z.object({ connectionId: connectionIdSchema.describe('Connection id to execute against. Required for raw SQL.'), sql: z.string().min(1).describe('Parser-validated read-only SQL, e.g. "select count(*) from public.orders".'), - maxRows: z.number().int().min(1).max(10_000).default(1000).optional().describe('Maximum rows to return. Defaults to 1000.'), + maxRows: z.number().int().min(1).max(10_000).default(1000).optional().describe('Maximum rows to return.'), }); const memoryIngestSchema = z.object({ @@ -307,10 +276,14 @@ const slReadSourceOutputSchema = z.object({ const slQueryOutputSchema = z.object({ connectionId: z.string().optional(), dialect: z.string().optional(), - sql: z.string(), headers: z.array(z.string()), rows: z.array(z.array(z.unknown())), totalRows: z.number(), + // Correctness signals hoisted out of `plan` so they survive default projection (e.g. compile-only + // status, fan-out warnings). Present only when there is something to report. + notes: z.array(z.string()).optional(), + // Opt-in detail, attached only when requested via the `include` input. + sql: z.string().optional(), plan: unknownRecordSchema.optional(), }); @@ -452,12 +425,59 @@ const memoryIngestStatusOutputSchema = z.object({ /** @internal */ export function jsonToolResult(structuredContent: T): KtxMcpToolResult { + // Compact (non-indented) JSON: this `content` text is the copy the model reads. Pretty-printing + // arrays-of-arrays (every `rows` payload) puts one scalar per line, inflating tabular results by + // a large constant factor. `structuredContent` carries the same data for structured-output clients. return { - content: [{ type: 'text', text: JSON.stringify(structuredContent, null, 2) }], + content: [{ type: 'text', text: JSON.stringify(structuredContent) }], structuredContent, }; } +/** + * Pull the correctness-critical signals out of a query plan so they survive even when the caller + * did not opt into the full `plan`. Returns an empty list when there is nothing to flag. + */ +function slQueryNotes(plan: Record | undefined): string[] { + if (!plan) { + return []; + } + const notes: string[] = []; + const execution = plan.execution; + if ( + execution && + typeof execution === 'object' && + (execution as Record).mode === 'compile_only' + ) { + const reason = (execution as Record).reason; + notes.push(typeof reason === 'string' ? reason : 'Compiled SQL only; no rows were executed.'); + } + if (plan.has_fan_out === true) { + const description = typeof plan.fan_out_description === 'string' ? plan.fan_out_description.trim() : ''; + notes.push(description.length > 0 ? description : 'Fan-out detected: measure totals may be inflated by joins.'); + } + return notes; +} + +/** + * Default sl_query response is the minimum the agent needs to read the result: connection, headers, + * rows, totals, plus any correctness notes. The generated `sql` and the full `plan` are attached only + * when explicitly requested via `include`, since both are large and echo information the caller already has. + */ +function projectSlQueryResult(result: KtxSemanticLayerQueryResponse, include: ('plan' | 'sql')[]) { + const notes = slQueryNotes(result.plan); + return { + ...(result.connectionId !== undefined ? { connectionId: result.connectionId } : {}), + ...(result.dialect !== undefined ? { dialect: result.dialect } : {}), + headers: result.headers, + rows: result.rows, + totalRows: result.totalRows, + ...(notes.length > 0 ? { notes } : {}), + ...(include.includes('sql') ? { sql: result.sql } : {}), + ...(include.includes('plan') && result.plan ? { plan: result.plan } : {}), + }; +} + function jsonErrorToolResult(text: string): KtxMcpToolResult> { return { content: [{ type: 'text', text }], @@ -504,19 +524,49 @@ function registerParsedTool( }, schema: TSchema, handler: (input: z.infer, context?: KtxMcpToolHandlerContext) => Promise, + telemetry?: { projectDir?: string; io?: KtxCliIo }, ): void { server.registerTool(name, config, async (input, context) => { try { return await handler(schema.parse(input), context); } catch (error) { + if (telemetry?.io) { + await reportException({ + error, + context: { source: `mcp:${name}`, handled: true, fatal: false }, + projectDir: telemetry.projectDir, + io: telemetry.io, + redactionSecrets: await collectTelemetryRedactionSecrets({ + projectDir: telemetry.projectDir, + includeLlm: true, + includeEmbeddings: true, + env: process.env, + }), + }); + } return jsonErrorToolResult(formatToolError(error)); } }); } +/** + * Resolves the connected client's identity into the raw telemetry fields. The + * strings are client-controlled and untrusted, so they only ever land in the + * telemetry property bag — never in paths, logs, or error messages. + */ +function clientTelemetryFields( + getClientInfo: (() => KtxMcpClientInfo | undefined) | undefined, +): { mcpClientName?: string; mcpClientVersion?: string } { + const client = getClientInfo?.(); + return { + ...(client?.name ? { mcpClientName: client.name } : {}), + ...(client?.version ? { mcpClientVersion: client.version } : {}), + }; +} + function instrumentMcpServer( server: KtxMcpServerLike, - telemetry: { projectDir?: string; io?: KtxCliIo }, + telemetry: { projectDir?: string; io?: KtxCliIo; getClientInfo?: () => KtxMcpClientInfo | undefined }, ): KtxMcpServerLike { return { registerTool(name, config, handler) { @@ -536,11 +586,26 @@ function instrumentMcpServer( outcome: isError ? 'error' : 'ok', durationMs: Math.max(0, performance.now() - startedAt), sampleRate: mcpTelemetrySampleRate(), + ...clientTelemetryFields(telemetry.getClientInfo), }, }); } return result; } catch (error) { + if (telemetry.io) { + await reportException({ + error, + context: { source: `mcp:${name}`, handled: true, fatal: false }, + projectDir: telemetry.projectDir, + io: telemetry.io, + redactionSecrets: await collectTelemetryRedactionSecrets({ + projectDir: telemetry.projectDir, + includeLlm: true, + includeEmbeddings: true, + env: process.env, + }), + }); + } if (telemetry.io && telemetry.projectDir && shouldEmitMcpTelemetry()) { const errorClass = scrubErrorClass(error); await emitTelemetryEvent({ @@ -553,6 +618,7 @@ function instrumentMcpServer( ...(errorClass ? { errorClass } : {}), durationMs: Math.max(0, performance.now() - startedAt), sampleRate: mcpTelemetrySampleRate(), + ...clientTelemetryFields(telemetry.getClientInfo), }, }); } @@ -565,7 +631,12 @@ function instrumentMcpServer( export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void { const { ports, userContext } = deps; - const server = instrumentMcpServer(deps.server, { projectDir: deps.projectDir, io: deps.io }); + const toolTelemetry = { projectDir: deps.projectDir, io: deps.io }; + const server = instrumentMcpServer(deps.server, { + projectDir: deps.projectDir, + io: deps.io, + getClientInfo: deps.getClientInfo, + }); if (ports.connections) { const connections = ports.connections; @@ -581,6 +652,7 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void }, connectionListSchema, async () => jsonToolResult({ connections: await connections.list() }), + toolTelemetry, ); } @@ -605,6 +677,7 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void limit: input.limit, }), ), + toolTelemetry, ); registerParsedTool( @@ -622,6 +695,7 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void const page = await knowledge.read({ userId: userContext.userId, key: input.key }); return page ? jsonToolResult(page) : jsonErrorToolResult(`Wiki page "${input.key}" was not found.`); }, + toolTelemetry, ); } @@ -644,6 +718,7 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void ? jsonToolResult(source) : jsonErrorToolResult(`Semantic-layer source "${input.sourceName}" was not found.`); }, + toolTelemetry, ); registerParsedTool( @@ -659,24 +734,24 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void slQuerySchema, async (input, context) => { const onProgress = mcpProgressCallback(context); - return jsonToolResult( - await semanticLayer.query( - { - connectionId: input.connectionId, - query: { - measures: input.measures, - dimensions: input.dimensions, - filters: input.filters, - segments: input.segments, - order_by: input.order_by, - limit: input.limit, - include_empty: input.include_empty, - }, + const result = await semanticLayer.query( + { + connectionId: input.connectionId, + query: { + measures: input.measures, + dimensions: input.dimensions, + filters: input.filters, + segments: input.segments, + order_by: input.order_by, + limit: input.limit, + include_empty: input.include_empty, }, - onProgress ? { onProgress } : undefined, - ), + }, + onProgress ? { onProgress } : undefined, ); + return jsonToolResult(projectSlQueryResult(result, input.include)); }, + toolTelemetry, ); } @@ -694,6 +769,7 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void }, entityDetailsSchema, async (input) => jsonToolResult(await entityDetails.read(input)), + toolTelemetry, ); } @@ -711,6 +787,7 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void }, dictionarySearchSchema, async (input) => jsonToolResult(await dictionarySearch.search(input)), + toolTelemetry, ); } @@ -728,6 +805,7 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void }, discoverDataSchema, async (input) => jsonToolResult({ refs: await discover.search(input) }), + toolTelemetry, ); } @@ -757,6 +835,7 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void ), ); }, + toolTelemetry, ); } @@ -784,6 +863,7 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void }; return jsonToolResult(await memoryIngest.ingest(ingestInput)); }, + toolTelemetry, ); registerParsedTool( @@ -801,6 +881,7 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void const status = await memoryIngest.status(input.runId); return status ? jsonToolResult(status) : jsonErrorToolResult(`Memory ingest run "${input.runId}" was not found.`); }, + toolTelemetry, ); } } diff --git a/packages/cli/src/context/mcp/local-project-ports.ts b/packages/cli/src/context/mcp/local-project-ports.ts index dbbb36d2..4c820b14 100644 --- a/packages/cli/src/context/mcp/local-project-ports.ts +++ b/packages/cli/src/context/mcp/local-project-ports.ts @@ -24,17 +24,14 @@ interface CreateLocalProjectMcpContextPortsOptions { function dialectForDriver(driver: string | undefined): string { const normalized = (driver ?? 'postgres').toUpperCase(); const map: Record = { - POSTGRESQL: 'postgres', POSTGRES: 'postgres', BIGQUERY: 'bigquery', SNOWFLAKE: 'snowflake', MYSQL: 'mysql', SQLSERVER: 'tsql', - MSSQL: 'tsql', SQLITE: 'sqlite', DUCKDB: 'duckdb', CLICKHOUSE: 'clickhouse', - REDSHIFT: 'redshift', DATABRICKS: 'databricks', }; return map[normalized] ?? 'postgres'; diff --git a/packages/cli/src/context/mcp/server.ts b/packages/cli/src/context/mcp/server.ts index 97d79525..85871467 100644 --- a/packages/cli/src/context/mcp/server.ts +++ b/packages/cli/src/context/mcp/server.ts @@ -11,6 +11,7 @@ export function createKtxMcpServer(deps: KtxMcpServerDeps): KtxMcpServerDeps['se userContext: deps.userContext, projectDir: deps.projectDir, io: deps.io, + getClientInfo: deps.getClientInfo, }); } @@ -30,6 +31,9 @@ export function createDefaultKtxMcpServer( contextTools: deps.contextTools, projectDir: deps.projectDir, io: deps.io, + // The SDK populates the client identity after the initialize handshake, so + // read it lazily at emit time rather than at registration (undefined here). + getClientInfo: () => server.server.getClientVersion(), }); return server; } diff --git a/packages/cli/src/context/mcp/types.ts b/packages/cli/src/context/mcp/types.ts index 29a8c069..3694e3d6 100644 --- a/packages/cli/src/context/mcp/types.ts +++ b/packages/cli/src/context/mcp/types.ts @@ -50,6 +50,16 @@ export interface KtxMcpUserContext { userId: string; } +/** + * Identity of the connected MCP client tool (e.g. Claude Desktop, Cursor), + * read from the initialize handshake. Untrusted, client-controlled strings — + * use only as telemetry properties, never to build paths or log lines. + */ +export interface KtxMcpClientInfo { + name: string; + version: string; +} + export interface KtxMcpServerLike { registerTool( name: string, @@ -110,7 +120,10 @@ interface KtxSemanticLayerReadResponse { yaml: string; } -interface KtxSemanticLayerQueryResponse { +/** @internal */ +export interface KtxSemanticLayerQueryResponse { + connectionId?: string; + dialect?: string; sql: string; headers: string[]; rows: unknown[][]; @@ -174,4 +187,6 @@ export interface KtxMcpServerDeps { contextTools?: KtxMcpContextPorts; projectDir?: string; io?: KtxCliIo; + /** Reads the connected client's identity once the initialize handshake completes. */ + getClientInfo?: () => KtxMcpClientInfo | undefined; } diff --git a/packages/cli/src/context/project/config.ts b/packages/cli/src/context/project/config.ts index cd0eaa4a..fd7f482c 100644 --- a/packages/cli/src/context/project/config.ts +++ b/packages/cli/src/context/project/config.ts @@ -3,7 +3,7 @@ import YAML from 'yaml'; import * as z from 'zod'; import { connectionConfigSchema } from './driver-schemas.js'; -const KTX_LLM_BACKENDS = ['none', 'anthropic', 'vertex', 'gateway', 'claude-code'] as const; +const KTX_LLM_BACKENDS = ['none', 'anthropic', 'vertex', 'gateway', 'claude-code', 'codex'] as const; const KTX_EMBEDDING_BACKENDS = ['none', 'openai', 'sentence-transformers'] as const; const KTX_PROMPT_CACHE_TTLS = ['5m', '1h'] as const; const KTX_ENRICHMENT_MODES = ['none', 'deterministic', 'llm'] as const; @@ -38,7 +38,7 @@ const llmProviderSchema = z .enum(KTX_LLM_BACKENDS) .default('none') .describe( - 'LLM provider backend. "none" disables LLM features; "anthropic" / "vertex" / "gateway" require the matching nested credentials block; "claude-code" uses the local Claude Code session.', + 'LLM provider backend. "none" disables LLM features; "anthropic" / "vertex" / "gateway" require the matching nested credentials block; "claude-code" uses the local Claude Code session; "codex" uses the local Codex session.', ), vertex: vertexProviderSchema.optional().describe('Vertex AI credentials, used when backend is "vertex".'), anthropic: apiCredentialsSchema.optional().describe('Anthropic API credentials, used when backend is "anthropic".'), @@ -100,6 +100,44 @@ const workUnitsSchema = z }) .describe('Concurrency and failure handling for ingest work units.'); +const ingestRateLimitRetrySchema = z + .strictObject({ + maxAttempts: z + .int() + .positive() + .default(6) + .describe( + 'Maximum attempts for a single rate-limited LLM call before the failure surfaces, counting the first try. Also bounds how far opaque backoff grows for providers that do not expose a reset time.', + ), + baseDelayMs: z.int().positive().default(1_000).describe('Initial opaque retry delay in milliseconds.'), + maxDelayMs: z.int().positive().default(60_000).describe('Maximum opaque retry delay in milliseconds.'), + jitter: z.boolean().default(true).describe('When true, apply bounded jitter to opaque retry delays.'), + }) + .describe('Retry policy for rate-limit responses that do not include a reset time or retry-after value.'); + +const ingestRateLimitSchema = z + .strictObject({ + enabled: z.boolean().default(true).describe('Master switch for ingest LLM rate-limit pacing and visible waits.'), + throttleThreshold: z + .number() + .min(0) + .max(1) + .default(0.8) + .describe('Provider utilization at or above which ingest throttles new work-unit starts.'), + minConcurrencyUnderPressure: z + .int() + .positive() + .default(1) + .describe('Effective work-unit concurrency while a provider is under rate-limit pressure.'), + maxWaitMs: z + .int() + .positive() + .optional() + .describe('Optional cap on a single provider reset wait. Omit to wait indefinitely until the provider reset time.'), + retry: ingestRateLimitRetrySchema.prefault({}).describe('Opaque retry policy for providers without reset hints.'), + }) + .describe('Rate-limit pacing and wait policy for ingest LLM calls.'); + const ingestSchema = z .strictObject({ adapters: z @@ -110,6 +148,13 @@ const ingestSchema = z .prefault({ backend: 'none' }) .describe('Embedding configuration used when ingest adapters need to embed documents.'), workUnits: workUnitsSchema.prefault({}).describe('Concurrency and failure handling for ingest work units.'), + rateLimit: ingestRateLimitSchema.prefault({}).describe('LLM rate-limit pacing and visible-wait policy for ingest.'), + profile: z + .union([z.boolean(), z.literal('json')]) + .default(false) + .describe( + 'Print a timing breakdown to stderr at the end of each ingest run. `true` prints a human table; `"json"` prints the raw structured profile for coding agents; `false` disables it. Equivalent to the KTX_PROFILE_INGEST environment variable (`1`/`true`/`json`).', + ), }) .describe('Ingest pipeline configuration: adapters, embeddings, and work-unit policy.'); diff --git a/packages/cli/src/context/project/driver-schemas.ts b/packages/cli/src/context/project/driver-schemas.ts index 3d6d1a84..f9a3639f 100644 --- a/packages/cli/src/context/project/driver-schemas.ts +++ b/packages/cli/src/context/project/driver-schemas.ts @@ -7,7 +7,6 @@ import { const warehouseDrivers = [ 'postgres', - 'postgresql', 'mysql', 'snowflake', 'bigquery', @@ -31,7 +30,7 @@ function warehouseConnectionSchema(driver: .array(z.string().min(1)) .optional() .describe( - 'Optional allowlist of fully-qualified table names ("schema.table") to ingest. When set, live-database ingest discards any table whose schema-qualified name is not in this list. Useful for smoke-testing deep ingest on a single table.', + 'Optional allowlist of fully-qualified table names ("schema.table") to ingest. When set, live-database ingest discards any table whose schema-qualified name is not in this list. Useful for smoke-testing ingest on a single table.', ), }) .describe( @@ -41,7 +40,6 @@ function warehouseConnectionSchema(driver: const warehouseConnectionSchemas = [ warehouseConnectionSchema('postgres'), - warehouseConnectionSchema('postgresql'), warehouseConnectionSchema('mysql'), warehouseConnectionSchema('snowflake'), warehouseConnectionSchema('bigquery'), diff --git a/packages/cli/src/context/scan/constraint-discovery.ts b/packages/cli/src/context/scan/constraint-discovery.ts new file mode 100644 index 00000000..d58e9053 --- /dev/null +++ b/packages/cli/src/context/scan/constraint-discovery.ts @@ -0,0 +1,42 @@ +import type { KtxScanWarning } from './types.js'; + +export type ConstraintDiscoveryKind = 'primary_key' | 'foreign_key'; + +export interface ConstraintQueryContext { + schema: string; + kind: ConstraintDiscoveryKind; + isDeniedError: (error: unknown) => boolean; +} + +export type ConstraintQueryOutcome = { ok: true; value: T } | { ok: false; warning: KtxScanWarning }; + +export function constraintDiscoveryWarning(input: { + schema: string; + kind: ConstraintDiscoveryKind; +}): KtxScanWarning { + return { + code: 'constraint_discovery_unauthorized', + message: + `Skipped ${input.kind === 'primary_key' ? 'primary-key' : 'foreign-key'} ` + + `discovery in ${input.schema} (insufficient grants on system catalogs)`, + recoverable: true, + metadata: { schema: input.schema, kind: input.kind }, + }; +} + +export async function tryConstraintQuery( + ctx: ConstraintQueryContext, + fn: () => Promise, +): Promise> { + try { + return { ok: true, value: await fn() }; + } catch (error) { + if (!ctx.isDeniedError(error)) { + throw error; + } + return { + ok: false, + warning: constraintDiscoveryWarning({ schema: ctx.schema, kind: ctx.kind }), + }; + } +} diff --git a/packages/cli/src/context/scan/enabled-tables.ts b/packages/cli/src/context/scan/enabled-tables.ts index 327992ac..96c94afd 100644 --- a/packages/cli/src/context/scan/enabled-tables.ts +++ b/packages/cli/src/context/scan/enabled-tables.ts @@ -8,12 +8,8 @@ import type { KtxTableRef } from './types.js'; * * Accepted entry forms: * "catalog.db.name" — fully qualified - * "db.name" — schema-qualified (catalog = null; legacy / Postgres-shape) + * "db.name" — schema-qualified (catalog = null) * "name" — bare (catalog = db = null; SQLite-shape) - * { catalog?, db?, name } — escape hatch for identifiers containing dots - * - * The setup wizard writes the fully-qualified form going forward; the lenient - * parser keeps existing project configs working. */ export function resolveEnabledTables( connection: Record | undefined, @@ -31,22 +27,13 @@ export function resolveEnabledTables( function parseEnabledTableEntry(value: unknown): KtxTableRef | null { if (typeof value === 'string') { - return parseDottedEntry(value); - } - if (value && typeof value === 'object' && !Array.isArray(value)) { - const entry = value as { catalog?: unknown; db?: unknown; name?: unknown }; - const name = typeof entry.name === 'string' ? entry.name : null; - if (!name) return null; - return { - catalog: typeof entry.catalog === 'string' ? entry.catalog : null, - db: typeof entry.db === 'string' ? entry.db : null, - name, - }; + return parseDottedTableEntry(value); } return null; } -function parseDottedEntry(value: string): KtxTableRef | null { +/** @internal */ +export function parseDottedTableEntry(value: string): KtxTableRef | null { const trimmed = value.trim(); if (trimmed.length === 0) return null; const parts = trimmed.split('.'); diff --git a/packages/cli/src/context/scan/entity-details.ts b/packages/cli/src/context/scan/entity-details.ts index 37e766b6..731eea8f 100644 --- a/packages/cli/src/context/scan/entity-details.ts +++ b/packages/cli/src/context/scan/entity-details.ts @@ -1,7 +1,7 @@ import type { KtxLocalProject } from '../../context/project/project.js'; +import { getDialectForDriver, type KtxDialect } from '../connections/dialects.js'; import { readLocalScanStructuralSnapshot } from './local-structural-artifacts.js'; import type { - KtxConnectionDriver, KtxScanReport, KtxSchemaColumn, KtxSchemaSnapshot, @@ -88,59 +88,23 @@ function refsEqual(left: KtxTableRef, right: KtxTableRef): boolean { ); } -function cleanIdentifierPart(part: string): string { - return part.trim().replace(/^["'`\[]|["'`\]]$/g, ''); -} - -function splitDisplay(display: string): string[] { - return display - .trim() - .split('.') - .map(cleanIdentifierPart) - .filter(Boolean); -} - -function displayForTable(driver: KtxConnectionDriver, table: KtxTableRef): string { - if (driver === 'sqlite') { - return table.name; - } - return [table.catalog, table.db, table.name].filter((part): part is string => Boolean(part)).join('.'); -} - function tableRef(table: KtxSchemaTable): KtxTableRef { return { catalog: table.catalog, db: table.db, name: table.name }; } function candidateList( - driver: KtxConnectionDriver, + dialect: KtxDialect, tables: KtxSchemaTable[], ): Array<{ tableRef: KtxTableRef; display: string }> { return tables .map((table) => ({ tableRef: tableRef(table), - display: displayForTable(driver, table), + display: dialect.formatDisplayRef(table), })) .sort((left, right) => left.display.localeCompare(right.display)); } -function parseDisplayRef(driver: KtxConnectionDriver, display: string): KtxTableRef | null { - const parts = splitDisplay(display); - if (driver === 'sqlite') { - return parts.length === 1 ? { catalog: null, db: null, name: parts[0]! } : null; - } - if (driver === 'bigquery' || driver === 'snowflake' || driver === 'sqlserver') { - return parts.length === 3 ? { catalog: parts[0]!, db: parts[1]!, name: parts[2]! } : null; - } - if (parts.length === 2) { - return { catalog: null, db: parts[0]!, name: parts[1]! }; - } - if (parts.length === 3) { - return { catalog: parts[0]!, db: parts[1]!, name: parts[2]! }; - } - return null; -} - -function resolveTable(snapshot: KtxSchemaSnapshot, input: KtxEntityDetailsTableInput): ResolveResult { +function resolveTable(snapshot: KtxSchemaSnapshot, input: KtxEntityDetailsTableInput, dialect: KtxDialect): ResolveResult { if (typeof input !== 'string') { const table = snapshot.tables.find((candidate) => refsEqual(candidate, input)) ?? null; return table @@ -149,13 +113,13 @@ function resolveTable(snapshot: KtxSchemaSnapshot, input: KtxEntityDetailsTableI table: null, error: { code: 'table_not_found', - message: `Table not found in latest scan: ${displayForTable(snapshot.driver, input)}`, - candidates: candidateList(snapshot.driver, snapshot.tables), + message: `Table not found in latest scan: ${dialect.formatDisplayRef(input)}`, + candidates: candidateList(dialect, snapshot.tables), }, }; } - const parsed = parseDisplayRef(snapshot.driver, input); + const parsed = dialect.parseDisplayRef(input); if (parsed) { const table = snapshot.tables.find((candidate) => refsEqual(candidate, parsed)) ?? null; return table @@ -165,7 +129,7 @@ function resolveTable(snapshot: KtxSchemaSnapshot, input: KtxEntityDetailsTableI error: { code: 'table_not_found', message: `Table not found in latest scan: ${input}`, - candidates: candidateList(snapshot.driver, snapshot.tables), + candidates: candidateList(dialect, snapshot.tables), }, }; } @@ -180,7 +144,7 @@ function resolveTable(snapshot: KtxSchemaSnapshot, input: KtxEntityDetailsTableI error: { code: 'ambiguous_table', message: `Table name "${input}" is ambiguous across schemas/catalogs; pass a structured table ref.`, - candidates: candidateList(snapshot.driver, byName), + candidates: candidateList(dialect, byName), }, }; } @@ -189,7 +153,7 @@ function resolveTable(snapshot: KtxSchemaSnapshot, input: KtxEntityDetailsTableI error: { code: 'table_not_found', message: `Table not found in latest scan: ${input}`, - candidates: candidateList(snapshot.driver, snapshot.tables), + candidates: candidateList(dialect, snapshot.tables), }, }; } @@ -261,9 +225,10 @@ export function createKtxEntityDetailsService(project: KtxLocalProject) { } const info = snapshotInfo(scan.report, scan.snapshot); + const dialect = getDialectForDriver(scan.snapshot.driver); const results: KtxEntityDetailsResponse['results'] = []; for (const entity of input.entities) { - const resolved = resolveTable(scan.snapshot, entity.table); + const resolved = resolveTable(scan.snapshot, entity.table, dialect); if (!resolved.table) { results.push({ ok: false, @@ -289,7 +254,7 @@ export function createKtxEntityDetailsService(project: KtxLocalProject) { snapshot: info, error: { code: 'column_not_found', - message: `Column(s) not found on ${displayForTable(scan.snapshot.driver, resolved.table)}: ${missing.join(', ')}`, + message: `Column(s) not found on ${dialect.formatDisplayRef(resolved.table)}: ${missing.join(', ')}`, candidates: resolved.table.columns.map((column) => column.name), }, }); @@ -300,7 +265,7 @@ export function createKtxEntityDetailsService(project: KtxLocalProject) { ok: true, connectionId: input.connectionId, tableRef: tableRef(resolved.table), - display: displayForTable(scan.snapshot.driver, resolved.table), + display: dialect.formatDisplayRef(resolved.table), kind: resolved.table.kind, comment: resolved.table.comment, estimatedRows: resolved.table.estimatedRows, diff --git a/packages/cli/src/context/scan/local-enrichment.ts b/packages/cli/src/context/scan/local-enrichment.ts index 545b2ad6..833cb5b1 100644 --- a/packages/cli/src/context/scan/local-enrichment.ts +++ b/packages/cli/src/context/scan/local-enrichment.ts @@ -1,5 +1,6 @@ import pLimit from 'p-limit'; import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; +import { getDialectForDriver } from '../connections/dialects.js'; import { buildDefaultKtxProjectConfig, type KtxScanRelationshipConfig } from '../project/config.js'; import { KtxDescriptionGenerator } from './description-generation.js'; import { buildKtxColumnEmbeddingText } from './embedding-text.js'; @@ -118,6 +119,18 @@ function targetMatchesForeignKey(table: KtxEnrichedTable, foreignKey: KtxSchemaF ); } +function assertConnectorDriverMatchesSnapshot(input: { + connector: KtxScanConnector; + snapshot: KtxSchemaSnapshot; + connectionId: string; +}): void { + if (input.connector.driver !== input.snapshot.driver) { + throw new Error( + `ktx scan connector driver "${input.connector.driver}" does not match snapshot driver "${input.snapshot.driver}" for connection "${input.connectionId}"`, + ); + } +} + function formalRelationshipsFromSnapshot( snapshot: KtxSchemaSnapshot, tables: readonly KtxEnrichedTable[], @@ -468,6 +481,12 @@ export async function runLocalScanEnrichment( )); await progress?.update(0.05, `Loaded schema snapshot with ${snapshot.tables.length} tables`); + assertConnectorDriverMatchesSnapshot({ + connector: input.connector, + snapshot, + connectionId: input.connectionId, + }); + const dialect = getDialectForDriver(snapshot.driver); const now = input.now ?? (() => new Date()); const state = completedKtxScanEnrichmentStateSummary(); const syncId = input.syncId ?? input.context.runId; @@ -575,7 +594,7 @@ export async function runLocalScanEnrichment( await relationshipProgress?.update(0, 'Detecting relationships'); const detection = await discoverKtxRelationships({ connectionId: input.connectionId, - driver: snapshot.driver, + dialect, connector: input.connector, schema, context: input.context, diff --git a/packages/cli/src/context/scan/local-scan.ts b/packages/cli/src/context/scan/local-scan.ts index cb886991..703ef73f 100644 --- a/packages/cli/src/context/scan/local-scan.ts +++ b/packages/cli/src/context/scan/local-scan.ts @@ -126,19 +126,17 @@ function normalizeDriver(driver: string | undefined): KtxConnectionDriver { const normalized = (driver ?? '').toLowerCase(); if ( normalized === 'postgres' || - normalized === 'postgresql' || normalized === 'sqlite' || - normalized === 'sqlite3' || normalized === 'mysql' || normalized === 'clickhouse' || normalized === 'sqlserver' || normalized === 'bigquery' || normalized === 'snowflake' ) { - return normalized === 'sqlite3' ? 'sqlite' : normalized; + return normalized; } throw new Error( - `Standalone ktx scan supports postgres/postgresql/sqlite/mysql/clickhouse/sqlserver/bigquery/snowflake in this phase, received "${driver ?? 'unknown'}"`, + `Standalone ktx scan supports postgres/sqlite/mysql/clickhouse/sqlserver/bigquery/snowflake in this phase, received "${driver ?? 'unknown'}"`, ); } @@ -469,6 +467,9 @@ export async function runLocalScan(options: RunLocalScanOptions): Promise { return isRecord(value) ? value : {}; } +const scanWarningCodes = new Set([ + 'connector_capability_missing', + 'sampling_failed', + 'statistics_failed', + 'llm_unavailable', + 'embedding_unavailable', + 'scan_enrichment_backend_not_configured', + 'relationship_validation_failed', + 'relationship_llm_invalid_reference', + 'relationship_llm_proposal_failed', + 'credential_redacted', + 'enrichment_failed', + 'description_fallback_used', + 'constraint_discovery_unauthorized', +]); + +function parseWarning(rawWarning: unknown, path: string): KtxScanWarning { + if ( + !isRecord(rawWarning) || + typeof rawWarning.code !== 'string' || + !scanWarningCodes.has(rawWarning.code as KtxScanWarning['code']) || + typeof rawWarning.message !== 'string' || + typeof rawWarning.recoverable !== 'boolean' + ) { + throw new Error(`Invalid KTX schema warning artifact: ${path}`); + } + return { + code: rawWarning.code as KtxScanWarning['code'], + message: rawWarning.message, + recoverable: rawWarning.recoverable, + ...(typeof rawWarning.table === 'string' ? { table: rawWarning.table } : {}), + ...(typeof rawWarning.column === 'string' ? { column: rawWarning.column } : {}), + ...(isRecord(rawWarning.metadata) ? { metadata: rawWarning.metadata } : {}), + }; +} + +async function readWarnings(input: ReadLocalScanStructuralSnapshotInput): Promise { + const path = `${input.rawSourcesDir}/warnings.json`; + try { + const warningRaw = await input.project.fileStore.readFile(path); + const parsed = JSON.parse(warningRaw.content) as unknown; + if (!isRecord(parsed) || !Array.isArray(parsed.warnings)) { + throw new Error(`Invalid KTX schema warnings artifact: ${path}`); + } + return parsed.warnings.map((warning) => parseWarning(warning, path)); + } catch (error) { + if (error instanceof Error && /not found|ENOENT|no such file/i.test(error.message)) { + return []; + } + throw error; + } +} + function optionalStringOrNull(value: unknown): string | null | undefined { if (value === undefined) { return undefined; @@ -113,6 +167,7 @@ export async function readLocalScanStructuralSnapshot( const tableRaw = await input.project.fileStore.readFile(path); tables.push(parseTable(tableRaw.content, path)); } + const warnings = await readWarnings(input); return { connectionId: typeof connection.connectionId === 'string' ? connection.connectionId : input.connectionId, @@ -121,5 +176,6 @@ export async function readLocalScanStructuralSnapshot( scope: isRecord(connection.scope) ? connection.scope : {}, metadata: metadataRecord(connection.metadata), tables, + warnings, }; } diff --git a/packages/cli/src/context/scan/relationship-benchmarks.ts b/packages/cli/src/context/scan/relationship-benchmarks.ts index f4367b5a..a07221a3 100644 --- a/packages/cli/src/context/scan/relationship-benchmarks.ts +++ b/packages/cli/src/context/scan/relationship-benchmarks.ts @@ -6,6 +6,7 @@ import { gunzipSync } from 'node:zlib'; import Database from 'better-sqlite3'; import YAML from 'yaml'; import { z } from 'zod'; +import { getDialectForDriver } from '../connections/dialects.js'; import type { KtxLlmRuntimePort } from '../llm/runtime-port.js'; import type { KtxEnrichedRelationship, KtxEnrichedSchema, KtxRelationshipType } from './enrichment-types.js'; import { snapshotToKtxEnrichedSchema } from './local-enrichment.js'; @@ -536,6 +537,7 @@ export function ktxRelationshipBenchmarkDetectorWithLlm( const formalLinks = formalMetadata.accepted.map((relationship) => relationshipToBenchmarkLink(relationship)); const acceptedKeys = new Set(formalLinks.map(fkKey)); const sqliteDataAvailable = Boolean(input.dataPath && input.snapshot.driver === 'sqlite'); + const dialect = getDialectForDriver(input.snapshot.driver); const profilingExecutor = sqliteDataAvailable && input.mode !== 'profiling_disabled' ? new KtxRelationshipBenchmarkSqliteExecutor(input.dataPath as string) @@ -550,7 +552,7 @@ export function ktxRelationshipBenchmarkDetectorWithLlm( }) : await profileKtxRelationshipSchema({ connectionId: input.snapshot.connectionId, - driver: input.snapshot.driver, + dialect, schema: input.schema, executor: profilingExecutor, ctx: { runId: `relationship-benchmark:${input.fixtureId}:${input.mode}:profile` }, @@ -580,7 +582,7 @@ export function ktxRelationshipBenchmarkDetectorWithLlm( : Math.max(0, input.validationBudget - profiles.queryCount); const validatedBroadCandidates = await validateKtxRelationshipDiscoveryCandidates({ connectionId: input.snapshot.connectionId, - driver: input.snapshot.driver, + dialect, candidates, profiles, executor: validationExecutor, @@ -597,7 +599,7 @@ export function ktxRelationshipBenchmarkDetectorWithLlm( input.mode !== 'validation_disabled' ? await discoverKtxCompositeRelationships({ connectionId: input.snapshot.connectionId, - driver: input.snapshot.driver, + dialect, schema: input.schema, profiles, executor: validationExecutor, @@ -671,6 +673,7 @@ export function currentKtxRelationshipBenchmarkDetector(): KtxRelationshipBenchm const formalLinks = formalMetadata.accepted.map((relationship) => relationshipToBenchmarkLink(relationship)); const acceptedKeys = new Set(formalLinks.map(fkKey)); const sqliteDataAvailable = Boolean(input.dataPath && input.snapshot.driver === 'sqlite'); + const dialect = getDialectForDriver(input.snapshot.driver); const profilingExecutor = sqliteDataAvailable && input.mode !== 'profiling_disabled' ? new KtxRelationshipBenchmarkSqliteExecutor(input.dataPath as string) @@ -685,7 +688,7 @@ export function currentKtxRelationshipBenchmarkDetector(): KtxRelationshipBenchm }) : await profileKtxRelationshipSchema({ connectionId: input.snapshot.connectionId, - driver: input.snapshot.driver, + dialect, schema: input.schema, executor: profilingExecutor, ctx: { runId: `relationship-benchmark:${input.fixtureId}:${input.mode}:profile` }, @@ -702,7 +705,7 @@ export function currentKtxRelationshipBenchmarkDetector(): KtxRelationshipBenchm : Math.max(0, input.validationBudget - profiles.queryCount); const validatedBroadCandidates = await validateKtxRelationshipDiscoveryCandidates({ connectionId: input.snapshot.connectionId, - driver: input.snapshot.driver, + dialect, candidates: broadRelationshipCandidates, profiles, executor: validationExecutor, @@ -719,7 +722,7 @@ export function currentKtxRelationshipBenchmarkDetector(): KtxRelationshipBenchm input.mode !== 'validation_disabled' ? await discoverKtxCompositeRelationships({ connectionId: input.snapshot.connectionId, - driver: input.snapshot.driver, + dialect, schema: input.schema, profiles, executor: validationExecutor, diff --git a/packages/cli/src/context/scan/relationship-composite-candidates.ts b/packages/cli/src/context/scan/relationship-composite-candidates.ts index d8ee650d..28263a15 100644 --- a/packages/cli/src/context/scan/relationship-composite-candidates.ts +++ b/packages/cli/src/context/scan/relationship-composite-candidates.ts @@ -1,11 +1,10 @@ +import type { KtxDialect } from '../connections/dialects.js'; import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable, KtxRelationshipType } from './enrichment-types.js'; import { - formatKtxRelationshipTableRef, - quoteKtxRelationshipIdentifier, type KtxRelationshipProfileArtifact, type KtxRelationshipReadOnlyExecutor, } from './relationship-profiling.js'; -import type { KtxConnectionDriver, KtxQueryResult, KtxScanContext, KtxTableRef } from './types.js'; +import type { KtxQueryResult, KtxScanContext, KtxTableRef } from './types.js'; type KtxCompositeRelationshipStatus = 'accepted' | 'review' | 'rejected'; @@ -57,7 +56,7 @@ export interface KtxCompositeRelationshipCandidate { export interface DiscoverKtxCompositeRelationshipsInput { connectionId: string; - driver: KtxConnectionDriver; + dialect: KtxDialect; schema: KtxEnrichedSchema; profiles: KtxRelationshipProfileArtifact; executor: KtxRelationshipReadOnlyExecutor | null; @@ -224,28 +223,16 @@ function numberAt(result: KtxQueryResult, header: string): number { return 0; } -function topSql(driver: KtxConnectionDriver, limit: number): string { - if (driver === 'sqlserver') { - return ` TOP (${Math.max(1, Math.floor(limit))})`; - } - return ''; +function sqlSuffix(fragment: string): string { + return fragment ? ` ${fragment}` : ''; } -function limitSql(driver: KtxConnectionDriver, limit: number): string { - if (driver === 'sqlserver') { - return ''; - } - return ` LIMIT ${Math.max(1, Math.floor(limit))}`; +function aliasedTupleSelect(dialect: KtxDialect, columns: readonly string[]): string { + return columns.map((column, index) => `${dialect.quoteIdentifier(column)} AS c${index}`).join(', '); } -function aliasedTupleSelect(driver: KtxConnectionDriver, columns: readonly string[]): string { - return columns - .map((column, index) => `${quoteKtxRelationshipIdentifier(driver, column)} AS c${index}`) - .join(', '); -} - -function nonNullPredicate(driver: KtxConnectionDriver, columns: readonly string[]): string { - return columns.map((column) => `${quoteKtxRelationshipIdentifier(driver, column)} IS NOT NULL`).join(' AND '); +function nonNullPredicate(dialect: KtxDialect, columns: readonly string[]): string { + return columns.map((column) => `${dialect.quoteIdentifier(column)} IS NOT NULL`).join(' AND '); } function tupleEquality(columns: number): string { @@ -255,39 +242,39 @@ function tupleEquality(columns: number): string { } function buildTupleDistinctSql(input: { - driver: KtxConnectionDriver; + dialect: KtxDialect; table: KtxTableRef; columns: readonly string[]; }): string { - const tableSql = formatKtxRelationshipTableRef(input.driver, input.table); + const tableSql = input.dialect.formatTableName(input.table); return [ 'WITH tuple_values AS (', - `SELECT DISTINCT ${aliasedTupleSelect(input.driver, input.columns)} FROM ${tableSql}`, - `WHERE ${nonNullPredicate(input.driver, input.columns)}`, + `SELECT DISTINCT ${aliasedTupleSelect(input.dialect, input.columns)} FROM ${tableSql}`, + `WHERE ${nonNullPredicate(input.dialect, input.columns)}`, ')', 'SELECT COUNT(*) AS distinct_count FROM tuple_values', ].join(' '); } function buildCompositeCoverageSql(input: { - driver: KtxConnectionDriver; + dialect: KtxDialect; childTable: KtxTableRef; childColumns: readonly string[]; parentTable: KtxTableRef; parentColumns: readonly string[]; maxDistinctSourceValues: number; }): string { - const childTableSql = formatKtxRelationshipTableRef(input.driver, input.childTable); - const parentTableSql = formatKtxRelationshipTableRef(input.driver, input.parentTable); - const top = topSql(input.driver, input.maxDistinctSourceValues); - const limit = limitSql(input.driver, input.maxDistinctSourceValues); + const childTableSql = input.dialect.formatTableName(input.childTable); + const parentTableSql = input.dialect.formatTableName(input.parentTable); + const top = input.dialect.getTopClause(input.maxDistinctSourceValues); + const limit = sqlSuffix(input.dialect.getLimitOffsetClause(input.maxDistinctSourceValues)); return [ 'WITH child_values AS (', - `SELECT DISTINCT${top} ${aliasedTupleSelect(input.driver, input.childColumns)} FROM ${childTableSql}`, - `WHERE ${nonNullPredicate(input.driver, input.childColumns)}${limit}`, + `SELECT DISTINCT${top ? ` ${top}` : ''} ${aliasedTupleSelect(input.dialect, input.childColumns)} FROM ${childTableSql}`, + `WHERE ${nonNullPredicate(input.dialect, input.childColumns)}${limit}`, '), parent_values AS (', - `SELECT DISTINCT ${aliasedTupleSelect(input.driver, input.parentColumns)} FROM ${parentTableSql}`, - `WHERE ${nonNullPredicate(input.driver, input.parentColumns)}`, + `SELECT DISTINCT ${aliasedTupleSelect(input.dialect, input.parentColumns)} FROM ${parentTableSql}`, + `WHERE ${nonNullPredicate(input.dialect, input.parentColumns)}`, ')', 'SELECT', '(SELECT COUNT(*) FROM child_values) AS child_distinct,', @@ -335,7 +322,7 @@ function hasAcceptedSubset( async function detectCompositePrimaryKeys(input: { connectionId: string; - driver: KtxConnectionDriver; + dialect: KtxDialect; table: KtxEnrichedTable; profiles: KtxRelationshipProfileArtifact; executor: KtxRelationshipReadOnlyExecutor; @@ -379,7 +366,7 @@ async function detectCompositePrimaryKeys(input: { { connectionId: input.connectionId, sql: buildTupleDistinctSql({ - driver: input.driver, + dialect: input.dialect, table: input.table.ref, columns: columnNames, }), @@ -439,7 +426,7 @@ function compatibleTuple(sourceColumns: readonly KtxEnrichedColumn[], targetColu async function validateCompositeRelationship(input: { connectionId: string; - driver: KtxConnectionDriver; + dialect: KtxDialect; sourceTable: KtxEnrichedTable; sourceColumns: readonly KtxEnrichedColumn[]; targetKey: KtxCompositePrimaryKeyCandidate; @@ -454,7 +441,7 @@ async function validateCompositeRelationship(input: { { connectionId: input.connectionId, sql: buildCompositeCoverageSql({ - driver: input.driver, + dialect: input.dialect, childTable: input.sourceTable.ref, childColumns: input.sourceColumns.map((column) => column.name), parentTable: input.targetTable.ref, @@ -552,7 +539,7 @@ export async function discoverKtxCompositeRelationships( for (const table of tables) { const result = await detectCompositePrimaryKeys({ connectionId: input.connectionId, - driver: input.driver, + dialect: input.dialect, table, profiles: input.profiles, executor: input.executor, @@ -595,7 +582,7 @@ export async function discoverKtxCompositeRelationships( const result = await validateCompositeRelationship({ connectionId: input.connectionId, - driver: input.driver, + dialect: input.dialect, sourceTable, sourceColumns, targetKey, diff --git a/packages/cli/src/context/scan/relationship-discovery.ts b/packages/cli/src/context/scan/relationship-discovery.ts index 66a47395..fc755536 100644 --- a/packages/cli/src/context/scan/relationship-discovery.ts +++ b/packages/cli/src/context/scan/relationship-discovery.ts @@ -1,4 +1,5 @@ import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; +import type { KtxDialect } from '../connections/dialects.js'; import type { KtxScanRelationshipConfig } from '../project/config.js'; import type { KtxEnrichedRelationship, KtxEnrichedSchema, KtxRelationshipUpdate } from './enrichment-types.js'; import { @@ -24,7 +25,6 @@ import { } from './relationship-profiling.js'; import { validateKtxRelationshipDiscoveryCandidates } from './relationship-validation.js'; import type { - KtxConnectionDriver, KtxScanConnector, KtxScanContext, KtxScanEnrichmentSummary, @@ -34,7 +34,7 @@ import type { export interface DiscoverKtxRelationshipsInput { connectionId: string; - driver: KtxConnectionDriver; + dialect: KtxDialect; connector: KtxScanConnector; schema: KtxEnrichedSchema; context: KtxScanContext; @@ -122,7 +122,7 @@ function compositeSummary(relationships: readonly KtxCompositeRelationshipCandid async function detectCompositeRelationships(input: { connectionId: string; - driver: DiscoverKtxRelationshipsInput['driver']; + dialect: KtxDialect; schema: KtxEnrichedSchema; profile: KtxRelationshipProfileArtifact; executor: KtxRelationshipReadOnlyExecutor | null; @@ -135,7 +135,7 @@ async function detectCompositeRelationships(input: { try { const compositeDetection = await discoverKtxCompositeRelationships({ connectionId: input.connectionId, - driver: input.driver, + dialect: input.dialect, schema: input.schema, profiles: input.profile, executor: input.executor, @@ -223,7 +223,7 @@ export async function discoverKtxRelationships( const profileCache = createKtxRelationshipProfileCache(); const profile = await profileKtxRelationshipSchema({ connectionId: input.connectionId, - driver: input.driver, + dialect: input.dialect, schema: input.schema, executor, ctx: input.context, @@ -256,7 +256,7 @@ export async function discoverKtxRelationships( warnings.push(...llmProposalResult.warnings); const validated = await validateKtxRelationshipDiscoveryCandidates({ connectionId: input.connectionId, - driver: input.driver, + dialect: input.dialect, candidates, profiles: profile, executor, @@ -282,7 +282,7 @@ export async function discoverKtxRelationships( }); const compositeRelationships = await detectCompositeRelationships({ connectionId: input.connectionId, - driver: input.driver, + dialect: input.dialect, schema: input.schema, profile, executor, diff --git a/packages/cli/src/context/scan/relationship-profiling.ts b/packages/cli/src/context/scan/relationship-profiling.ts index 1824d263..0f22c21c 100644 --- a/packages/cli/src/context/scan/relationship-profiling.ts +++ b/packages/cli/src/context/scan/relationship-profiling.ts @@ -1,3 +1,4 @@ +import type { KtxDialect } from '../connections/dialects.js'; import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from './enrichment-types.js'; import { mapWithConcurrency } from './relationship-validation.js'; import type { @@ -55,7 +56,7 @@ export interface KtxRelationshipProfileCache { export interface ProfileKtxRelationshipSchemaInput { connectionId: string; - driver: KtxConnectionDriver; + dialect: KtxDialect; schema: KtxEnrichedSchema; executor: KtxRelationshipReadOnlyExecutor | null; ctx: KtxScanContext; @@ -71,75 +72,6 @@ export function createKtxRelationshipProfileCache(): KtxRelationshipProfileCache const SAMPLE_VALUE_DELIMITER = '\u001f'; -type QuoteStyle = 'double' | 'backtick' | 'bracket'; - -function quoteStyle(driver: KtxConnectionDriver): QuoteStyle { - if (driver === 'mysql' || driver === 'clickhouse') { - return 'backtick'; - } - if (driver === 'sqlserver') { - return 'bracket'; - } - return 'double'; -} - -export function quoteKtxRelationshipIdentifier(driver: KtxConnectionDriver, identifier: string): string { - switch (quoteStyle(driver)) { - case 'backtick': - return `\`${identifier.replace(/`/g, '``')}\``; - case 'bracket': - return `[${identifier.replace(/\]/g, ']]')}]`; - case 'double': - return `"${identifier.replace(/"/g, '""')}"`; - } -} - -export function formatKtxRelationshipTableRef(driver: KtxConnectionDriver, table: KtxTableRef): string { - const parts = - driver === 'sqlite' - ? [table.name] - : [table.catalog, table.db, table.name].filter((value): value is string => Boolean(value)); - return parts.map((part) => quoteKtxRelationshipIdentifier(driver, part)).join('.'); -} - -function textLengthExpression(driver: KtxConnectionDriver, columnSql: string): string { - if (driver === 'mysql') { - return `CHAR_LENGTH(CAST(${columnSql} AS CHAR))`; - } - if (driver === 'sqlserver') { - return `LEN(CAST(${columnSql} AS NVARCHAR(MAX)))`; - } - if (driver === 'bigquery') { - return `LENGTH(CAST(${columnSql} AS STRING))`; - } - if (driver === 'clickhouse') { - return `length(toString(${columnSql}))`; - } - return `LENGTH(CAST(${columnSql} AS TEXT))`; -} - -function limitSql(driver: KtxConnectionDriver, limit: number): string { - if (driver === 'sqlserver') { - return ''; - } - return ` LIMIT ${Math.max(1, Math.floor(limit))}`; -} - -function topSql(driver: KtxConnectionDriver, limit: number): string { - if (driver === 'sqlserver') { - return ` TOP (${Math.max(1, Math.floor(limit))})`; - } - return ''; -} - -function sampledTableSql(driver: KtxConnectionDriver, tableSql: string, limit: number): string { - const safeLimit = Math.max(1, Math.floor(limit)); - if (driver === 'sqlserver') { - return `(SELECT TOP (${safeLimit}) * FROM ${tableSql}) AS relationship_profile_sample`; - } - return `(SELECT * FROM ${tableSql}${limitSql(driver, safeLimit)}) AS relationship_profile_sample`; -} - function firstRow(result: KtxQueryResult): unknown[] { return result.rows[0] ?? []; } @@ -191,7 +123,7 @@ function columnKey(table: KtxEnrichedTable, column: KtxEnrichedColumn): string { function tableProfileCacheKey(input: { connectionId: string; - driver: KtxConnectionDriver; + dialect: KtxDialect; ctx: KtxScanContext; table: KtxTableRef; sampleValuesPerColumn: number; @@ -200,7 +132,7 @@ function tableProfileCacheKey(input: { return [ input.ctx.runId, input.connectionId, - input.driver, + input.dialect.type, input.table.catalog ?? '', input.table.db ?? '', input.table.name, @@ -213,57 +145,47 @@ function sqlStringLiteral(value: string): string { return `'${value.replace(/'/g, "''")}'`; } -function sampleAggregateSql(driver: KtxConnectionDriver, innerSql: string): string { - if (driver === 'postgres') { - return `(SELECT STRING_AGG(CAST(value AS TEXT), CHR(31)) FROM (${innerSql}) AS relationship_profile_values)`; +function sqlSuffix(fragment: string): string { + return fragment ? ` ${fragment}` : ''; +} + +function sampledTableSql(dialect: KtxDialect, tableSql: string, limit: number): string { + const top = dialect.getTopClause(limit); + if (top) { + return `(SELECT ${top} * FROM ${tableSql}) AS relationship_profile_sample`; } - if (driver === 'bigquery') { - return `(SELECT STRING_AGG(CAST(value AS STRING), '\\u001F') FROM (${innerSql}) AS relationship_profile_values)`; - } - if (driver === 'mysql') { - return `(SELECT GROUP_CONCAT(CAST(value AS CHAR) SEPARATOR CHAR(31)) FROM (${innerSql}) AS relationship_profile_values)`; - } - if (driver === 'sqlserver') { - return `(SELECT STRING_AGG(CAST(value AS NVARCHAR(MAX)), CHAR(31)) FROM (${innerSql}) AS relationship_profile_values)`; - } - if (driver === 'clickhouse') { - return `(SELECT arrayStringConcat(groupArray(toString(value)), '\\x1F') FROM (${innerSql}) AS relationship_profile_values)`; - } - if (driver === 'snowflake') { - return `(SELECT LISTAGG(CAST(value AS VARCHAR), '\\x1f') FROM (${innerSql}) AS relationship_profile_values)`; - } - return `(SELECT GROUP_CONCAT(CAST(value AS TEXT), char(31)) FROM (${innerSql}) AS relationship_profile_values)`; + return `(SELECT * FROM ${tableSql}${sqlSuffix(dialect.getLimitOffsetClause(limit))}) AS relationship_profile_sample`; } function sampleValuesSql(input: { - driver: KtxConnectionDriver; + dialect: KtxDialect; tableSql: string; columnSql: string; limit: number; }): string { + const top = input.dialect.getTopClause(input.limit); return [ - `SELECT${topSql(input.driver, input.limit)} ${input.columnSql} AS value`, + `SELECT${top ? ` ${top}` : ''} ${input.columnSql} AS value`, `FROM ${input.tableSql}`, `WHERE ${input.columnSql} IS NOT NULL`, `GROUP BY ${input.columnSql}`, `ORDER BY COUNT(*) DESC, ${input.columnSql} ASC`, - limitSql(input.driver, input.limit), + sqlSuffix(input.dialect.getLimitOffsetClause(input.limit)), ].join(' '); } function columnProfileSelectSql(input: { - connectionDriver: KtxConnectionDriver; + dialect: KtxDialect; tableSql: string; profileTableSql: string; column: KtxEnrichedColumn; sampleValuesPerColumn: number; }): string { - const columnSql = quoteKtxRelationshipIdentifier(input.connectionDriver, input.column.name); - const textLengthSql = textLengthExpression(input.connectionDriver, columnSql); - const samplesSql = sampleAggregateSql( - input.connectionDriver, + const columnSql = input.dialect.quoteIdentifier(input.column.name); + const textLengthSql = input.dialect.textLengthExpression(columnSql); + const samplesSql = input.dialect.getSampleValueAggregation( sampleValuesSql({ - driver: input.connectionDriver, + dialect: input.dialect, tableSql: input.profileTableSql, columnSql, limit: input.sampleValuesPerColumn, @@ -296,12 +218,12 @@ function splitSampleValues(value: unknown): string[] { async function queryCount(input: { connectionId: string; - driver: KtxConnectionDriver; + dialect: KtxDialect; table: KtxTableRef; executor: KtxRelationshipReadOnlyExecutor; ctx: KtxScanContext; }): Promise<{ rowCount: number; queryCount: number }> { - const tableSql = formatKtxRelationshipTableRef(input.driver, input.table); + const tableSql = input.dialect.formatTableName(input.table); const result = await input.executor.executeReadOnly( { connectionId: input.connectionId, sql: `SELECT COUNT(*) AS row_count FROM ${tableSql}`, maxRows: 1 }, input.ctx, @@ -311,7 +233,7 @@ async function queryCount(input: { async function queryTableProfile(input: { connectionId: string; - driver: KtxConnectionDriver; + dialect: KtxDialect; table: KtxEnrichedTable; executor: KtxRelationshipReadOnlyExecutor; ctx: KtxScanContext; @@ -325,7 +247,7 @@ async function queryTableProfile(input: { if (input.table.columns.length === 0) { const rowCount = await queryCount({ connectionId: input.connectionId, - driver: input.driver, + dialect: input.dialect, table: input.table.ref, executor: input.executor, ctx: input.ctx, @@ -337,12 +259,12 @@ async function queryTableProfile(input: { }; } - const tableSql = formatKtxRelationshipTableRef(input.driver, input.table.ref); - const profileTableSql = sampledTableSql(input.driver, tableSql, input.profileSampleRows); + const tableSql = input.dialect.formatTableName(input.table.ref); + const profileTableSql = sampledTableSql(input.dialect, tableSql, input.profileSampleRows); const sql = input.table.columns .map((column) => columnProfileSelectSql({ - connectionDriver: input.driver, + dialect: input.dialect, tableSql, profileTableSql, column, @@ -401,7 +323,7 @@ export async function profileKtxRelationshipSchema( if (!input.executor) { return { connectionId: input.connectionId, - driver: input.driver, + driver: input.dialect.type, sqlAvailable: false, queryCount: 0, tables: [], @@ -425,7 +347,7 @@ export async function profileKtxRelationshipSchema( const profileSampleRows = input.profileSampleRows ?? 10000; const cacheKey = tableProfileCacheKey({ connectionId: input.connectionId, - driver: input.driver, + dialect: input.dialect, ctx: input.ctx, table: table.ref, sampleValuesPerColumn, @@ -439,7 +361,7 @@ export async function profileKtxRelationshipSchema( try { const tableProfile = await queryTableProfile({ connectionId: input.connectionId, - driver: input.driver, + dialect: input.dialect, table, executor, ctx: input.ctx, @@ -481,7 +403,7 @@ export async function profileKtxRelationshipSchema( return { connectionId: input.connectionId, - driver: input.driver, + driver: input.dialect.type, sqlAvailable: true, queryCount: queryTotal, tables, diff --git a/packages/cli/src/context/scan/relationship-validation.ts b/packages/cli/src/context/scan/relationship-validation.ts index 685d1ea9..5fc0f3fb 100644 --- a/packages/cli/src/context/scan/relationship-validation.ts +++ b/packages/cli/src/context/scan/relationship-validation.ts @@ -1,13 +1,12 @@ +import type { KtxDialect } from '../connections/dialects.js'; import type { KtxRelationshipEndpoint } from './enrichment-types.js'; import { applyKtxRelationshipValidationBudget, type KtxRelationshipValidationBudget } from './relationship-budget.js'; import type { KtxRelationshipDiscoveryCandidate } from './relationship-candidates.js'; import { - formatKtxRelationshipTableRef, type KtxRelationshipProfileArtifact, type KtxRelationshipReadOnlyExecutor, - quoteKtxRelationshipIdentifier, } from './relationship-profiling.js'; -import type { KtxConnectionDriver, KtxQueryResult, KtxScanContext } from './types.js'; +import type { KtxQueryResult, KtxScanContext, KtxTableRef } from './types.js'; type KtxValidatedRelationshipStatus = 'accepted' | 'review' | 'rejected'; @@ -45,7 +44,7 @@ export interface KtxValidatedRelationshipDiscoveryCandidate export interface ValidateKtxRelationshipDiscoveryCandidatesInput { connectionId: string; - driver: KtxConnectionDriver; + dialect: KtxDialect; candidates: readonly KtxRelationshipDiscoveryCandidate[]; profiles: KtxRelationshipProfileArtifact; executor: KtxRelationshipReadOnlyExecutor | null; @@ -104,38 +103,28 @@ function numberAt(result: KtxQueryResult, header: string): number { return 0; } -function limitSql(driver: KtxConnectionDriver, limit: number): string { - if (driver === 'sqlserver') { - return ''; - } - return ` LIMIT ${Math.max(1, Math.floor(limit))}`; -} - -function topSql(driver: KtxConnectionDriver, limit: number): string { - if (driver === 'sqlserver') { - return ` TOP (${Math.max(1, Math.floor(limit))})`; - } - return ''; +function sqlSuffix(fragment: string): string { + return fragment ? ` ${fragment}` : ''; } function buildCoverageSql(input: { - driver: KtxConnectionDriver; - childTable: string; + dialect: KtxDialect; + childTable: KtxTableRef; childColumn: string; - parentTable: string; + parentTable: KtxTableRef; parentColumn: string; maxDistinctSourceValues: number; }): string { - const childTable = formatKtxRelationshipTableRef(input.driver, { catalog: null, db: null, name: input.childTable }); - const parentTable = formatKtxRelationshipTableRef(input.driver, { catalog: null, db: null, name: input.parentTable }); - const childColumn = quoteKtxRelationshipIdentifier(input.driver, input.childColumn); - const parentColumn = quoteKtxRelationshipIdentifier(input.driver, input.parentColumn); - const limit = limitSql(input.driver, input.maxDistinctSourceValues); - const top = topSql(input.driver, input.maxDistinctSourceValues); + const childTable = input.dialect.formatTableName(input.childTable); + const parentTable = input.dialect.formatTableName(input.parentTable); + const childColumn = input.dialect.quoteIdentifier(input.childColumn); + const parentColumn = input.dialect.quoteIdentifier(input.parentColumn); + const limit = sqlSuffix(input.dialect.getLimitOffsetClause(input.maxDistinctSourceValues)); + const top = input.dialect.getTopClause(input.maxDistinctSourceValues); return [ 'WITH child_values AS (', - `SELECT DISTINCT${top} ${childColumn} AS value FROM ${childTable} WHERE ${childColumn} IS NOT NULL${limit}`, + `SELECT DISTINCT${top ? ` ${top}` : ''} ${childColumn} AS value FROM ${childTable} WHERE ${childColumn} IS NOT NULL${limit}`, '), parent_values AS (', `SELECT DISTINCT ${parentColumn} AS value FROM ${parentTable} WHERE ${parentColumn} IS NOT NULL`, ')', @@ -271,10 +260,10 @@ export async function validateKtxRelationshipDiscoveryCandidates( { connectionId: input.connectionId, sql: buildCoverageSql({ - driver: input.driver, - childTable: candidate.from.table.name, + dialect: input.dialect, + childTable: candidate.from.table, childColumn: sourceColumn, - parentTable: candidate.to.table.name, + parentTable: candidate.to.table, parentColumn: targetColumn, maxDistinctSourceValues: settings.maxDistinctSourceValues, }), diff --git a/packages/cli/src/context/scan/table-ref.ts b/packages/cli/src/context/scan/table-ref.ts index 1a2abd70..368d4adb 100644 --- a/packages/cli/src/context/scan/table-ref.ts +++ b/packages/cli/src/context/scan/table-ref.ts @@ -33,8 +33,7 @@ export function tableRefSet(refs: readonly KtxTableRef[]): ReadonlySet, @@ -45,8 +44,8 @@ export function scopedTableNames( const wantDb = namespace.db ?? null; for (const key of scope) { const ref = tableRefFromKey(key); - if (wantCatalog !== null && ref.catalog !== null && ref.catalog !== wantCatalog) continue; - if (wantDb !== null && ref.db !== null && ref.db !== wantDb) continue; + if (ref.catalog !== wantCatalog) continue; + if (ref.db !== wantDb) continue; names.add(ref.name); } return [...names]; diff --git a/packages/cli/src/context/scan/types.ts b/packages/cli/src/context/scan/types.ts index 95e6b590..fc445b5e 100644 --- a/packages/cli/src/context/scan/types.ts +++ b/packages/cli/src/context/scan/types.ts @@ -3,7 +3,6 @@ import type { KtxTableRefKey } from './table-ref.js'; export type KtxConnectionDriver = | 'sqlite' | 'postgres' - | 'postgresql' | 'sqlserver' | 'bigquery' | 'snowflake' @@ -91,6 +90,7 @@ export interface KtxSchemaSnapshot { scope: KtxSchemaScope; tables: KtxSchemaTable[]; metadata: Record; + warnings?: KtxScanWarning[]; } interface KtxCredentialEnvReference { @@ -297,14 +297,35 @@ export interface KtxQueryResult { } export interface KtxTableListEntry { + catalog: string | null; schema: string; name: string; kind: 'table' | 'view'; } -interface KtxConnectorTestResult { +export interface KtxConnectorTestResult { success: boolean; error?: string; + /** + * The original error thrown by the driver, preserved unflattened so the + * connection-test path can re-throw it. Keeping the real error object lets + * telemetry record the driver's actual error class (e.g. `ConnectionError`) + * and `.code` (e.g. `ELOGIN`) instead of collapsing every failure to `Error`. + */ + cause?: unknown; +} + +/** + * Single source of truth for a failed connector test result. Captures the + * driver's message for display while preserving the original error as `cause` + * so callers can surface its real class and code. + */ +export function connectorTestFailure(error: unknown): KtxConnectorTestResult { + return { + success: false, + error: error instanceof Error ? error.message : String(error), + cause: error, + }; } export interface KtxScanConnector { @@ -313,6 +334,8 @@ export interface KtxScanConnector { capabilities: KtxConnectorCapabilities; eventStreamDiscovery?: KtxEventStreamDiscoveryPort; introspect(input: KtxScanInput, ctx: KtxScanContext): Promise; + listSchemas(): Promise; + listTables(schemas?: string[]): Promise; testConnection?(): Promise; sampleColumn?(input: KtxColumnSampleInput, ctx: KtxScanContext): Promise; sampleTable?(input: KtxTableSampleInput, ctx: KtxScanContext): Promise; @@ -365,7 +388,8 @@ type KtxScanWarningCode = | 'relationship_llm_proposal_failed' | 'credential_redacted' | 'enrichment_failed' - | 'description_fallback_used'; + | 'description_fallback_used' + | 'constraint_discovery_unauthorized'; export interface KtxScanWarning { code: KtxScanWarningCode; diff --git a/packages/cli/src/context/scan/warehouse-catalog.ts b/packages/cli/src/context/scan/warehouse-catalog.ts index 2f360eeb..f224432b 100644 --- a/packages/cli/src/context/scan/warehouse-catalog.ts +++ b/packages/cli/src/context/scan/warehouse-catalog.ts @@ -1,4 +1,4 @@ -import { getDialectForDriver } from '../../context/connections/dialects.js'; +import { getDialectForDriver, type KtxDialect } from '../connections/dialects.js'; import type { KtxFileStorePort } from '../../context/core/file-store.js'; import type { KtxConnectionDriver, @@ -8,7 +8,7 @@ import type { KtxTableRef, } from './types.js'; -type CatalogDriver = KtxConnectionDriver | 'sqlite3'; +type CatalogDriver = KtxConnectionDriver; export interface WarehouseCatalogServiceDeps { fileStore: KtxFileStorePort; @@ -128,46 +128,22 @@ function splitDisplay(display: string): string[] { .filter(Boolean); } -function formatDisplay(driver: CatalogDriver, table: KtxTableRef): string { - if (driver === 'sqlite' || driver === 'sqlite3') { - return table.name; - } - return [table.catalog, table.db, table.name].filter((part): part is string => Boolean(part)).join('.'); +function formatDisplay(dialect: KtxDialect, table: KtxTableRef): string { + return dialect.formatDisplayRef(table); } -function parseDisplay(driver: CatalogDriver, display: string): KtxTableRef | null { +function parseDisplay(dialect: KtxDialect, display: string): KtxTableRef | null { + const parsed = dialect.parseDisplayRef(display); + if (parsed) { + return parsed; + } const parts = splitDisplay(display); - if (driver === 'sqlite' || driver === 'sqlite3') { - return parts.length === 1 ? { catalog: null, db: null, name: parts[0]! } : null; - } - if (driver === 'bigquery' || driver === 'snowflake' || driver === 'sqlserver') { - if (parts.length !== 3) { - return null; - } - return { catalog: parts[0]!, db: parts[1]!, name: parts[2]! }; - } - if (parts.length === 2) { - return { catalog: null, db: parts[0]!, name: parts[1]! }; - } - if (parts.length === 3) { - return { catalog: parts[0]!, db: parts[1]!, name: parts[2]! }; - } return parts.length === 1 ? { catalog: null, db: null, name: parts[0]! } : null; } -function expectedDisplayPartCount(driver: CatalogDriver): number { - if (driver === 'sqlite' || driver === 'sqlite3') { - return 1; - } - if (driver === 'bigquery' || driver === 'snowflake' || driver === 'sqlserver') { - return 3; - } - return 2; -} - -function parseColumnDisplay(driver: CatalogDriver, display: string): (KtxTableRef & { column: string }) | null { +function parseColumnDisplay(dialect: KtxDialect, display: string): (KtxTableRef & { column: string }) | null { const parts = splitDisplay(display); - const tablePartCount = expectedDisplayPartCount(driver); + const tablePartCount = dialect.columnDisplayTablePartCount(); if (parts.length !== tablePartCount + 1) { return null; } @@ -175,7 +151,7 @@ function parseColumnDisplay(driver: CatalogDriver, display: string): (KtxTableRe if (!column) { return null; } - const table = parseDisplay(driver, parts.slice(0, -1).join('.')); + const table = dialect.parseDisplayRef(parts.slice(0, -1).join('.')); return table ? { ...table, column } : null; } @@ -272,6 +248,7 @@ export class WarehouseCatalogService { if (!table) { return null; } + const dialect = getDialectForDriver(catalog.driver); const profileTables = catalog.profile?.tables ?? []; const profileTable = profileTables.find((candidate) => candidate.table && refsEqual(candidate.table, table)); const profileColumns = catalog.profile?.columns ?? {}; @@ -281,7 +258,7 @@ export class WarehouseCatalogService { catalog: table.catalog, db: table.db, name: table.name, - display: formatDisplay(catalog.driver, table), + display: formatDisplay(dialect, table), kind: table.kind, comment: table.comment, description: firstDescription(table.descriptions), @@ -321,16 +298,21 @@ export class WarehouseCatalogService { if (!catalog) { return { resolved: null, candidates: [], dialect: 'unknown' }; } - const dialect = getDialectForDriver(catalog.driver).type; - const parsed = parseDisplay(catalog.driver, display); + const dialect = getDialectForDriver(catalog.driver); + const parsed = parseDisplay(dialect, display); if (!parsed) { - return { resolved: null, candidates: bestCandidates(catalog.tables, display), dialect }; + return { resolved: null, candidates: bestCandidates(catalog.tables, display), dialect: dialect.type }; } - const table = catalog.tables.find((candidate) => refsEqual(candidate, parsed)); + const exactTable = catalog.tables.find((candidate) => refsEqual(candidate, parsed)); + const looseNameMatches = + parsed.catalog === null && parsed.db === null + ? catalog.tables.filter((candidate) => normalize(candidate.name) === normalize(parsed.name)) + : []; + const table = exactTable ?? (looseNameMatches.length === 1 ? looseNameMatches[0] : undefined); if (!table) { - return { resolved: null, candidates: bestCandidates(catalog.tables, display), dialect }; + return { resolved: null, candidates: bestCandidates(catalog.tables, display), dialect: dialect.type }; } - return { resolved: { catalog: table.catalog, db: table.db, name: table.name }, candidates: [], dialect }; + return { resolved: { catalog: table.catalog, db: table.db, name: table.name }, candidates: [], dialect: dialect.type }; } async resolveDisplayTarget(connectionId: string, display: string): Promise { @@ -339,20 +321,20 @@ export class WarehouseCatalogService { return { resolved: null, candidates: [], dialect: 'unknown' }; } - const dialect = getDialectForDriver(catalog.driver).type; + const dialect = getDialectForDriver(catalog.driver); const tableResolution = await this.resolveDisplay(connectionId, display); if (tableResolution.resolved) { return tableResolution; } - const parsedColumn = parseColumnDisplay(catalog.driver, display); + const parsedColumn = parseColumnDisplay(dialect, display); if (!parsedColumn) { - return { resolved: null, candidates: bestCandidates(catalog.tables, display), dialect }; + return { resolved: null, candidates: bestCandidates(catalog.tables, display), dialect: dialect.type }; } const table = catalog.tables.find((candidate) => refsEqual(candidate, parsedColumn)); if (!table) { - return { resolved: null, candidates: bestCandidates(catalog.tables, display), dialect }; + return { resolved: null, candidates: bestCandidates(catalog.tables, display), dialect: dialect.type }; } return { @@ -363,7 +345,7 @@ export class WarehouseCatalogService { column: parsedColumn.column, }, candidates: [], - dialect, + dialect: dialect.type, }; } @@ -372,6 +354,7 @@ export class WarehouseCatalogService { if (!catalog) { return []; } + const dialect = getDialectForDriver(catalog.driver); const hits: RawSchemaHit[] = []; for (const table of catalog.tables as TableWithDescriptions[]) { const tableMatch = matchedOnTable(table, query); @@ -380,7 +363,7 @@ export class WarehouseCatalogService { kind: 'table', connectionId, ref: { catalog: table.catalog, db: table.db, name: table.name }, - display: formatDisplay(catalog.driver, table), + display: formatDisplay(dialect, table), matchedOn: tableMatch, }); } @@ -393,7 +376,7 @@ export class WarehouseCatalogService { kind: 'column', connectionId, ref: { catalog: table.catalog, db: table.db, name: table.name, column: column.name }, - display: `${formatDisplay(catalog.driver, table)}.${column.name}`, + display: `${formatDisplay(dialect, table)}.${column.name}`, matchedOn: columnMatch, }); } diff --git a/packages/cli/src/context/search/discover.ts b/packages/cli/src/context/search/discover.ts index b3456459..9a572daf 100644 --- a/packages/cli/src/context/search/discover.ts +++ b/packages/cli/src/context/search/discover.ts @@ -167,7 +167,7 @@ async function wikiCandidates( query: input.query, userId: options.userId, embeddingService: options.embeddingService ?? null, - limit: Math.max(input.limit ?? 15, 25), + limit: Math.max(input.limit ?? 10, 25), }); const records: CandidateRecord[] = []; for (const result of searchResults) { @@ -421,7 +421,8 @@ function hydrate( } return { ...ref, - score: maxScore > 0 ? Number((candidate.score / maxScore).toFixed(6)) : 0, + // 3 decimals is plenty for a relative-rank hint; 6 just spent bytes on noise. + score: maxScore > 0 ? Number((candidate.score / maxScore).toFixed(3)) : 0, }; }) .filter((result): result is KtxDiscoverDataRef => result !== null); @@ -433,7 +434,7 @@ export function createKtxDiscoverDataService( ): { search(input: KtxDiscoverDataInput): Promise } { return { async search(input) { - const limit = Math.max(1, Math.min(input.limit ?? 15, 50)); + const limit = Math.max(1, Math.min(input.limit ?? 10, 50)); const query = input.query.trim(); if (!query) { return []; diff --git a/packages/cli/src/context/sl/local-query.ts b/packages/cli/src/context/sl/local-query.ts index 4d71504e..781d8b94 100644 --- a/packages/cli/src/context/sl/local-query.ts +++ b/packages/cli/src/context/sl/local-query.ts @@ -48,17 +48,14 @@ function assertSafeConnectionId(connectionId: string): string { function dialectForDriver(driver: string | undefined): string { const normalized = (driver ?? 'postgres').toUpperCase(); const map: Record = { - POSTGRESQL: 'postgres', POSTGRES: 'postgres', BIGQUERY: 'bigquery', SNOWFLAKE: 'snowflake', MYSQL: 'mysql', SQLSERVER: 'tsql', - MSSQL: 'tsql', SQLITE: 'sqlite', DUCKDB: 'duckdb', CLICKHOUSE: 'clickhouse', - REDSHIFT: 'redshift', DATABRICKS: 'databricks', }; return map[normalized] ?? 'postgres'; diff --git a/packages/cli/src/context/sl/local-sl.ts b/packages/cli/src/context/sl/local-sl.ts index 18ec8417..fb573392 100644 --- a/packages/cli/src/context/sl/local-sl.ts +++ b/packages/cli/src/context/sl/local-sl.ts @@ -50,6 +50,7 @@ export interface LocalSlSearchInput { pglite?: PgliteSlSearchPrototypeOwnerOptions; } +/** @internal */ export interface LocalSlSource extends LocalSlSourceSummary { yaml: string; } @@ -63,6 +64,11 @@ export interface LocalSlValidationResult { errors: string[]; } +export type ResolvedSlSource = + | { kind: 'found'; source: LocalSlSource } + | { kind: 'not-found' } + | { kind: 'ambiguous'; connectionIds: string[] }; + const LOCAL_AUTHOR = 'ktx'; const LOCAL_AUTHOR_EMAIL = 'ktx@example.com'; @@ -266,23 +272,6 @@ export async function validateLocalSlSource( try { const parsed = parseYamlRecord(rawYaml); const schema = parsed.table || parsed.sql ? sourceDefinitionSchema : sourceOverlaySchema; - if (schema === sourceOverlaySchema && Array.isArray(parsed.columns)) { - const sourceName = options?.sourceName ?? (typeof parsed.name === 'string' ? parsed.name : 'source'); - const path = - options?.connectionId && isSafeConnectionId(options.connectionId) - ? `semantic-layer/${options.connectionId}/${sourceName}.yaml` - : `${sourceName}.yaml`; - const legacyColumnPatchErrors = parsed.columns - .filter((column): column is Record => isRecord(column)) - .filter((column) => typeof column.name === 'string' && (!column.expr || !column.type)) - .map( - (column) => - `${path}: column '${column.name}' patches a manifest column but is in 'columns:' — move it to 'column_overrides:'`, - ); - if (legacyColumnPatchErrors.length > 0) { - return { valid: false, errors: legacyColumnPatchErrors }; - } - } const result = schema.parse(parsed); const errors: string[] = []; @@ -328,6 +317,7 @@ export async function writeLocalSlSource( ); } +/** @internal */ export async function readLocalSlSource( project: KtxLocalProject, input: { connectionId: string; sourceName: string }, @@ -348,6 +338,41 @@ export async function readLocalSlSource( } } +export async function resolveLocalSlSource( + project: KtxLocalProject, + input: { sourceName: string; connectionId?: string }, +): Promise { + if (input.connectionId !== undefined) { + const source = await readLocalSlSource(project, { + connectionId: input.connectionId, + sourceName: input.sourceName, + }); + return source ? { kind: 'found', source } : { kind: 'not-found' }; + } + + const summaries = await listLocalSlSources(project, {}); + const matches = summaries.filter((summary) => summary.name === input.sourceName); + if (matches.length === 0) { + return { kind: 'not-found' }; + } + if (matches.length > 1) { + return { + kind: 'ambiguous', + connectionIds: [...new Set(matches.map((match) => match.connectionId))].sort(), + }; + } + + const match = matches[0]; + if (match === undefined) { + return { kind: 'not-found' }; + } + const source = await readLocalSlSource(project, { + connectionId: match.connectionId, + sourceName: input.sourceName, + }); + return source ? { kind: 'found', source } : { kind: 'not-found' }; +} + export async function listLocalSlSources( project: KtxLocalProject, input: { connectionId?: string } = {}, diff --git a/packages/cli/src/context/sl/semantic-layer.service.ts b/packages/cli/src/context/sl/semantic-layer.service.ts index e6afdeaf..28bf826d 100644 --- a/packages/cli/src/context/sl/semantic-layer.service.ts +++ b/packages/cli/src/context/sl/semantic-layer.service.ts @@ -1082,17 +1082,14 @@ export class SemanticLayerService { static mapDialect(connectionType: string): string { const normalized = connectionType.toUpperCase(); const map: Record = { - POSTGRESQL: 'postgres', POSTGRES: 'postgres', BIGQUERY: 'bigquery', SNOWFLAKE: 'snowflake', MYSQL: 'mysql', SQLSERVER: 'tsql', - MSSQL: 'tsql', SQLITE: 'sqlite', DUCKDB: 'duckdb', CLICKHOUSE: 'clickhouse', - REDSHIFT: 'redshift', DATABRICKS: 'databricks', }; return map[normalized] ?? 'postgres'; @@ -1513,7 +1510,7 @@ export function composeOverlay(base: SemanticLayerSource, overlay: Record, field: string): string | n throw new Error(`sql analysis response has invalid optional string field ${field}`); } +function optionalNullableStringField(raw: Record, field: string): string | null { + const value = raw[field]; + if (value === null || value === undefined || typeof value === 'string') { + return value ?? null; + } + throw new Error(`sql analysis response has invalid optional nullable string field ${field}`); +} + function requiredStringArray(raw: Record, field: string): string[] { const value = raw[field]; if (!Array.isArray(value) || value.some((item) => typeof item !== 'string')) { @@ -175,10 +185,34 @@ function mapColumnsByClause(raw: Record): SqlAnalysisBatchResul return result; } +function requiredTableRef(raw: unknown, field: string): KtxTableRef { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error(`sql analysis response contains invalid table ref in ${field}`); + } + const record = raw as Record; + const name = record.name; + if (typeof name !== 'string' || name.length === 0) { + throw new Error(`sql analysis response table ref in ${field} is missing name`); + } + return { + catalog: optionalNullableStringField(record, 'catalog'), + db: optionalNullableStringField(record, 'db'), + name, + }; +} + +function requiredTableRefArray(raw: Record, field: string): KtxTableRef[] { + const value = raw[field]; + if (!Array.isArray(value)) { + throw new Error(`sql analysis response is missing table-ref[] field ${field}`); + } + return value.map((item, index) => requiredTableRef(item, `${field}.${index}`)); +} + function mapBatchResult(raw: Record): SqlAnalysisBatchResult { const error = optionalString(raw, 'error'); return { - tablesTouched: requiredStringArray(raw, 'tables_touched'), + tablesTouched: requiredTableRefArray(raw, 'tables_touched'), columnsByClause: mapColumnsByClause(raw), ...(error !== undefined ? { error } : {}), }; @@ -215,10 +249,11 @@ export function createHttpSqlAnalysisPort(options: HttpSqlAnalysisPortOptions): }); return mapResult(raw); }, - async analyzeBatch(items: SqlAnalysisBatchItem[], dialect: SqlAnalysisDialect) { + async analyzeBatch(items: SqlAnalysisBatchItem[], dialect: SqlAnalysisDialect, options?: SqlAnalysisBatchOptions) { const raw = await requestJson('/sql/analyze-batch', { dialect, items, + ...(options?.catalog ? { catalog: options.catalog } : {}), }); return mapBatchResponse(raw); }, diff --git a/packages/cli/src/context/sql-analysis/ports.ts b/packages/cli/src/context/sql-analysis/ports.ts index 887be605..898fca38 100644 --- a/packages/cli/src/context/sql-analysis/ports.ts +++ b/packages/cli/src/context/sql-analysis/ports.ts @@ -1,3 +1,5 @@ +import type { KtxTableRef } from '../scan/types.js'; + export type SqlAnalysisDialect = | 'bigquery' | 'snowflake' @@ -32,8 +34,20 @@ export interface SqlAnalysisBatchItem { sql: string; } +interface SqlAnalysisCatalogTable extends KtxTableRef { + columns?: string[]; +} + +interface SqlAnalysisCatalog { + tables: SqlAnalysisCatalogTable[]; +} + +export interface SqlAnalysisBatchOptions { + catalog?: SqlAnalysisCatalog; +} + export interface SqlAnalysisBatchResult { - tablesTouched: string[]; + tablesTouched: KtxTableRef[]; columnsByClause: Partial>; error?: string | null; } @@ -48,6 +62,7 @@ export interface SqlAnalysisPort { analyzeBatch( items: SqlAnalysisBatchItem[], dialect: SqlAnalysisDialect, + options?: SqlAnalysisBatchOptions, ): Promise>; validateReadOnly(sql: string, dialect: SqlAnalysisDialect): Promise; } diff --git a/packages/cli/src/context/wiki/local-knowledge.ts b/packages/cli/src/context/wiki/local-knowledge.ts index b7132b50..dd9b9ad7 100644 --- a/packages/cli/src/context/wiki/local-knowledge.ts +++ b/packages/cli/src/context/wiki/local-knowledge.ts @@ -201,6 +201,32 @@ export async function listLocalKnowledgePages( return pages.sort((left, right) => left.path.localeCompare(right.path)); } +/** + * List wiki page keys without reading or parsing file contents. + * + * Keys are derived purely from file paths, so this stays cheap enough for + * shell tab-completion (unlike `listLocalKnowledgePages`, which reads every + * page to populate summaries). + */ +export async function listLocalKnowledgePageKeys( + project: KtxLocalProject, + input: { userId?: string } = {}, +): Promise { + const userId = input.userId ?? 'local'; + const keys = new Set(); + for (const scope of ['GLOBAL', 'USER'] as const) { + const root = scope === 'GLOBAL' ? 'wiki/global' : `wiki/user/${assertSafePathToken('user id', userId)}`; + const listed = await project.fileStore.listFiles(root); + for (const path of listed.files.filter((file) => file.endsWith('.md'))) { + const key = keyFromKnowledgePath(path, scope, userId); + if (key) { + keys.add(key); + } + } + } + return [...keys].sort(); +} + function scorePage(page: LocalKnowledgePage, terms: string[]): number { const haystack = buildKnowledgeSearchText(page.key, page.summary, page.content, page.tags).toLowerCase(); return terms.some((term) => haystack.includes(term)) ? 3 : 0; diff --git a/packages/cli/src/database-tree-picker.ts b/packages/cli/src/database-tree-picker.ts index 6b357de6..6698a0d2 100644 --- a/packages/cli/src/database-tree-picker.ts +++ b/packages/cli/src/database-tree-picker.ts @@ -1,3 +1,4 @@ +import { parseDottedTableEntry } from './context/scan/enabled-tables.js'; import type { KtxTableListEntry } from './context/scan/types.js'; import type { KtxCliIo } from './cli-runtime.js'; import { profileMark } from './startup-profile.js'; @@ -73,7 +74,9 @@ export interface PickDatabaseScopeArgs { } function qualifiedTableId(entry: KtxTableListEntry): string { - return `${entry.schema}.${entry.name}`; + return entry.catalog !== null + ? `${entry.catalog}.${entry.schema}.${entry.name}` + : `${entry.schema}.${entry.name}`; } function tableTitle(entry: KtxTableListEntry): string { @@ -177,7 +180,8 @@ function schemasFromEnabledTables(enabledTables: readonly string[]): string[] { const seen = new Set(); const result: string[] = []; for (const qualified of enabledTables) { - const schema = qualified.split('.')[0] ?? ''; + const ref = parseDottedTableEntry(qualified); + const schema = ref?.db ?? ''; if (schema.length === 0 || seen.has(schema)) continue; seen.add(schema); result.push(schema); @@ -228,11 +232,14 @@ async function runStageTwoTreePicker(input: { ? initialSelectionForExisting(args.existing.enabledTables, byId) : initialSelectionFromDefaults(selectedSchemas, schemaIds); - const initialState = buildInitialState({ - tree, - existingSelectedIds: initialSelection, - skipEmptyAction: 'save-empty', - }); + const initialState = { + ...buildInitialState({ + tree, + existingSelectedIds: initialSelection, + skipEmptyAction: 'save-empty', + }), + expanded: new Set(schemaIds), + }; const schemaWordPlural = schemaCount === 1 ? args.schemaNoun : args.schemaNounPlural; const subtitleLines = [ diff --git a/packages/cli/src/error-message.ts b/packages/cli/src/error-message.ts new file mode 100644 index 00000000..8ec1355a --- /dev/null +++ b/packages/cli/src/error-message.ts @@ -0,0 +1,28 @@ +export function describeError(error: unknown): string { + if (!(error instanceof Error)) { + const text = String(error); + return text.length > 0 ? text : 'unknown error'; + } + const parts: string[] = []; + if (error.message.length > 0) { + parts.push(error.message); + } + const seen = new Set([error]); + let cause: unknown = error.cause; + while (cause && !seen.has(cause)) { + seen.add(cause); + if (cause instanceof Error) { + if (cause.message.length > 0) { + parts.push(cause.message); + } + cause = cause.cause; + } else { + const text = String(cause); + if (text.length > 0) { + parts.push(text); + } + break; + } + } + return parts.length > 0 ? parts.join(': ') : 'unknown error'; +} diff --git a/packages/cli/src/ingest-depth.ts b/packages/cli/src/ingest-depth.ts deleted file mode 100644 index 489c44e8..00000000 --- a/packages/cli/src/ingest-depth.ts +++ /dev/null @@ -1,76 +0,0 @@ -import type { KtxProjectConfig, KtxProjectConnectionConfig } from './context/project/config.js'; - -export type KtxDatabaseContextDepth = 'fast' | 'deep'; - -const KTX_DATABASE_DRIVER_IDS = new Set([ - 'sqlite', - 'postgres', - 'postgresql', - 'mysql', - 'clickhouse', - 'sqlserver', - 'bigquery', - 'snowflake', -]); - -export function normalizeConnectionDriver(connection: KtxProjectConnectionConfig): string { - return String(connection.driver ?? '') - .trim() - .toLowerCase(); -} - -export function isDatabaseDriver(driver: string): boolean { - return KTX_DATABASE_DRIVER_IDS.has(driver.trim().toLowerCase()); -} - -function connectionContextRecord(connection: KtxProjectConnectionConfig): Record { - const context = connection.context; - return typeof context === 'object' && context !== null && !Array.isArray(context) - ? (context as Record) - : {}; -} - -export function databaseContextDepth(connection: KtxProjectConnectionConfig): KtxDatabaseContextDepth | undefined { - const depth = connectionContextRecord(connection).depth; - return depth === 'fast' || depth === 'deep' ? depth : undefined; -} - -export function withDatabaseContextDepth( - connection: KtxProjectConnectionConfig, - depth: KtxDatabaseContextDepth, -): KtxProjectConnectionConfig { - return { - ...connection, - context: { - ...connectionContextRecord(connection), - depth, - }, - }; -} - -export function deepReadinessGaps(config: KtxProjectConfig): string[] { - const gaps: string[] = []; - if (config.llm.provider.backend === 'none' || !config.llm.models.default) { - gaps.push('model configuration'); - } - - if (config.scan.enrichment.mode !== 'llm') { - gaps.push('scan enrichment mode'); - } - - const embeddings = config.scan.enrichment.embeddings; - if ( - !embeddings || - embeddings.backend === 'none' || - !embeddings.model || - embeddings.dimensions <= 0 - ) { - gaps.push('scan embeddings'); - } - - return gaps; -} - -export function recommendedDatabaseContextDepth(config: KtxProjectConfig): KtxDatabaseContextDepth { - return deepReadinessGaps(config).length === 0 ? 'deep' : 'fast'; -} diff --git a/packages/cli/src/ingest.ts b/packages/cli/src/ingest.ts index 3615f401..319c3d1b 100644 --- a/packages/cli/src/ingest.ts +++ b/packages/cli/src/ingest.ts @@ -2,7 +2,7 @@ import { buildMemoryFlowViewModel } from './context/ingest/memory-flow/view-mode import { createMemoryFlowLiveBuffer, sanitizeMemoryFlowError } from './context/ingest/memory-flow/live-buffer.js'; import { formatMemoryFlowFinalSummary } from './context/ingest/memory-flow/summary.js'; import { getLatestLocalIngestStatus, getLocalIngestStatus, type LocalMetabaseFanoutResult, type LocalMetabaseFanoutProgress, type RunLocalIngestOptions, runLocalIngest, runLocalMetabaseIngest } from './context/ingest/local-ingest.js'; -import { type IngestReportSnapshot, savedMemoryCountsForReport } from './context/ingest/reports.js'; +import { type IngestReportSnapshot, ingestReportOutcome, savedMemoryCountsForReport } from './context/ingest/reports.js'; import { ingestReportToMemoryFlowReplay } from './context/ingest/memory-flow/events.js'; import type { MemoryFlowEvent, MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; import { renderMemoryFlowReplay } from './context/ingest/memory-flow/render.js'; @@ -78,6 +78,7 @@ export interface KtxIngestDeps { readReportFile?: typeof readIngestReportSnapshotFile; renderStoredMemoryFlow?: typeof renderMemoryFlowTui; startLiveMemoryFlow?: typeof startLiveMemoryFlowTui; + abortSignal?: AbortSignal; env?: NodeJS.ProcessEnv; localIngestOptions?: Pick< RunLocalIngestOptions, @@ -93,8 +94,21 @@ export interface KtxIngestDeps { runtimeIo?: KtxIngestIo; } -function reportStatus(report: IngestReportSnapshot): 'done' | 'error' { - return report.body.status === 'failed' || report.body.failedWorkUnits.length > 0 ? 'error' : 'done'; +function createCliAbortSignal(): { signal: AbortSignal; dispose: () => void } { + const controller = new AbortController(); + let interrupted = false; + const onSigint = () => { + if (interrupted) { + process.exit(130); + } + interrupted = true; + controller.abort(new DOMException('Aborted', 'AbortError')); + }; + process.on('SIGINT', onSigint); + return { + signal: controller.signal, + dispose: () => process.off('SIGINT', onSigint), + }; } const REPORT_SOURCE_LABELS = new Map([ @@ -193,7 +207,7 @@ function writeReportStatus(report: IngestReportSnapshot, io: KtxIngestIo): void if (report.body.tracePath) { io.stdout.write(`Trace: ${report.body.tracePath}\n`); } - io.stdout.write(`Status: ${reportStatus(report)}\n`); + io.stdout.write(`Status: ${ingestReportOutcome(report)}\n`); io.stdout.write(`Source: ${reportSourceLabel(report.sourceKey)}\n`); io.stdout.write(`Connection: ${report.connectionId}\n`); io.stdout.write(`Sync: ${report.body.syncId}\n`); @@ -222,7 +236,7 @@ function writeMetabaseFanoutStatus(result: LocalMetabaseFanoutResult, io: KtxIng }, { wikiCount: 0, slCount: 0 }, ); - io.stdout.write(`Metabase fan-out: ${result.status}\n`); + io.stdout.write(`Metabase fanout: ${result.status}\n`); io.stdout.write(`Source: ${result.metabaseConnectionId}\n`); io.stdout.write(`Children: ${result.children.length}\n`); if (result.totals) { @@ -231,7 +245,7 @@ function writeMetabaseFanoutStatus(result: LocalMetabaseFanoutResult, io: KtxIng } io.stdout.write(`Saved memory: ${counts.wikiCount} wiki, ${counts.slCount} SL\n`); for (const child of result.children) { - const status = reportStatus(child.report); + const status = ingestReportOutcome(child.report); io.stdout.write( `- target=${child.targetConnectionId} database=${child.metabaseDatabaseId} status=${status} job=${child.jobId} report=${child.report.id}\n`, ); @@ -368,6 +382,12 @@ function plainIngestEventProgress( message: event.message, ...(event.transient !== undefined ? { transient: event.transient } : {}), }; + case 'rate_limit_wait': + return { + percent: 50, + message: `Rate-limited (${event.provider}${event.rateLimitType ? ` ${event.rateLimitType}` : ''}); resuming in ${Math.ceil(event.remainingMs / 1_000)}s`, + transient: true, + }; case 'work_unit_started': { const total = plannedWorkUnitCountThrough(snapshot, eventIndex); const ordinal = workUnitOrdinalThrough(snapshot, eventIndex, event.unitKey); @@ -595,7 +615,7 @@ function initialRunMemoryFlowInput( } function finalRunMemoryFlowInput(snapshot: MemoryFlowReplayInput, report: IngestReportSnapshot): MemoryFlowReplayInput { - const status = reportStatus(report); + const status = ingestReportOutcome(report) === 'error' ? 'error' : 'done'; return { ...snapshot, runId: report.runId, @@ -719,7 +739,7 @@ export async function runKtxIngest( localIngestOptions.queryExecutor ?? (deps.createQueryExecutor ?? createKtxCliIngestQueryExecutor)(ingestProject); if (args.adapter === 'metabase' && args.sourceDir) { - throw new Error('source-dir uploads are not supported for the Metabase fan-out adapter'); + throw new Error('source-dir uploads are not supported for the Metabase fanout adapter'); } if (args.adapter === 'metabase') { const executeMetabaseFanout = deps.runLocalMetabaseIngest ?? runLocalMetabaseIngest; @@ -754,6 +774,8 @@ export async function runKtxIngest( ); plainProgress?.start(); structuredProgress?.start(); + const cliAbort = deps.abortSignal ? null : createCliAbortSignal(); + const abortSignal = deps.abortSignal ?? cliAbort?.signal; let result: LocalMetabaseFanoutResult; try { result = await executeMetabaseFanout({ @@ -767,6 +789,7 @@ export async function runKtxIngest( embeddingProvider, ...(memoryFlow ? { memoryFlow } : {}), ...(progress ? { progress } : {}), + ...(abortSignal ? { abortSignal } : {}), }); plainProgress?.flush(); if (args.outputMode === 'json') { @@ -776,8 +799,9 @@ export async function runKtxIngest( } } finally { plainProgress?.flush(); + cliAbort?.dispose(); } - return result.status === 'all_succeeded' ? 0 : 1; + return result.status === 'all_failed' ? 1 : 0; } const jobId = deps.jobIdFactory?.(); @@ -824,6 +848,8 @@ export async function runKtxIngest( plainProgress?.start(); structuredProgress?.start(); + const cliAbort = deps.abortSignal ? null : createCliAbortSignal(); + const abortSignal = deps.abortSignal ?? cliAbort?.signal; try { const result = await executeLocalIngest({ @@ -840,13 +866,14 @@ export async function runKtxIngest( embeddingProvider, ...(args.debugLlmRequestFile ? { llmDebugRequestFile: args.debugLlmRequestFile } : {}), ...(memoryFlow ? { memoryFlow } : {}), + ...(abortSignal ? { abortSignal } : {}), }); if (shouldUseLiveViz && memoryFlow) { latestMemoryFlowSnapshot = finalRunMemoryFlowInput(memoryFlow.snapshot(), result.report); liveTui?.close(); liveTui = null; io.stdout.write(formatMemoryFlowFinalSummary(latestMemoryFlowSnapshot)); - return reportStatus(result.report) === 'done' ? 0 : 1; + return ingestReportOutcome(result.report) === 'error' ? 1 : 0; } plainProgress?.flush(); await writeReportRecord(result.report, runOutputMode, io, { @@ -854,10 +881,11 @@ export async function runKtxIngest( renderStoredMemoryFlow: deps.renderStoredMemoryFlow, env, }); - return reportStatus(result.report) === 'done' ? 0 : 1; + return ingestReportOutcome(result.report) === 'error' ? 1 : 0; } finally { plainProgress?.flush(); liveTui?.close(); + cliAbort?.dispose(); } } diff --git a/packages/cli/src/io/buffered-command-io.ts b/packages/cli/src/io/buffered-command-io.ts new file mode 100644 index 00000000..6d16f385 --- /dev/null +++ b/packages/cli/src/io/buffered-command-io.ts @@ -0,0 +1,35 @@ +import type { KtxCliIo } from '../cli-runtime.js'; + +export interface BufferedCommandIo extends KtxCliIo { + stdoutText(): string; + stderrText(): string; +} + +/** + * Captures stdout/stderr from a command (e.g. `runKtxConnection`) into buffers + * instead of the terminal. Callers decide whether to flush the captured text to + * the user or discard it. + */ +export function createBufferedCommandIo(): BufferedCommandIo { + let stdout = ''; + let stderr = ''; + return { + stdout: { + isTTY: false, + write(chunk: string) { + stdout += chunk; + }, + }, + stderr: { + write(chunk: string) { + stderr += chunk; + }, + }, + stdoutText() { + return stdout; + }, + stderrText() { + return stderr; + }, + }; +} diff --git a/packages/cli/src/knowledge.ts b/packages/cli/src/knowledge.ts index d6246fef..346d3d9a 100644 --- a/packages/cli/src/knowledge.ts +++ b/packages/cli/src/knowledge.ts @@ -1,7 +1,13 @@ import { KtxIngestEmbeddingPortAdapter } from './context/llm/embedding-port.js'; import type { KtxEmbeddingPort } from './context/core/embedding.js'; import { loadKtxProject } from './context/project/project.js'; -import { type LocalKnowledgeSearchResult, type LocalKnowledgeSummary, listLocalKnowledgePages, searchLocalKnowledgePages as defaultSearchLocalKnowledgePages } from './context/wiki/local-knowledge.js'; +import { + type LocalKnowledgeSearchResult, + type LocalKnowledgeSummary, + listLocalKnowledgePages, + readLocalKnowledgePage, + searchLocalKnowledgePages as defaultSearchLocalKnowledgePages, +} from './context/wiki/local-knowledge.js'; import { resolveProjectEmbeddingProvider, type EmbeddingProviderResolution, @@ -22,7 +28,8 @@ export type KtxKnowledgeArgs = limit?: number; debug?: boolean; cliVersion: string; - }; + } + | { command: 'read'; projectDir: string; key: string; userId: string }; type KtxKnowledgeIo = import('./cli-runtime.js').KtxCliIo; @@ -128,6 +135,15 @@ export async function runKtxKnowledge( }); return 0; } + if (args.command === 'read') { + const page = await readLocalKnowledgePage(project, { key: args.key, userId: args.userId }); + if (!page) { + throw new Error(`No wiki page found for key '${args.key}'`); + } + const raw = await project.fileStore.readFile(page.path); + io.stdout.write(raw.content); + return 0; + } if (args.command === 'search') { const embeddingService = await wikiSearchEmbeddingService(project, deps, { cliVersion: args.cliVersion }, io); const search = deps.searchLocalKnowledgePages ?? defaultSearchLocalKnowledgePages; diff --git a/packages/cli/src/llm/embedding-health.ts b/packages/cli/src/llm/embedding-health.ts index 2c8ac53d..47d3f14f 100644 --- a/packages/cli/src/llm/embedding-health.ts +++ b/packages/cli/src/llm/embedding-health.ts @@ -1,3 +1,4 @@ +import { describeError } from '../error-message.js'; import { createKtxEmbeddingProvider, type KtxEmbeddingProviderDeps } from './embedding-provider.js'; import type { KtxEmbeddingConfig } from './types.js'; @@ -48,7 +49,6 @@ export async function runKtxEmbeddingHealthCheck( } return { ok: true }; } catch (error) { - const message = error instanceof Error ? error.message : String(error); - return { ok: false, message: redactHealthCheckMessage(message, config) }; + return { ok: false, message: redactHealthCheckMessage(describeError(error), config) }; } } diff --git a/packages/cli/src/llm/types.ts b/packages/cli/src/llm/types.ts index 3f7f67e2..a190b1c0 100644 --- a/packages/cli/src/llm/types.ts +++ b/packages/cli/src/llm/types.ts @@ -3,7 +3,7 @@ import type { LanguageModel, TelemetrySettings, ToolCallRepairFunction, ToolSet export const KTX_MODEL_ROLES = ['default', 'triage', 'candidateExtraction', 'curator', 'reconcile', 'repair'] as const; export type KtxModelRole = (typeof KTX_MODEL_ROLES)[number]; -type KtxLlmBackend = 'anthropic' | 'vertex' | 'gateway' | 'claude-code'; +type KtxLlmBackend = 'anthropic' | 'vertex' | 'gateway' | 'claude-code' | 'codex'; export type KtxPromptCacheTtl = '5m' | '1h'; type KtxJsonValue = diff --git a/packages/cli/src/local-adapters.ts b/packages/cli/src/local-adapters.ts index cfc57adc..3e3b0486 100644 --- a/packages/cli/src/local-adapters.ts +++ b/packages/cli/src/local-adapters.ts @@ -12,10 +12,10 @@ import { isKtxSqliteConnectionConfig } from './connectors/sqlite/connector.js'; import { createSqlServerLiveDatabaseIntrospection } from './connectors/sqlserver/live-database-introspection.js'; import { isKtxSqlServerConnectionConfig } from './connectors/sqlserver/connector.js'; import { BigQueryHistoricSqlQueryHistoryReader } from './context/ingest/adapters/historic-sql/bigquery-query-history-reader.js'; -import { queryHistoryDialectForConnection } from './context/ingest/adapters/historic-sql/connection-dialect.js'; +import { historicSqlDialectForConnectionDriver } from './context/ingest/adapters/historic-sql/connection-dialect.js'; import { createDaemonLiveDatabaseIntrospection } from './context/ingest/adapters/live-database/daemon-introspection.js'; import { createDefaultLocalIngestAdapters, type DefaultLocalIngestAdaptersOptions } from './context/ingest/local-adapters.js'; -import type { HistoricSqlReader } from './context/ingest/adapters/historic-sql/types.js'; +import type { HistoricSqlDialect, HistoricSqlReader } from './context/ingest/adapters/historic-sql/types.js'; import type { LiveDatabaseIntrospectionOptions, LiveDatabaseIntrospectionPort, @@ -31,7 +31,7 @@ import { createManagedDaemonLookerTableIdentifierParser, createManagedDaemonSqlAnalysisPort, managedDaemonDatabaseIntrospectionOptions, - type ManagedPythonCoreDaemonOptions, + type ManagedPythonDaemonHttpOptions, } from './managed-python-http.js'; import type { KtxOperationalLogger } from './io/logger.js'; import { resolveKtxConfigReference } from './context/core/config-reference.js'; @@ -161,10 +161,17 @@ export interface KtxCliLocalIngestAdaptersOptions extends DefaultLocalIngestAdap historicSqlConnectionId?: string; sqlAnalysis?: SqlAnalysisPort; sqlAnalysisUrl?: string; - managedDaemon?: ManagedPythonCoreDaemonOptions; + managedDaemon?: ManagedPythonDaemonHttpOptions; logger?: KtxOperationalLogger; } +export interface KtxCliHistoricSqlRuntime { + dialect: HistoricSqlDialect; + sqlAnalysis: SqlAnalysisPort; + reader: HistoricSqlReader; + queryClient: unknown; +} + function createEphemeralPostgresHistoricSqlClient(project: KtxLocalProject, connectionId: string) { const connection = project.config.connections[connectionId] as KtxPostgresConnectionConfig | undefined; const inputDriver = connection?.driver ?? 'unknown'; @@ -262,13 +269,21 @@ function bigQueryRegion(connection: KtxBigQueryConnectionConfig): string { : 'us'; } -function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCliLocalIngestAdaptersOptions) { +function historicSqlOptionsForLocalRun( + project: KtxLocalProject, + options: KtxCliLocalIngestAdaptersOptions, +): KtxCliHistoricSqlRuntime | undefined { const connectionId = options.historicSqlConnectionId; if (!connectionId) { return undefined; } const connection = project.config.connections[connectionId]; - const dialect = queryHistoryDialectForConnection(connection); + // historicSqlConnectionId is only set when query history was explicitly + // requested for this run (e.g. `--query-history`), so resolve the dialect from + // driver capability rather than the persisted context.queryHistory.enabled + // flag — otherwise the adapter is missing and findAdapter('historic-sql') + // throws even though the run asked for it. + const dialect = historicSqlDialectForConnectionDriver(connection); if (!dialect) { return undefined; } @@ -280,6 +295,7 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli if (dialect === 'postgres') { return { ...base, + dialect, reader: new PostgresPgssReader() satisfies HistoricSqlReader, queryClient: createEphemeralPostgresHistoricSqlClient(project, connectionId), }; @@ -292,6 +308,7 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli } return { ...base, + dialect, reader: new BigQueryHistoricSqlQueryHistoryReader({ projectId: bigQueryProjectId(connection, process.env), region: bigQueryRegion(connection), @@ -302,6 +319,7 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli return { ...base, + dialect, reader: new SnowflakeHistoricSqlQueryHistoryReader() satisfies HistoricSqlReader, queryClient: { async executeQuery(query: string) { @@ -313,11 +331,24 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli }; } +export function createKtxCliHistoricSqlRuntime( + project: KtxLocalProject, + connectionId: string, + options: KtxCliLocalIngestAdaptersOptions = {}, +): KtxCliHistoricSqlRuntime | undefined { + return historicSqlOptionsForLocalRun(project, { + ...options, + historicSqlConnectionId: connectionId, + }); +} + export function createKtxCliLocalIngestAdapters( project: KtxLocalProject, options: KtxCliLocalIngestAdaptersOptions = {}, ): SourceAdapter[] { - const historicSql = historicSqlOptionsForLocalRun(project, options); + const historicSql = options.historicSqlConnectionId + ? createKtxCliHistoricSqlRuntime(project, options.historicSqlConnectionId, options) + : undefined; const base = createDefaultLocalIngestAdapters(project, { ...options, databaseIntrospection: ktxCliDaemonDatabaseIntrospectionOptions(options), diff --git a/packages/cli/src/local-scan-connectors.ts b/packages/cli/src/local-scan-connectors.ts index 4f763be5..4d98bc0c 100644 --- a/packages/cli/src/local-scan-connectors.ts +++ b/packages/cli/src/local-scan-connectors.ts @@ -1,7 +1,11 @@ +import { + getDriverRegistration, + listSupportedDrivers, +} from './context/connections/drivers.js'; import type { KtxLocalProject } from './context/project/project.js'; import type { KtxScanConnector } from './context/scan/types.js'; -const SUPPORTED_DRIVERS = 'sqlite, postgres, mysql, clickhouse, sqlserver, bigquery, snowflake'; +const SUPPORTED_DRIVERS = listSupportedDrivers().join(', '); export async function createKtxCliScanConnector( project: KtxLocalProject, @@ -17,58 +21,23 @@ export async function createKtxCliScanConnector( `Connection "${connectionId}" has no \`driver\` field in ktx.yaml. Supported drivers: ${SUPPORTED_DRIVERS}.`, ); } - if (driver === 'sqlite' || driver === 'sqlite3') { - const { KtxSqliteScanConnector, isKtxSqliteConnectionConfig } = await import('./connectors/sqlite/connector.js');; - if (!isKtxSqliteConnectionConfig(connection)) { - throw invalidConnectionConfigError(connectionId, driver); - } - return new KtxSqliteScanConnector({ connectionId, connection, projectDir: project.projectDir }); + + const registration = getDriverRegistration(driver); + if (!registration) { + throw new Error( + `Connection "${connectionId}" uses driver "${driver}", which has no native standalone KTX scan connector. Supported drivers: ${SUPPORTED_DRIVERS}.`, + ); } - if (driver === 'postgres' || driver === 'postgresql') { - const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('./connectors/postgres/connector.js');; - if (!isKtxPostgresConnectionConfig(connection)) { - throw invalidConnectionConfigError(connectionId, driver); - } - return new KtxPostgresScanConnector({ connectionId, connection }); + + const connectorModule = await registration.load(); + if (!connectorModule.isConnectionConfig(connection)) { + throw invalidConnectionConfigError(connectionId, driver); } - if (driver === 'mysql') { - const { KtxMysqlScanConnector, isKtxMysqlConnectionConfig } = await import('./connectors/mysql/connector.js');; - if (!isKtxMysqlConnectionConfig(connection)) { - throw invalidConnectionConfigError(connectionId, driver); - } - return new KtxMysqlScanConnector({ connectionId, connection }); - } - if (driver === 'clickhouse') { - const { KtxClickHouseScanConnector, isKtxClickHouseConnectionConfig } = await import('./connectors/clickhouse/connector.js');; - if (!isKtxClickHouseConnectionConfig(connection)) { - throw invalidConnectionConfigError(connectionId, driver); - } - return new KtxClickHouseScanConnector({ connectionId, connection }); - } - if (driver === 'sqlserver') { - const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('./connectors/sqlserver/connector.js');; - if (!isKtxSqlServerConnectionConfig(connection)) { - throw invalidConnectionConfigError(connectionId, driver); - } - return new KtxSqlServerScanConnector({ connectionId, connection }); - } - if (driver === 'bigquery') { - const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('./connectors/bigquery/connector.js');; - if (!isKtxBigQueryConnectionConfig(connection)) { - throw invalidConnectionConfigError(connectionId, driver); - } - return new KtxBigQueryScanConnector({ connectionId, connection }); - } - if (driver === 'snowflake') { - const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/connector.js');; - if (!isKtxSnowflakeConnectionConfig(connection)) { - throw invalidConnectionConfigError(connectionId, driver); - } - return new KtxSnowflakeScanConnector({ connectionId, connection, projectDir: project.projectDir }); - } - throw new Error( - `Connection "${connectionId}" uses driver "${driver}", which has no native standalone KTX scan connector. Supported drivers: ${SUPPORTED_DRIVERS}.`, - ); + return connectorModule.createScanConnector({ + connectionId, + connection, + projectDir: project.projectDir, + }); } function invalidConnectionConfigError(connectionId: string, driver: string): Error { diff --git a/packages/cli/src/managed-local-embeddings.ts b/packages/cli/src/managed-local-embeddings.ts index b178be47..768648c1 100644 --- a/packages/cli/src/managed-local-embeddings.ts +++ b/packages/cli/src/managed-local-embeddings.ts @@ -1,5 +1,6 @@ import type { KtxEmbeddingConfig } from './llm/types.js'; import type { KtxCliIo } from './cli-runtime.js'; +import { writePrefixedLines } from './clack.js'; import { ensureManagedPythonCommandRuntime, type KtxManagedPythonInstallPolicy, @@ -73,7 +74,7 @@ export async function ensureManagedLocalEmbeddingsDaemon( }); const verb = daemon.status === 'started' ? 'Started' : 'Using'; - options.io.stderr.write(`${verb} KTX daemon: ${daemon.baseUrl}\n`); + writePrefixedLines((chunk) => options.io.stderr.write(chunk), `${verb} KTX daemon: ${daemon.baseUrl}`); return { baseUrl: daemon.baseUrl, diff --git a/packages/cli/src/managed-python-daemon.ts b/packages/cli/src/managed-python-daemon.ts index 7bc92e14..4e56ca47 100644 --- a/packages/cli/src/managed-python-daemon.ts +++ b/packages/cli/src/managed-python-daemon.ts @@ -4,6 +4,7 @@ import { createServer } from 'node:net'; import { setTimeout as delay } from 'node:timers/promises'; import { promisify } from 'node:util'; import { z } from 'zod'; +import { describeError } from './error-message.js'; import { installManagedPythonRuntime, managedPythonDaemonLayout, @@ -16,6 +17,17 @@ import { } from './managed-python-runtime.js'; import { sanitizeChildProxyEnv } from './proxy-env.js'; +export class ManagedPythonDaemonStartError extends Error { + readonly detail: string; + readonly stderrLog: string; + constructor(detail: string, stderrLog: string) { + super(`KTX daemon failed to start: ${detail}. stderr: ${stderrLog}`); + this.name = 'ManagedPythonDaemonStartError'; + this.detail = detail; + this.stderrLog = stderrLog; + } +} + export interface ManagedPythonDaemonState { schemaVersion: 1; pid: number; @@ -237,7 +249,7 @@ async function healthOk(input: { } return { ok: true }; } catch (error) { - return { ok: false, detail: error instanceof Error ? error.message : String(error) }; + return { ok: false, detail: describeError(error) }; } } @@ -328,7 +340,7 @@ async function waitForHealth(input: { return; } lastDetail = finalHealth.detail; - throw new Error(`KTX daemon failed to start: ${lastDetail}. stderr: ${input.state.stderrLog}`); + throw new ManagedPythonDaemonStartError(lastDetail, input.state.stderrLog); } async function removeState(layout: ManagedPythonDaemonLayout): Promise { @@ -721,13 +733,21 @@ export async function startManagedPythonDaemon( stdoutLog: layout.daemonStdoutPath, stderrLog: layout.daemonStderrPath, }; - await waitForHealth({ - state, - cliVersion: options.cliVersion, - fetch: fetchImpl, - timeoutMs: options.startupTimeoutMs ?? 10_000, - pollIntervalMs: options.pollIntervalMs ?? 100, - }); + try { + await waitForHealth({ + state, + cliVersion: options.cliVersion, + fetch: fetchImpl, + timeoutMs: options.startupTimeoutMs ?? 30_000, + pollIntervalMs: options.pollIntervalMs ?? 100, + }); + } catch (error) { + if (processAlive(state.pid)) { + killProcess(state.pid); + } + await removeState(layout); + throw error; + } await writeState(layout.daemonStatePath, state); return { status: 'started', layout, state, baseUrl: baseUrl(state) }; } finally { diff --git a/packages/cli/src/managed-python-http.ts b/packages/cli/src/managed-python-http.ts index 0c9b24b3..728aa3ca 100644 --- a/packages/cli/src/managed-python-http.ts +++ b/packages/cli/src/managed-python-http.ts @@ -7,6 +7,7 @@ import type { LookerTableIdentifierParser } from './context/ingest/adapters/look import { createHttpSqlAnalysisPort, type KtxSqlAnalysisHttpJsonRunner } from './context/sql-analysis/http-sql-analysis-port.js'; import type { SqlAnalysisPort } from './context/sql-analysis/ports.js'; import type { KtxCliIo } from './cli-runtime.js'; +import { writePrefixedLines } from './clack.js'; import { ensureManagedPythonCommandRuntime, type KtxManagedPythonInstallPolicy, @@ -137,7 +138,7 @@ export function createManagedPythonDaemonBaseUrlResolver( force: false, }); const verb = daemon.status === 'started' ? 'Started' : 'Using existing'; - options.io.stderr.write(`${verb} KTX daemon: ${daemon.baseUrl}\n`); + writePrefixedLines((chunk) => options.io.stderr.write(chunk), `${verb} KTX daemon: ${daemon.baseUrl}`); cachedBaseUrl = daemon.baseUrl; return cachedBaseUrl; }; diff --git a/packages/cli/src/next-steps.ts b/packages/cli/src/next-steps.ts index 80a1b441..b6726e3c 100644 --- a/packages/cli/src/next-steps.ts +++ b/packages/cli/src/next-steps.ts @@ -70,8 +70,7 @@ export function formatSetupNextStepLines(state: KtxSetupNextStepState, indent = if (!state.contextReady) { return [ - `${indent}Build KTX context next.`, - `${indent}Run ingest to build database schema context before context-source ingest.`, + `${indent}Setup is complete. The only step left is to build context for your agents.`, ...commandLines(KTX_CONTEXT_BUILD_COMMANDS, indent), ]; } diff --git a/packages/cli/src/progress-port-adapter.ts b/packages/cli/src/progress-port-adapter.ts new file mode 100644 index 00000000..1f73636b --- /dev/null +++ b/packages/cli/src/progress-port-adapter.ts @@ -0,0 +1,29 @@ +import type { KtxProgressPort, KtxProgressUpdateOptions } from './context/scan/types.js'; +import type { KtxIngestProgressUpdate } from './ingest.js'; + +export interface AggregateProgressState { + progress: number; +} + +export function createAggregateProgressPort( + onProgress: (update: KtxIngestProgressUpdate) => void, + state: AggregateProgressState = { progress: 0 }, + start = 0, + weight = 1, +): KtxProgressPort { + return { + async update(value: number, message?: string, options?: KtxProgressUpdateOptions): Promise { + const absoluteValue = start + Math.max(0, Math.min(1, value)) * weight; + state.progress = Math.max(state.progress, Math.min(1, absoluteValue)); + if (!message) return; + onProgress({ + percent: Math.max(0, Math.min(100, Math.round(state.progress * 100))), + message, + ...(options?.transient !== undefined ? { transient: options.transient } : {}), + }); + }, + startPhase(phaseWeight: number): KtxProgressPort { + return createAggregateProgressPort(onProgress, state, state.progress, weight * phaseWeight); + }, + }; +} diff --git a/packages/cli/src/public-ingest-copy.ts b/packages/cli/src/public-ingest-copy.ts index be1206c1..86423f74 100644 --- a/packages/cli/src/public-ingest-copy.ts +++ b/packages/cli/src/public-ingest-copy.ts @@ -12,7 +12,7 @@ const DATABASE_INGEST_REPLACEMENTS: Array<[RegExp, string]> = [ 'Database enrichment failed after schema context completed', ], [/\bstructural scan\b/gi, 'schema context'], - [/\benriched scan\b/gi, 'deep database ingest'], + [/\benriched scan\b/gi, 'database ingest'], [/\bscan results\b/gi, 'database context'], ]; diff --git a/packages/cli/src/public-ingest.ts b/packages/cli/src/public-ingest.ts index ce6c6344..07f805b8 100644 --- a/packages/cli/src/public-ingest.ts +++ b/packages/cli/src/public-ingest.ts @@ -1,35 +1,37 @@ import { getKtxCliPackageInfo } from './cli-runtime.js'; import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; -import type { KtxProjectConnectionConfig } from './context/project/config.js'; +import type { KtxProjectConfig, KtxProjectConnectionConfig } from './context/project/config.js'; import type { KtxProgressPort } from './context/scan/types.js'; import type { KtxCliIo } from './index.js'; import type { KtxIngestArgs, KtxIngestDeps, KtxIngestProgressUpdate } from './ingest.js'; -import { - type KtxDatabaseContextDepth, - databaseContextDepth, - deepReadinessGaps, - isDatabaseDriver, - normalizeConnectionDriver, -} from './ingest-depth.js'; +import { isDatabaseDriver, normalizeConnectionDriver } from './connection-drivers.js'; +import { resolveQueryHistoryScopeFloor } from './context/ingest/adapters/historic-sql/scope-floor.js'; import { ensureManagedPythonCommandRuntime, type KtxManagedPythonInstallPolicy, type ManagedPythonCommandRuntime, } from './managed-python-command.js'; import type { KtxRuntimeFeature } from './managed-python-runtime.js'; -import { publicIngestOutputLine } from './public-ingest-copy.js'; +import { + publicDatabaseIngestMessage, + publicIngestOutputLine, + publicQueryHistoryMessage, +} from './public-ingest-copy.js'; +import { createAggregateProgressPort } from './progress-port-adapter.js'; import { resolvePublicIngestRuntimeRequirements } from './runtime-requirements.js'; import type { KtxScanArgs, KtxScanDeps } from './scan.js'; +import type { KtxTableRef } from './context/scan/types.js'; import { profileMark } from './startup-profile.js'; import { isDemoConnection } from './telemetry/demo-detect.js'; -import { emitProjectStackSnapshot, emitTelemetryEvent } from './telemetry/index.js'; +import { emitProjectStackSnapshot, emitTelemetryEvent, reportException } from './telemetry/index.js'; +import { collectTelemetryRedactionSecrets } from './telemetry/redaction-secrets.js'; +import { formatErrorDetail } from './telemetry/scrubber.js'; profileMark('module:public-ingest'); type KtxPublicIngestStepName = 'database-schema' | 'query-history' | 'source-ingest' | 'memory-update'; type KtxPublicIngestStepStatus = 'done' | 'skipped' | 'failed' | 'not-run'; type KtxPublicIngestInputMode = 'auto' | 'disabled'; -type KtxPublicIngestDepth = KtxDatabaseContextDepth; type KtxPublicIngestQueryHistoryFlag = 'default' | 'enabled' | 'disabled'; type HistoricSqlDialect = 'postgres' | 'bigquery' | 'snowflake'; @@ -41,7 +43,6 @@ export type KtxPublicIngestArgs = all: boolean; json: boolean; inputMode: KtxPublicIngestInputMode; - depth?: KtxPublicIngestDepth; queryHistory?: KtxPublicIngestQueryHistoryFlag; queryHistoryWindowDays?: number; scanMode?: Extract['mode']; @@ -58,7 +59,6 @@ export interface KtxPublicIngestPlanTarget { sourceDir?: string; debugCommand: string; steps: KtxPublicIngestStepName[]; - databaseDepth?: KtxPublicIngestDepth; detectRelationships?: boolean; preflightFailure?: string; queryHistory?: { @@ -67,7 +67,6 @@ export interface KtxPublicIngestPlanTarget { windowDays?: number; pullConfig?: Record; unsupported?: boolean; - skippedStoredByFast?: boolean; }; } @@ -121,7 +120,6 @@ interface KtxPublicContextBuildArgs { inputMode: 'auto' | 'disabled'; targetConnectionId?: string; all?: boolean; - depth?: KtxPublicIngestDepth; queryHistory?: KtxPublicIngestQueryHistoryFlag; queryHistoryWindowDays?: number; scanMode?: Extract['mode']; @@ -134,16 +132,25 @@ const sourceAdapterByDriver = new Map([ ['metabase', 'metabase'], ['local_metabase', 'metabase'], ['looker', 'looker'], - ['local_looker', 'looker'], ['notion', 'notion'], ['metricflow', 'metricflow'], ['dbt', 'dbt'], ['lookml', 'lookml'], ]); +export function publicProgressMessage(message: string, target: KtxPublicIngestPlanTarget): string { + let current = message; + if (target.operation === 'database-ingest') { + current = publicDatabaseIngestMessage(current); + } + if (target.steps.includes('query-history')) { + current = publicQueryHistoryMessage(current, target.connectionId); + } + return current; +} + const queryHistoryDialectByDriver = new Map([ ['postgres', 'postgres'], - ['postgresql', 'postgres'], ['bigquery', 'bigquery'], ['snowflake', 'snowflake'], ]); @@ -156,7 +163,6 @@ interface KtxUnsupportedQueryHistoryWarning { interface KtxPublicIngestWarningAccumulator { warnings: string[]; - ignoredDepthForSources: string[]; ignoredQueryHistoryForSources: string[]; unsupportedQueryHistoryForDatabases: KtxUnsupportedQueryHistoryWarning[]; } @@ -164,7 +170,6 @@ interface KtxPublicIngestWarningAccumulator { function createWarningAccumulator(): KtxPublicIngestWarningAccumulator { return { warnings: [], - ignoredDepthForSources: [], ignoredQueryHistoryForSources: [], unsupportedQueryHistoryForDatabases: [], }; @@ -235,7 +240,6 @@ function finalizeWarnings( accumulator: KtxPublicIngestWarningAccumulator, args: { all: boolean; - depth?: KtxPublicIngestDepth; queryHistory?: KtxPublicIngestQueryHistoryFlag; queryHistoryWindowDays?: number; }, @@ -244,11 +248,6 @@ function finalizeWarnings( ...accumulator.warnings, ...unsupportedQueryHistoryWarnings(accumulator.unsupportedQueryHistoryForDatabases, args.all), ]; - const depthOption = args.depth ? `--${args.depth}` : null; - if (depthOption) { - const warning = sourceIgnoredWarning(depthOption, accumulator.ignoredDepthForSources, args.all); - if (warning) warnings.push(warning); - } if (args.queryHistory === 'enabled' || args.queryHistoryWindowDays !== undefined) { const warning = sourceIgnoredWarning('--query-history', accumulator.ignoredQueryHistoryForSources, args.all); if (warning) warnings.push(warning); @@ -285,36 +284,39 @@ function positiveInteger(value: unknown): number | undefined { return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : undefined; } -function enabledTablesForConnection(connection: KtxProjectConnectionConfig): string[] | undefined { - const raw = connection.enabled_tables; - if (!Array.isArray(raw)) { - return undefined; - } - const tables = raw.filter((value): value is string => typeof value === 'string' && value.trim().length > 0); - return tables.length > 0 ? tables : undefined; -} - -function queryHistoryPullConfig(input: { +/** @internal */ +export function queryHistoryPullConfig(input: { stored: Record; dialect: HistoricSqlDialect; windowDays?: number; - enabledTables?: string[]; + enabledTables?: KtxTableRef[]; + enabledSchemas?: string[]; + modeledTableCatalog?: KtxTableRef[]; + scopeFloorWarnings?: string[]; }): Record { - const { enabled: _enabled, dialect: _dialect, ...storedConfig } = input.stored; + const { + enabled: _enabled, + dialect: _dialect, + enabledTables: _enabledTables, + enabledSchemas: _enabledSchemas, + scopeFloorWarnings: _scopeFloorWarnings, + ...storedConfig + } = input.stored; return { ...storedConfig, dialect: input.dialect, - ...(input.enabledTables ? { enabledTables: input.enabledTables } : {}), + ...(input.enabledTables && input.enabledTables.length > 0 ? { enabledTables: input.enabledTables } : {}), + ...(input.enabledSchemas && input.enabledSchemas.length > 0 ? { enabledSchemas: input.enabledSchemas } : {}), + ...(input.modeledTableCatalog && input.modeledTableCatalog.length > 0 + ? { modeledTableCatalog: input.modeledTableCatalog } + : {}), + ...(input.scopeFloorWarnings && input.scopeFloorWarnings.length > 0 + ? { scopeFloorWarnings: input.scopeFloorWarnings } + : {}), ...(input.windowDays !== undefined ? { windowDays: input.windowDays } : {}), }; } -function depthFromLegacyScanMode( - mode: Extract['mode'] | undefined, -): KtxPublicIngestDepth | undefined { - return mode === 'enriched' || mode === 'relationships' ? 'deep' : undefined; -} - function sourceDirForConnection(connection: KtxProjectConnectionConfig): string | undefined { const value = connection.source_dir; return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined; @@ -325,13 +327,12 @@ function resolveDatabaseTargetOptions(input: { driver: string; connection: KtxProjectConnectionConfig; args: { - depth?: KtxPublicIngestDepth; queryHistory?: KtxPublicIngestQueryHistoryFlag; queryHistoryWindowDays?: number; scanMode?: Extract['mode']; }; warnings: KtxPublicIngestWarningAccumulator; -}): Pick { +}): Pick { const storedQh = storedQueryHistory(input.connection); const dialect = queryHistoryDialectByDriver.get(input.driver); const explicitQueryHistory = input.args.queryHistory ?? 'default'; @@ -340,8 +341,6 @@ function resolveDatabaseTargetOptions(input: { const requestedQh = explicitQueryHistory === 'enabled' || (explicitQueryHistory !== 'disabled' && (windowOverrideRequested || storedEnabled)); - let depth = - input.args.depth ?? depthFromLegacyScanMode(input.args.scanMode) ?? databaseContextDepth(input.connection) ?? 'fast'; const queryHistory = { enabled: false, ...(input.args.queryHistoryWindowDays !== undefined @@ -359,19 +358,13 @@ function resolveDatabaseTargetOptions(input: { explicitQueryHistory === 'enabled' || input.args.queryHistoryWindowDays !== undefined ? 'explicit' : 'stored', }); return { - databaseDepth: depth, queryHistory: { ...queryHistory, unsupported: true }, steps: ['database-schema'], }; } if (requestedQh && dialect) { - if (depth === 'fast') { - input.warnings.warnings.push(`--query-history requires deep ingest; running ${input.connectionId} with --deep.`); - } - depth = 'deep'; return { - databaseDepth: depth, queryHistory: { ...queryHistory, enabled: true, @@ -380,37 +373,78 @@ function resolveDatabaseTargetOptions(input: { stored: storedQh, dialect, windowDays: queryHistory.windowDays, - enabledTables: enabledTablesForConnection(input.connection), }), }, steps: ['database-schema', 'query-history'], }; } - if (input.args.depth === 'fast' && explicitQueryHistory !== 'enabled' && storedEnabled) { - input.warnings.warnings.push( - `${input.connectionId} has query history enabled in ktx.yaml, but --fast skips query-history processing.`, - ); - return { - databaseDepth: 'fast', - queryHistory: { ...queryHistory, skippedStoredByFast: true }, - steps: ['database-schema'], - }; - } - return { - databaseDepth: depth, queryHistory, steps: ['database-schema'], }; } +async function resolvedQueryHistoryPullConfigForTarget( + target: KtxPublicIngestPlanTarget, + project: KtxPublicIngestProject, +): Promise | null> { + if (target.operation !== 'database-ingest' || target.queryHistory?.enabled !== true || !target.queryHistory.dialect) { + return null; + } + const connection = project.config.connections[target.connectionId]; + if (!connection) { + return ( + target.queryHistory.pullConfig ?? + queryHistoryPullConfig({ + stored: {}, + dialect: target.queryHistory.dialect, + windowDays: target.queryHistory.windowDays, + }) + ); + } + const stored = storedQueryHistory(connection); + const scopeFloor = await resolveQueryHistoryScopeFloor({ + projectDir: project.projectDir, + connectionId: target.connectionId, + driver: target.driver, + connection: connection as Record, + storedQueryHistory: stored, + }); + return queryHistoryPullConfig({ + stored, + dialect: target.queryHistory.dialect, + windowDays: target.queryHistory.windowDays, + enabledTables: scopeFloor.enabledTables, + enabledSchemas: scopeFloor.enabledSchemas, + modeledTableCatalog: scopeFloor.modeledTableCatalog, + scopeFloorWarnings: scopeFloor.warnings, + }); +} + +function enrichmentReadinessGaps(config: KtxProjectConfig): string[] { + const gaps: string[] = []; + if (config.llm.provider.backend === 'none' || !config.llm.models.default) { + gaps.push('model configuration'); + } + + if (config.scan.enrichment.mode !== 'llm') { + gaps.push('scan enrichment mode'); + } + + const embeddings = config.scan.enrichment.embeddings; + if (!embeddings || embeddings.backend === 'none' || !embeddings.model || embeddings.dimensions <= 0) { + gaps.push('scan embeddings'); + } + + return gaps; +} + function targetForConnection( connectionId: string, connection: KtxProjectConnectionConfig, projectConfig: KtxPublicIngestProject['config'], args: { - depth?: KtxPublicIngestDepth; queryHistory?: KtxPublicIngestQueryHistoryFlag; queryHistoryWindowDays?: number; scanMode?: Extract['mode']; @@ -421,9 +455,6 @@ function targetForConnection( const adapter = sourceAdapterByDriver.get(driver); const sourceDir = sourceDirForConnection(connection); if (adapter) { - if (args.depth) { - warnings.ignoredDepthForSources.push(connectionId); - } if (args.queryHistory === 'enabled' || args.queryHistoryWindowDays !== undefined) { warnings.ignoredQueryHistoryForSources.push(connectionId); } @@ -440,18 +471,18 @@ function targetForConnection( if (isDatabaseDriver(driver)) { const options = resolveDatabaseTargetOptions({ connectionId, driver, connection, args, warnings }); - const gaps = options.databaseDepth === 'deep' ? deepReadinessGaps(projectConfig) : []; + const gaps = enrichmentReadinessGaps(projectConfig); return { connectionId, driver, operation: 'database-ingest', debugCommand: `ktx ingest ${connectionId} --debug`, - detectRelationships: options.databaseDepth === 'deep' && projectConfig.scan.relationships.enabled, + detectRelationships: projectConfig.scan.relationships.enabled, ...(gaps.length > 0 ? { - preflightFailure: `${connectionId} requires deep ingest readiness: ${gaps.join( + preflightFailure: `${connectionId} cannot be ingested: enrichment is not configured (${gaps.join( ', ', - )}. Run ktx setup or rerun with --fast.`, + )}). Run ktx setup to configure a model and embeddings.`, } : {}), ...options, @@ -467,7 +498,6 @@ export function buildPublicIngestPlan( projectDir: string; targetConnectionId?: string; all: boolean; - depth?: KtxPublicIngestDepth; queryHistory?: KtxPublicIngestQueryHistoryFlag; queryHistoryWindowDays?: number; scanMode?: Extract['mode']; @@ -531,13 +561,12 @@ function retryCommandForTarget( args: Extract, ): string { const projectPart = ` --project-dir ${args.projectDir}`; - const depthPart = target.databaseDepth ? ` --${target.databaseDepth}` : ''; const queryHistoryPart = target.queryHistory?.enabled === true ? ' --query-history' : ''; const windowPart = target.queryHistory?.enabled === true && target.queryHistory.windowDays !== undefined ? ` --query-history-window-days ${target.queryHistory.windowDays}` : ''; - return `ktx ingest ${target.connectionId}${projectPart}${depthPart}${queryHistoryPart}${windowPart}`; + return `ktx ingest ${target.connectionId}${projectPart}${queryHistoryPart}${windowPart}`; } function trimTrailingPeriod(value: string): string { @@ -655,6 +684,9 @@ async function emitIngestCompleted(input: { io: KtxCliIo; }): Promise { const failed = resultFailed(input.result); + const failureDetail = failed + ? formatErrorDetail(input.result.steps.find((step) => step.status === 'failed')?.detail) + : undefined; await emitTelemetryEvent({ name: 'ingest_completed', projectDir: input.args.projectDir, @@ -671,6 +703,7 @@ async function emitIngestCompleted(input: { rowsBucket: rowsBucket(), durationMs: Math.max(0, performance.now() - input.startedAt), outcome: failed ? 'error' : 'ok', + ...(failureDetail ? { errorDetail: failureDetail } : {}), }, }); } @@ -765,6 +798,80 @@ function createCapturedPublicIngestIo(): CapturedPublicIngestIo { }; } +function isCapturedPublicIngestIo(io: KtxCliIo): io is CapturedPublicIngestIo { + return typeof (io as Partial).capturedOutput === 'function'; +} + +const PLAIN_PUBLIC_INGEST_PHASE_LABELS: Record = { + 'database-schema': 'database schema', + 'query-history': 'query history', + 'source-ingest': 'source ingest', +}; + +interface PlainPublicIngestProgressOptions { + target: KtxPublicIngestPlanTarget; + index: number; + total: number; +} + +function firstSummaryLine(summary: string | undefined): string | undefined { + if (!summary) return undefined; + return summary.split(/\r?\n/).find((line) => line.trim().length > 0)?.trim(); +} + +function plainPhaseHeader(options: PlainPublicIngestProgressOptions, phaseKey: KtxPublicIngestPhaseKey): string { + const prefix = options.total > 1 ? `[${options.index + 1}/${options.total}] ` : ''; + return `${prefix}${options.target.connectionId} · ${PLAIN_PUBLIC_INGEST_PHASE_LABELS[phaseKey]}`; +} + +function plainPhaseEndLine(status: 'done' | 'failed' | 'skipped', summary?: string): string { + const firstLine = firstSummaryLine(summary); + return firstLine ? ` ${status} · ${firstLine}` : ` ${status}`; +} + +function createPlainPublicIngestProgress(io: KtxCliIo, options: PlainPublicIngestProgressOptions): Required< + Pick +> { + let currentPhase: KtxPublicIngestPhaseKey | null = null; + const startedPhases = new Set(); + const lastPercentByPhase = new Map(); + + const startPhase = (phaseKey: KtxPublicIngestPhaseKey): void => { + currentPhase = phaseKey; + startedPhases.add(phaseKey); + lastPercentByPhase.set(phaseKey, -1); + io.stderr.write(`${plainPhaseHeader(options, phaseKey)}\n`); + }; + + const ensurePhaseStarted = (phaseKey: KtxPublicIngestPhaseKey): void => { + if (!startedPhases.has(phaseKey)) { + startPhase(phaseKey); + return; + } + currentPhase = phaseKey; + }; + + const emitProgress = (update: KtxIngestProgressUpdate): void => { + if (currentPhase === null) return; + const rounded = Math.max(0, Math.min(100, Math.round(update.percent))); + const lastPercent = lastPercentByPhase.get(currentPhase) ?? -1; + if (rounded <= lastPercent) return; + lastPercentByPhase.set(currentPhase, rounded); + io.stderr.write(` [${rounded}%] ${publicProgressMessage(update.message, options.target)}\n`); + }; + + return { + onPhaseStart: startPhase, + onPhaseEnd(phaseKey, status, summary) { + ensurePhaseStarted(phaseKey); + io.stderr.write(`${plainPhaseEndLine(status, summary)}\n`); + currentPhase = null; + }, + scanProgress: createAggregateProgressPort(emitProgress), + ingestProgress: emitProgress, + }; +} + const INTERNAL_STATUS_LINE_RE = /^(Report|Run|Job|Status|Adapter|Connection|Sync|Diff|Tasks|Work units|Failed tasks|Saved memory|Provenance rows):\s*/; const ACTIONABLE_FAILURE_LINE_RE = @@ -803,11 +910,35 @@ function capturedFailureMessage(output: string): string | undefined { return [firstLine, ...followupLines].join('\n'); } +/** + * Run one ingest target through its scan/ingest steps. The single per-target + * chokepoint reached by every entrypoint — standalone `ktx ingest` (plain/json + * and foreground) and `ktx setup` (via `runContextBuild`). The exported + * `executePublicIngestTarget` wraps this and emits the `ingest_completed` + * telemetry event exactly once, so every path is counted. + */ export async function executePublicIngestTarget( target: KtxPublicIngestPlanTarget, args: Extract, io: KtxCliIo, deps: KtxPublicIngestDeps, + project: KtxPublicIngestProject, +): Promise { + const startedAt = performance.now(); + const result = await runIngestTargetSteps(target, args, io, deps, project); + // `io` may be a capture buffer for the scan/ingest step output; the telemetry + // debug echo belongs on the real user-facing stream, which callers expose as + // `deps.runtimeIo` (falling back to `io` when the step io is already real). + await emitIngestCompleted({ args, project, target, result, startedAt, io: deps.runtimeIo ?? io }); + return result; +} + +async function runIngestTargetSteps( + target: KtxPublicIngestPlanTarget, + args: Extract, + io: KtxCliIo, + deps: KtxPublicIngestDeps, + project: KtxPublicIngestProject, ): Promise { if (target.preflightFailure) { if (target.operation === 'database-ingest') { @@ -826,7 +957,7 @@ export async function executePublicIngestTarget( ? { ...step, status: 'failed', - detail: target.preflightFailure, + detail: `${target.connectionId} failed: ${target.preflightFailure}`, } : step, ), @@ -839,14 +970,18 @@ export async function executePublicIngestTarget( command: 'run', projectDir: args.projectDir, connectionId: target.connectionId, - mode: target.databaseDepth === 'deep' ? 'enriched' : 'structural', + mode: 'enriched', detectRelationships: target.detectRelationships === true, dryRun: false, ...(args.cliVersion ? { cliVersion: args.cliVersion } : {}), ...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}), }; const runScan = deps.runScan ?? runKtxScan; - const capturedScanIo = deps.scanProgress ? null : createCapturedPublicIngestIo(); + const capturedScanIo = deps.scanProgress + ? isCapturedPublicIngestIo(io) + ? io + : null + : createCapturedPublicIngestIo(); const scanIo = capturedScanIo ?? io; const scanDeps = { ...(deps.scanProgress ? { progress: deps.scanProgress } : {}), @@ -873,6 +1008,11 @@ export async function executePublicIngestTarget( if (target.queryHistory?.enabled === true) { const { runKtxIngest } = await import('./ingest.js'); const runIngest = deps.runIngest ?? runKtxIngest; + const historicSqlPullConfigOverride = + (await resolvedQueryHistoryPullConfigForTarget(target, project)) ?? { + dialect: target.queryHistory.dialect, + ...(target.queryHistory.windowDays !== undefined ? { windowDays: target.queryHistory.windowDays } : {}), + }; const ingestArgs: KtxIngestArgs = { command: 'run', projectDir: args.projectDir, @@ -883,13 +1023,15 @@ export async function executePublicIngestTarget( ...(args.cliVersion ? { cliVersion: args.cliVersion } : {}), ...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}), allowImplicitAdapter: true, - historicSqlPullConfigOverride: - target.queryHistory.pullConfig ?? { - dialect: target.queryHistory.dialect, - ...(target.queryHistory.windowDays !== undefined ? { windowDays: target.queryHistory.windowDays } : {}), - }, + historicSqlPullConfigOverride, }; - const capturedIngestIo = deps.ingestProgress ? null : createCapturedPublicIngestIo(); + // Query history runs after the schema scan has already written its report + // into the shared target io, so it needs a phase-local capture. Reusing + // `io` here would let leftover scan text (e.g. "Mode: enriched") surface as + // the query-history failure detail. Only skip capture when progress is + // active and the caller manages its own buffer (io is not a capture). + const capturedIngestIo = + deps.ingestProgress && !isCapturedPublicIngestIo(io) ? null : createCapturedPublicIngestIo(); const ingestIo = capturedIngestIo ?? io; const ingestDeps = { ...(deps.ingestProgress ? { progress: deps.ingestProgress } : {}), @@ -929,7 +1071,11 @@ export async function executePublicIngestTarget( allowImplicitAdapter: true, }; const runIngest = deps.runIngest ?? runKtxIngest; - const capturedIngestIo = deps.ingestProgress ? null : createCapturedPublicIngestIo(); + const capturedIngestIo = deps.ingestProgress + ? isCapturedPublicIngestIo(io) + ? io + : null + : createCapturedPublicIngestIo(); const ingestIo = capturedIngestIo ?? io; const ingestDeps = { ...(deps.ingestProgress ? { progress: deps.ingestProgress } : {}), @@ -974,31 +1120,63 @@ export async function runKtxPublicIngest( feature, }); } catch (error) { + await reportException({ + error, + context: { source: 'ingest runtime', handled: true, fatal: false }, + projectDir: args.projectDir, + io, + redactionSecrets: await collectTelemetryRedactionSecrets({ + project, + projectDir: args.projectDir, + connectionId: args.targetConnectionId, + includeLlm: true, + includeEmbeddings: true, + env: deps.env ?? process.env, + }), + }); io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); return 1; } } const { runContextBuild } = await import('./context-build-view.js'); const contextBuild = deps.runContextBuild ?? runContextBuild; - const result = await contextBuild( - project, - { + try { + const result = await contextBuild( + project, + { + projectDir: args.projectDir, + ...(args.targetConnectionId ? { targetConnectionId: args.targetConnectionId } : {}), + all: args.all, + entrypoint: 'ingest', + inputMode: args.inputMode, + ...(args.queryHistory ? { queryHistory: args.queryHistory } : {}), + ...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}), + ...(args.scanMode ? { scanMode: args.scanMode } : {}), + ...(args.detectRelationships !== undefined ? { detectRelationships: args.detectRelationships } : {}), + ...(args.cliVersion ? { cliVersion: args.cliVersion } : {}), + ...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}), + }, + io, + ); + return result.exitCode; + } catch (error) { + await reportException({ + error, + context: { source: 'ingest context-build', handled: true, fatal: false }, projectDir: args.projectDir, - ...(args.targetConnectionId ? { targetConnectionId: args.targetConnectionId } : {}), - all: args.all, - entrypoint: 'ingest', - inputMode: args.inputMode, - ...(args.depth ? { depth: args.depth } : {}), - ...(args.queryHistory ? { queryHistory: args.queryHistory } : {}), - ...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}), - ...(args.scanMode ? { scanMode: args.scanMode } : {}), - ...(args.detectRelationships !== undefined ? { detectRelationships: args.detectRelationships } : {}), - ...(args.cliVersion ? { cliVersion: args.cliVersion } : {}), - ...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}), - }, - io, - ); - return result.exitCode; + io, + redactionSecrets: await collectTelemetryRedactionSecrets({ + project, + projectDir: args.projectDir, + connectionId: args.targetConnectionId, + includeLlm: true, + includeEmbeddings: true, + env: deps.env ?? process.env, + }), + }); + io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + return 1; + } } const plan = buildPublicIngestPlan(project, args); @@ -1013,11 +1191,27 @@ export async function runKtxPublicIngest( } } - for (const target of plan.targets) { - const startedAt = performance.now(); - const result = await executePublicIngestTarget(target, args, io, deps); - results.push(result); - await emitIngestCompleted({ args, project, target, result, startedAt, io }); + for (const [index, target] of plan.targets.entries()) { + if (args.json) { + results.push(await executePublicIngestTarget(target, args, io, deps, project)); + continue; + } + + const capture = createCapturedPublicIngestIo(); + const progress = createPlainPublicIngestProgress(io, { + target, + index, + total: plan.targets.length, + }); + const targetDeps: KtxPublicIngestDeps = { + ...deps, + scanProgress: progress.scanProgress, + ingestProgress: progress.ingestProgress, + onPhaseStart: progress.onPhaseStart, + onPhaseEnd: progress.onPhaseEnd, + runtimeIo: deps.runtimeIo ?? io, + }; + results.push(await executePublicIngestTarget(target, args, capture, targetDeps, project)); } if (args.json) { diff --git a/packages/cli/src/reveal-password-prompt.ts b/packages/cli/src/reveal-password-prompt.ts new file mode 100644 index 00000000..3fe3ed66 --- /dev/null +++ b/packages/cli/src/reveal-password-prompt.ts @@ -0,0 +1,93 @@ +import { styleText } from 'node:util'; +import { PasswordPrompt, type PasswordOptions } from '@clack/core'; +import { S_BAR, S_BAR_END, S_PASSWORD_MASK, settings, symbol } from '@clack/prompts'; + +// How many trailing characters of a pasted secret to leave visible so the user +// can confirm what landed (e.g. `••••••a1b2`). Kept small on purpose. +const REVEAL_TAIL_COUNT = 4; + +/** + * Mask every character of `userInput` except the last `tail`, but only reveal the + * tail once the secret is long enough that the hidden portion still dominates + * (`length > tail * 2`). Short secrets stay fully masked so we never expose most + * of a small value. The returned string keeps the same code-unit length as the + * input so clack's cursor slicing in `userInputWithCursor` stays aligned. + * + * @internal + */ +export function maskRevealingTail(userInput: string, maskChar: string, tail: number): string { + const revealLength = userInput.length > tail * 2 ? tail : 0; + const hiddenLength = userInput.length - revealLength; + return maskChar.repeat(hiddenLength) + userInput.slice(hiddenLength); +} + +class RevealTailPasswordPrompt extends PasswordPrompt { + readonly #maskChar: string; + readonly #tail: number; + + constructor(options: PasswordOptions & { tail: number }) { + super(options); + this.#maskChar = options.mask ?? S_PASSWORD_MASK; + this.#tail = options.tail; + } + + override get masked(): string { + return maskRevealingTail(this.userInput, this.#maskChar, this.#tail); + } +} + +// Reproduces the @clack/prompts password frame (pinned to the installed version) +// so this prompt is visually identical to every other setup prompt; the only +// behavioral change is the tail-revealing `masked` getter above. +function renderPasswordFrame(prompt: Omit, message: string): string { + const withGuide = settings.withGuide; + const title = `${withGuide ? `${styleText('gray', S_BAR)}\n` : ''}${symbol(prompt.state)} ${message}\n`; + const masked = prompt.masked; + switch (prompt.state) { + case 'error': { + const bar = withGuide ? `${styleText('yellow', S_BAR)} ` : ''; + const end = withGuide ? `${styleText('yellow', S_BAR_END)} ` : ''; + return `${title.trim()}\n${bar}${masked}\n${end}${styleText('yellow', prompt.error)}\n`; + } + case 'submit': { + const bar = withGuide ? `${styleText('gray', S_BAR)} ` : ''; + return `${title}${bar}${masked ? styleText('dim', masked) : ''}`; + } + case 'cancel': { + const bar = withGuide ? `${styleText('gray', S_BAR)} ` : ''; + const body = masked ? styleText(['strikethrough', 'dim'], masked) : ''; + return `${title}${bar}${body}${masked && withGuide ? `\n${styleText('gray', S_BAR)}` : ''}`; + } + default: { + const bar = withGuide ? `${styleText('cyan', S_BAR)} ` : ''; + const end = withGuide ? styleText('cyan', S_BAR_END) : ''; + return `${title}${bar}${prompt.userInputWithCursor}\n${end}\n`; + } + } +} + +export interface RevealPasswordOptions { + message: string; + mask?: string; + tail?: number; + validate?: PasswordOptions['validate']; + signal?: AbortSignal; +} + +/** + * Drop-in replacement for clack's `password()` that reveals the last few + * characters of the entered value while typing. Resolves to the raw value or the + * clack cancel symbol, matching `password()`'s contract. + */ +export function revealPassword(options: RevealPasswordOptions): Promise { + const prompt = new RevealTailPasswordPrompt({ + mask: options.mask ?? S_PASSWORD_MASK, + tail: options.tail ?? REVEAL_TAIL_COUNT, + validate: options.validate, + signal: options.signal, + render() { + return renderPasswordFrame(this, options.message); + }, + }); + return prompt.prompt() as Promise; +} diff --git a/packages/cli/src/runtime-requirements.ts b/packages/cli/src/runtime-requirements.ts index 31ad1be0..0253db8c 100644 --- a/packages/cli/src/runtime-requirements.ts +++ b/packages/cli/src/runtime-requirements.ts @@ -96,7 +96,7 @@ export function resolveProjectRuntimeRequirements( for (const [connectionId, connection] of Object.entries(config.connections)) { const driver = normalizeDriver(connection.driver); - if ((driver === 'looker' || driver === 'local_looker') && !hasDaemonOverride(env)) { + if (driver === 'looker' && !hasDaemonOverride(env)) { requirements.push({ feature: 'core', reason: 'looker-source', @@ -141,7 +141,7 @@ export function resolvePublicIngestRuntimeRequirements( detail: `${target.connectionId} query-history ingest uses SQL analysis.`, }); } - if ((driver === 'looker' || driver === 'local_looker' || adapter === 'looker') && !hasDaemonOverride(env)) { + if ((driver === 'looker' || adapter === 'looker') && !hasDaemonOverride(env)) { requirements.push({ feature: 'core', reason: 'looker-source', diff --git a/packages/cli/src/scan.ts b/packages/cli/src/scan.ts index 94b80f65..5961e3f1 100644 --- a/packages/cli/src/scan.ts +++ b/packages/cli/src/scan.ts @@ -1,6 +1,6 @@ import type { KtxProgressPort, KtxScanMode, KtxScanReport, KtxScanWarning } from './context/scan/types.js'; import { runLocalScan } from './context/scan/local-scan.js'; -import { loadKtxProject } from './context/project/project.js'; +import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; import { getKtxCliPackageInfo } from './cli-runtime.js'; import { resolveProjectEmbeddingProvider } from './embedding-resolution.js'; import type { KtxCliIo } from './index.js'; @@ -8,8 +8,9 @@ import { createKtxCliLocalIngestAdapters } from './local-adapters.js'; import { createKtxCliScanConnector } from './local-scan-connectors.js'; import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; import { profileMark } from './startup-profile.js'; -import { emitTelemetryEvent } from './telemetry/index.js'; -import { scrubErrorClass } from './telemetry/scrubber.js'; +import { emitTelemetryEvent, reportException } from './telemetry/index.js'; +import { collectTelemetryRedactionSecrets } from './telemetry/redaction-secrets.js'; +import { formatErrorDetail, scrubErrorClass } from './telemetry/scrubber.js'; profileMark('module:scan'); @@ -322,8 +323,9 @@ export function createCliScanProgress( export async function runKtxScan(args: KtxScanArgs, io: KtxCliIo = process, deps: KtxScanDeps = {}): Promise { const startedAt = performance.now(); + let project: KtxLocalProject | undefined; try { - const project = await loadKtxProject({ projectDir: args.projectDir }); + project = await loadKtxProject({ projectDir: args.projectDir }); const resolveEmbeddingProvider = deps.resolveEmbeddingProvider ?? resolveProjectEmbeddingProvider; const resolution = await resolveEmbeddingProvider(project, { mode: 'ensure', @@ -380,6 +382,7 @@ export async function runKtxScan(args: KtxScanArgs, io: KtxCliIo = process, deps return 0; } catch (error) { const errorClass = scrubErrorClass(error); + const errorDetail = formatErrorDetail(error); await emitTelemetryEvent({ name: 'scan_completed', projectDir: args.projectDir, @@ -393,8 +396,23 @@ export async function runKtxScan(args: KtxScanArgs, io: KtxCliIo = process, deps durationMs: Math.max(0, performance.now() - startedAt), outcome: 'error', ...(errorClass ? { errorClass } : {}), + ...(errorDetail ? { errorDetail } : {}), }, }); + await reportException({ + error, + context: { source: 'scan run', handled: true, fatal: false }, + projectDir: args.projectDir, + io, + redactionSecrets: await collectTelemetryRedactionSecrets({ + project, + projectDir: args.projectDir, + connectionId: args.connectionId, + includeLlm: true, + includeEmbeddings: true, + env: process.env, + }), + }); io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); return 1; } diff --git a/packages/cli/src/setup-agents.ts b/packages/cli/src/setup-agents.ts index 240622f6..a671ba4b 100644 --- a/packages/cli/src/setup-agents.ts +++ b/packages/cli/src/setup-agents.ts @@ -1,5 +1,5 @@ import { existsSync } from 'node:fs'; -import { chmod, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { dirname, join, relative, resolve } from 'node:path'; import type { Writable } from 'node:stream'; import { fileURLToPath } from 'node:url'; @@ -10,6 +10,7 @@ import { markKtxSetupStateStepComplete } from './context/project/setup-config.js import { serializeKtxProjectConfig } from './context/project/config.js'; import { strToU8, zipSync } from 'fflate'; import type { KtxCliIo } from './cli-runtime.js'; +import { errorMessage, writePrefixedLines } from './clack.js'; import { createKtxSetupPromptAdapter, createKtxSetupUiAdapter, @@ -21,6 +22,7 @@ export type KtxAgentTarget = 'claude-code' | 'claude-desktop' | 'codex' | 'curso export type KtxAgentScope = 'project' | 'global' | 'local'; /** @internal */ export type KtxAgentInstallMode = 'mcp' | 'mcp-cli'; +type KtxAgentModePromptChoice = KtxAgentInstallMode | 'skip' | 'back'; export interface KtxSetupAgentsArgs { projectDir: string; @@ -55,7 +57,7 @@ export interface KtxAgentInstallManifest { | { kind: 'file'; path: string; - role?: 'skill' | 'rule' | 'analytics-skill' | 'claude-desktop-skill-bundle' | 'launcher'; + role?: 'skill' | 'rule' | 'analytics-skill' | 'claude-desktop-skill-bundle'; } | { kind: 'json-key'; path: string; jsonPath: string[] } >; @@ -312,15 +314,12 @@ function collectClaudeDesktopForwardedEnv(source: NodeJS.ProcessEnv): Record { +function claudeDesktopMcpEntry(input: { projectDir: string; env?: NodeJS.ProcessEnv }): Record { const captured = collectClaudeDesktopForwardedEnv(input.env ?? process.env); + const launcher = ktxCliLauncher(); return { - command: input.launcherPath, - args: ['--project-dir', input.projectDir, 'mcp', 'stdio'], + command: launcher.command, + args: [...launcher.args, '--project-dir', input.projectDir, 'mcp', 'stdio'], ...(Object.keys(captured).length > 0 ? { env: captured } : {}), }; } @@ -336,11 +335,10 @@ async function installMcpClientConfig(input: { if (input.target === 'claude-desktop') { const config = claudeDesktopConfigPath(); - const launcherPath = claudeDesktopLauncherPath(input.projectDir); await writeJsonKey( config.path, config.jsonPath, - claudeDesktopMcpEntry({ launcherPath, projectDir: input.projectDir }), + claudeDesktopMcpEntry({ projectDir: input.projectDir }), ); entries.push({ kind: 'json-key', path: config.path, jsonPath: config.jsonPath }); return { entries, snippets, notices }; @@ -406,10 +404,6 @@ function claudeDesktopAdminSkillBundlePath(projectDir: string): string { return join(resolve(projectDir), '.ktx/agents/claude/ktx.zip'); } -function claudeDesktopLauncherPath(projectDir: string): string { - return join(resolve(projectDir), '.ktx/agents/claude/ktx-plugin-runner.sh'); -} - /** @internal */ export function plannedKtxAgentFiles(input: { projectDir: string; @@ -449,7 +443,6 @@ export function plannedKtxAgentFiles(input: { } if (input.target === 'claude-desktop') { return [ - { kind: 'file', path: claudeDesktopLauncherPath(input.projectDir), role: 'launcher' as const }, { kind: 'file', path: claudeDesktopAnalyticsSkillBundlePath(input.projectDir), @@ -593,61 +586,6 @@ function cliInstructionContent(input: { projectDir: string; launcher: KtxCliLaun ].join('\n'); } -function claudeDesktopLauncherContent(input: { launcher: KtxCliLauncher }): string { - const binPath = input.launcher.args[0]; - if (!binPath) { - throw new Error('Expected KTX CLI launcher to include a bin path.'); - } - const candidates = [ - input.launcher.command, - '/opt/homebrew/bin/node', - '/usr/local/bin/node', - '/usr/bin/node', - ]; - return [ - '#!/bin/sh', - 'set -eu', - '', - `KTX_CLI_BIN=${shellScriptQuote(binPath)}`, - '', - 'run_with_node() {', - ' node_bin=$1', - ' shift', - ' exec "$node_bin" "$KTX_CLI_BIN" "$@"', - '}', - '', - 'if [ -n "${KTX_NODE:-}" ] && [ -x "${KTX_NODE:-}" ]; then', - ' run_with_node "$KTX_NODE" "$@"', - 'fi', - '', - 'if [ -x "$HOME/.volta/bin/node" ]; then', - ' run_with_node "$HOME/.volta/bin/node" "$@"', - 'fi', - '', - ...candidates.map((candidate) => - [ - `if [ -x ${shellScriptQuote(candidate)} ]; then`, - ` run_with_node ${shellScriptQuote(candidate)} "$@"`, - 'fi', - ].join('\n'), - ), - '', - 'for candidate in "$HOME"/.nvm/versions/node/*/bin/node; do', - ' if [ -x "$candidate" ]; then', - ' run_with_node "$candidate" "$@"', - ' fi', - 'done', - '', - 'if command -v node >/dev/null 2>&1; then', - ' run_with_node "$(command -v node)" "$@"', - 'fi', - '', - 'echo "KTX Claude Desktop launcher could not find Node.js. Set KTX_NODE to a Node executable and rerun ktx setup --agents." >&2', - 'exit 127', - '', - ].join('\n'); -} - async function writeClaudeDesktopSkillBundle(input: { projectDir: string; path: string; @@ -675,15 +613,6 @@ function claudeDesktopSkillNameForBundle(path: string): 'ktx-analytics' | 'ktx' throw new Error(`Unsupported Claude Desktop skill bundle path: ${path}`); } -async function writeClaudeDesktopLauncher(input: { - path: string; - launcher: KtxCliLauncher; -}): Promise { - await mkdir(dirname(input.path), { recursive: true }); - await writeFile(input.path, claudeDesktopLauncherContent({ launcher: input.launcher }), 'utf-8'); - await chmod(input.path, 0o755); -} - function ruleInstructionContent(input: { projectDir: string }): string { return [ `Use the \`ktx\` CLI to query local semantic context and wiki knowledge for this project ` + @@ -941,10 +870,6 @@ export function formatInstallSummaryLines( lines.push(`${guidanceInstallLine(install.target)}.`); } - if (hasEntryRole(targetEntries, 'launcher')) { - lines.push('Starts KTX over stdio from Claude Desktop.'); - } - return { title: `${targetDisplayName(install.target)} · ${scopeDisplayName(install.scope)}`, lines, @@ -1139,10 +1064,6 @@ async function installTarget(input: { const launcher = ktxCliLauncher(); for (const entry of entries) { if (entry.kind !== 'file') continue; - if (entry.role === 'launcher') { - await writeClaudeDesktopLauncher({ path: entry.path, launcher }); - continue; - } if (entry.role === 'claude-desktop-skill-bundle') { await writeClaudeDesktopSkillBundle({ projectDir: input.projectDir, @@ -1203,9 +1124,18 @@ export async function runKtxSetupAgentsStep( label: 'Ask data questions + manage KTX with CLI commands', hint: 'Adds an admin CLI skill so agents can run ktx status, sl, wiki, and setup commands.', }, + { + value: 'skip', + label: 'Skip agent setup for now', + hint: 'Leaves agent integration incomplete. You can run ktx setup --agents later.', + }, ], - })) as KtxAgentInstallMode | 'back'); + })) as KtxAgentModePromptChoice); if (mode === 'back') return { status: 'skipped', projectDir: args.projectDir }; + if (mode === 'skip') { + io.stdout.write('│ Agent integration skipped.\n'); + return { status: 'skipped', projectDir: args.projectDir }; + } const targets = args.target !== undefined @@ -1301,7 +1231,7 @@ export async function runKtxSetupAgentsStep( } return { status: 'ready', projectDir: args.projectDir, installs, nextActions }; } catch (error) { - io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + writePrefixedLines((chunk) => io.stderr.write(chunk), errorMessage(error)); return { status: 'failed', projectDir: args.projectDir }; } } diff --git a/packages/cli/src/setup-context.ts b/packages/cli/src/setup-context.ts index aa519111..be458d2a 100644 --- a/packages/cli/src/setup-context.ts +++ b/packages/cli/src/setup-context.ts @@ -5,13 +5,12 @@ import { type KtxLocalProject, loadKtxProject } from './context/project/project. import { markKtxSetupStateStepComplete, readKtxSetupState } from './context/project/setup-config.js'; import { serializeKtxProjectConfig } from './context/project/config.js'; import type { KtxCliIo } from './cli-runtime.js'; +import { errorMessage, writePrefixedLines } from './clack.js'; +import { formatErrorDetail } from './telemetry/scrubber.js'; import { buildPublicIngestPlan } from './public-ingest.js'; -import { - type KtxDatabaseContextDepth, - databaseContextDepth, -} from './ingest-depth.js'; +import { runKtxConnection } from './connection.js'; +import { type BufferedCommandIo, createBufferedCommandIo } from './io/buffered-command-io.js'; import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; -import { ensureSetupDatabaseContextDepths } from './setup-database-context-depth.js'; import { type ContextBuildSourceProgressUpdate, runContextBuild, @@ -71,7 +70,7 @@ export type KtxSetupContextResult = | { status: 'skipped'; projectDir: string } | { status: 'back'; projectDir: string } | { status: 'missing-input'; projectDir: string } - | { status: 'failed'; projectDir: string }; + | { status: 'failed'; projectDir: string; errorDetail?: string }; export interface KtxSetupContextStepArgs { projectDir: string; @@ -94,6 +93,7 @@ export interface KtxSetupContextDeps { now?: () => Date; runContextBuild?: typeof runContextBuild; verifyContextReady?: (projectDir: string) => Promise; + testConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise; } interface KtxSetupContextTargets { @@ -280,6 +280,140 @@ function listContextTargets(project: KtxLocalProject): KtxSetupContextTargets { }; } +interface ConnectionGateFailure { + connectionId: string; + driver: string; +} + +type ConnectionGateResult = { ok: true } | { ok: false; failures: ConnectionGateFailure[] }; + +type PreparedBuild = + | { kind: 'ready'; project: KtxLocalProject; targets: KtxSetupContextTargets } + | { kind: 'result'; result: KtxSetupContextResult }; + +function requiredConnectionIds(targets: KtxSetupContextTargets): string[] { + return [...targets.primarySourceConnectionIds, ...targets.contextSourceConnectionIds]; +} + +function connectorTypeLabel(project: KtxLocalProject, connectionId: string): string { + const driver = String(project.config.connections[connectionId]?.driver ?? '') + .trim() + .toLowerCase(); + return driver.length > 0 ? driver : 'unknown'; +} + +async function defaultGateTestConnection( + projectDir: string, + connectionId: string, + io: KtxCliIo, +): Promise { + return await runKtxConnection({ command: 'test', projectDir, connectionId }, io); +} + +/** + * Runs a live connection test for every connection the build depends on. Each + * test's output is captured in a buffer and discarded so raw error text never + * reaches the user — callers surface only the connection id and connector type. + */ +async function testRequiredConnections( + projectDir: string, + project: KtxLocalProject, + targets: KtxSetupContextTargets, + testConnection: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise, +): Promise { + const failures: ConnectionGateFailure[] = []; + for (const connectionId of requiredConnectionIds(targets)) { + const buffered: BufferedCommandIo = createBufferedCommandIo(); + const exitCode = await testConnection(projectDir, connectionId, buffered); + if (exitCode !== 0) { + failures.push({ connectionId, driver: connectorTypeLabel(project, connectionId) }); + } + } + return failures.length === 0 ? { ok: true } : { ok: false, failures }; +} + +/** + * Loads the project and resolves the connections the build depends on, applying + * the empty-targets and preflight-capability checks. Used both on first entry + * and on interactive retry so a fix that adds, removes, or reconfigures a + * connection is honored. + */ +async function prepareBuildTargets(args: KtxSetupContextStepArgs, io: KtxCliIo): Promise { + const project = await loadKtxProject({ projectDir: args.projectDir }); + const targets = listContextTargets(project); + if (targets.primarySourceConnectionIds.length === 0 && targets.contextSourceConnectionIds.length === 0) { + if (args.allowEmpty === true) { + return { kind: 'result', result: { status: 'skipped', projectDir: args.projectDir } }; + } + io.stderr.write('No databases or context sources are configured for a KTX context build.\n'); + return { kind: 'result', result: { status: 'failed', projectDir: args.projectDir } }; + } + const preflightPlan = buildPublicIngestPlan(project, { projectDir: project.projectDir, all: true }); + const preflightFailures = preflightPlan.targets.flatMap((target) => + target.preflightFailure ? [`${target.connectionId}: ${target.preflightFailure}`] : [], + ); + if (preflightFailures.length > 0) { + if (args.allowEmpty === true) { + return { kind: 'result', result: { status: 'skipped', projectDir: args.projectDir } }; + } + writeMissingCapabilities(preflightFailures, io); + return { kind: 'result', result: { status: 'missing-input', projectDir: args.projectDir } }; + } + return { kind: 'ready', project, targets }; +} + +function writeConnectionGateFailureLines( + io: KtxCliIo, + projectDir: string, + failures: ConnectionGateFailure[], +): void { + io.stderr.write('KTX cannot build context: a required connection failed its live test.\n\n'); + io.stderr.write('Failed connections:\n'); + for (const failure of failures) { + io.stderr.write(` ${failure.connectionId} (${failure.driver})\n`); + } + io.stderr.write('\nEach connection must be reachable before KTX builds context.\n'); + io.stderr.write( + `Run \`ktx connection test --project-dir ${resolve(projectDir)}\` to see the error, fix the connection, then retry.\n`, + ); +} + +function connectionGateFailureReason(failures: ConnectionGateFailure[]): string { + const names = failures.map((failure) => `${failure.connectionId} (${failure.driver})`).join(', '); + return `Required connections failed their live test: ${names}.`; +} + +async function writeConnectionGateFailedState( + args: KtxSetupContextStepArgs, + deps: KtxSetupContextDeps, + targets: KtxSetupContextTargets, + failures: ConnectionGateFailure[], +): Promise { + const at = (deps.now ?? (() => new Date()))().toISOString(); + await writeKtxSetupContextState(args.projectDir, { + status: 'failed', + startedAt: at, + updatedAt: at, + primarySourceConnectionIds: targets.primarySourceConnectionIds, + contextSourceConnectionIds: targets.contextSourceConnectionIds, + reportIds: [], + artifactPaths: [], + retryableFailedTargets: [], + commands: contextBuildCommands(args.projectDir), + failureReason: connectionGateFailureReason(failures), + }); +} + +async function promptConnectionGateRetry(prompts: KtxSetupContextPromptAdapter): Promise<'retry' | 'back'> { + return (await prompts.select({ + message: 'Fix the failing connection, then choose how to proceed.', + options: [ + { value: 'retry', label: 'Retry connection tests' }, + { value: 'back', label: 'Back' }, + ], + })) as 'retry' | 'back'; +} + async function hasFileWithExtension( root: string, extensions: Set, @@ -352,16 +486,6 @@ async function readLatestScanReport(projectDir: string, connectionId: string): P return reports.at(-1)?.report ?? null; } -function scanReportHasSchemaManifest(report: unknown, connectionId: string): boolean { - if (!isRecord(report)) { - return false; - } - if (report.connectionId !== connectionId || report.dryRun === true) { - return false; - } - return stringArrayValue(isRecord(report.artifactPaths) ? report.artifactPaths.manifestShards : undefined).length > 0; -} - function scanReportHasCompletedDeepEnrichment( report: unknown, connectionId: string, @@ -388,18 +512,6 @@ function scanReportHasCompletedDeepEnrichment( ); } -function scanReportSatisfiesDepth(input: { - report: unknown; - connectionId: string; - depth: KtxDatabaseContextDepth; - relationshipsRequired: boolean; -}): boolean { - if (input.depth === 'fast') { - return scanReportHasSchemaManifest(input.report, input.connectionId); - } - return scanReportHasCompletedDeepEnrichment(input.report, input.connectionId, input.relationshipsRequired); -} - async function verifyPrimarySourceScans( project: KtxLocalProject, connectionIds: string[], @@ -407,15 +519,9 @@ async function verifyPrimarySourceScans( const details: string[] = []; const relationshipsRequired = project.config.scan.relationships.enabled; for (const connectionId of connectionIds) { - const connection = project.config.connections[connectionId]; - const depth = connection ? (databaseContextDepth(connection) ?? 'fast') : 'fast'; const report = await readLatestScanReport(project.projectDir, connectionId); - if (!scanReportSatisfiesDepth({ report, connectionId, depth, relationshipsRequired })) { - details.push( - depth === 'fast' - ? `${connectionId}: schema context has not completed.` - : `${connectionId}: deep database context has not completed.`, - ); + if (!scanReportHasCompletedDeepEnrichment(report, connectionId, relationshipsRequired)) { + details.push(`${connectionId}: database context has not completed.`); } } return { ready: details.length === 0, details }; @@ -472,16 +578,13 @@ function writeMissingCapabilities(missing: string[], io: KtxCliIo): void { io.stderr.write('\nFix this in setup before building context.\n'); } -function writeSkippedContext(projectDir: string, io: KtxCliIo): void { - io.stdout.write('\nKTX is configured, but context has not been built yet.\n\n'); - io.stdout.write('Agents were not connected because KTX has not prepared searchable context for them.\n\n'); - io.stdout.write(`Resume setup:\n ktx setup --project-dir ${resolve(projectDir)}\n\n`); - io.stdout.write(`Build context:\n ktx setup --project-dir ${resolve(projectDir)}\n\n`); - io.stdout.write(`Check status:\n ktx status --project-dir ${resolve(projectDir)}\n`); +function writeSkippedContext(io: KtxCliIo): void { + // The setup completion screen owns "what to do next" (it points at `ktx ingest`), + // so keep this to a short acknowledgement rather than a competing command list. + io.stdout.write('\nLeaving context unbuilt for now.\n'); } function writeSuccess( - project: KtxLocalProject, readiness: KtxSetupContextReadiness, targets: KtxSetupContextTargets, io: KtxCliIo, @@ -492,9 +595,7 @@ function writeSuccess( io.stdout.write(' none\n'); } else { for (const connectionId of targets.primarySourceConnectionIds) { - const connection = project.config.connections[connectionId]; - const depth = connection ? (databaseContextDepth(connection) ?? 'fast') : 'fast'; - io.stdout.write(` ${connectionId}: ${depth === 'deep' ? 'deep context complete' : 'schema context complete'}\n`); + io.stdout.write(` ${connectionId}: database context complete\n`); } } io.stdout.write('\nContext sources:\n'); @@ -635,7 +736,7 @@ async function runBuild( failureReason: undefined, ...(lastSourceProgress ? { sourceProgress: lastSourceProgress } : {}), }); - writeSuccess(project, readiness, targets, io); + writeSuccess(readiness, targets, io); return { status: 'ready', projectDir: args.projectDir, runId }; } @@ -677,17 +778,7 @@ export async function runKtxSetupContextStep( deps: KtxSetupContextDeps = {}, ): Promise { try { - let project = await loadKtxProject({ projectDir: args.projectDir }); const prompts = deps.prompts ?? createPromptAdapter(); - const depthProject = await ensureSetupDatabaseContextDepths({ - project, - args, - prompts, - }); - if (depthProject === 'back') { - return { status: 'back', projectDir: args.projectDir }; - } - project = depthProject; const existingState = await readKtxSetupContextState(args.projectDir); const completedSteps = (await readKtxSetupState(args.projectDir)).completed_steps; if (completedSteps.includes('context') && existingState.status === 'completed') { @@ -704,26 +795,12 @@ export async function runKtxSetupContextStep( io.stdout.write('Previous context build state is stale; starting a fresh foreground build.\n'); } - const targets = listContextTargets(project); - if (targets.primarySourceConnectionIds.length === 0 && targets.contextSourceConnectionIds.length === 0) { - if (args.allowEmpty === true) { - return { status: 'skipped', projectDir: args.projectDir }; - } - io.stderr.write('No databases or context sources are configured for a KTX context build.\n'); - return { status: 'failed', projectDir: args.projectDir }; - } - - const preflightPlan = buildPublicIngestPlan(project, { projectDir: project.projectDir, all: true }); - const preflightFailures = preflightPlan.targets.flatMap((target) => - target.preflightFailure ? [`${target.connectionId}: ${target.preflightFailure}`] : [], - ); - if (preflightFailures.length > 0) { - if (args.allowEmpty === true) { - return { status: 'skipped', projectDir: args.projectDir }; - } - writeMissingCapabilities(preflightFailures, io); - return { status: 'missing-input', projectDir: args.projectDir }; + const prepared = await prepareBuildTargets(args, io); + if (prepared.kind === 'result') { + return prepared.result; } + let { project, targets } = prepared; + const interactive = args.inputMode !== 'disabled' && args.prompt !== false; if (args.forcePrompt !== true && args.prompt !== false && deps.verifyContextReady === undefined) { const existingContextResult = await completeExistingContext(args, io, deps, targets); @@ -732,20 +809,45 @@ export async function runKtxSetupContextStep( } } - if (args.inputMode !== 'disabled' && args.prompt !== false) { + if (interactive) { const choice = await promptForBuild(prompts); if (choice === 'back') { return { status: 'back', projectDir: args.projectDir }; } if (choice === 'skip') { - writeSkippedContext(args.projectDir, io); + writeSkippedContext(io); return { status: 'skipped', projectDir: args.projectDir }; } } - return await runBuild(args, io, deps, project, targets); + // Live-connection gate: every connection the build depends on must pass a + // live test before the (expensive) build starts. A red connection is a hard + // stop — we surface only the connection id and connector type, never raw + // error text. + const testConnection = deps.testConnection ?? defaultGateTestConnection; + while (true) { + const gate = await testRequiredConnections(args.projectDir, project, targets, testConnection); + if (gate.ok) { + return await runBuild(args, io, deps, project, targets); + } + writeConnectionGateFailureLines(io, args.projectDir, gate.failures); + if (!interactive) { + await writeConnectionGateFailedState(args, deps, targets, gate.failures); + return { status: 'failed', projectDir: args.projectDir }; + } + const choice = await promptConnectionGateRetry(prompts); + if (choice === 'back') { + return { status: 'back', projectDir: args.projectDir }; + } + const reprepared = await prepareBuildTargets(args, io); + if (reprepared.kind === 'result') { + return reprepared.result; + } + project = reprepared.project; + targets = reprepared.targets; + } } catch (error) { - io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); - return { status: 'failed', projectDir: args.projectDir }; + writePrefixedLines((chunk) => io.stderr.write(chunk), errorMessage(error)); + return { status: 'failed', projectDir: args.projectDir, errorDetail: formatErrorDetail(error) }; } } diff --git a/packages/cli/src/setup-database-context-depth.ts b/packages/cli/src/setup-database-context-depth.ts deleted file mode 100644 index 20df813c..00000000 --- a/packages/cli/src/setup-database-context-depth.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { writeFile } from 'node:fs/promises'; -import { type KtxLocalProject, loadKtxProject } from './context/project/project.js'; -import { type KtxProjectConnectionConfig, serializeKtxProjectConfig } from './context/project/config.js'; -import { - type KtxDatabaseContextDepth, - databaseContextDepth, - deepReadinessGaps, - isDatabaseDriver, - normalizeConnectionDriver, - recommendedDatabaseContextDepth, - withDatabaseContextDepth, -} from './ingest-depth.js'; -import type { KtxSetupPromptOption } from './setup-prompts.js'; - -export interface KtxSetupDatabaseContextDepthArgs { - inputMode: 'auto' | 'disabled'; -} - -export interface KtxSetupDatabaseContextDepthPromptAdapter { - select(options: { message: string; options: KtxSetupPromptOption[] }): Promise; -} - -function databaseConnectionsNeedingDepth(project: KtxLocalProject): string[] { - return Object.entries(project.config.connections) - .filter(([, connection]) => isDatabaseDriver(normalizeConnectionDriver(connection))) - .filter(([, connection]) => databaseContextDepth(connection) === undefined) - .map(([connectionId]) => connectionId) - .sort((left, right) => left.localeCompare(right)); -} - -async function chooseSetupDatabaseContextDepth(input: { - project: KtxLocalProject; - args: KtxSetupDatabaseContextDepthArgs; - prompts: KtxSetupDatabaseContextDepthPromptAdapter; -}): Promise { - const recommended = recommendedDatabaseContextDepth(input.project.config); - if (input.args.inputMode === 'disabled') { - return recommended; - } - - const deepReady = deepReadinessGaps(input.project.config).length === 0; - const options = - recommended === 'deep' - ? [ - { - value: 'deep', - label: 'Deep: AI descriptions, embeddings, relationships, slower', - hint: 'recommended', - }, - { value: 'fast', label: 'Fast: schema only, no AI, quickest' }, - { value: 'back', label: 'Back' }, - ] - : [ - { value: 'fast', label: 'Fast: schema only, no AI, quickest', hint: 'recommended' }, - { value: 'deep', label: 'Deep: AI descriptions, embeddings, relationships, slower' }, - { value: 'back', label: 'Back' }, - ]; - - const choice = await input.prompts.select({ - message: - 'How much database context should KTX build?\n\n' + - (deepReady - ? 'Deep is available because model, embedding, and scan enrichment are configured.' - : 'Fast is recommended because model, embedding, or scan enrichment is not configured.'), - options, - }); - if (choice === 'back') { - return 'back'; - } - if (choice === 'fast' || choice === 'deep') { - return choice; - } - return recommended; -} - -async function writeDatabaseContextDepths( - project: KtxLocalProject, - connectionIds: string[], - depth: KtxDatabaseContextDepth, -): Promise { - if (connectionIds.length === 0) { - return project; - } - const nextConnections = { ...project.config.connections }; - for (const connectionId of connectionIds) { - const connection = nextConnections[connectionId]; - if (connection) { - nextConnections[connectionId] = withDatabaseContextDepth(connection, depth); - } - } - const nextConfig = { ...project.config, connections: nextConnections }; - await writeFile(project.configPath, serializeKtxProjectConfig(nextConfig), 'utf-8'); - return await loadKtxProject({ projectDir: project.projectDir }); -} - -export async function ensureSetupDatabaseContextDepths(input: { - project: KtxLocalProject; - args: KtxSetupDatabaseContextDepthArgs; - prompts: KtxSetupDatabaseContextDepthPromptAdapter; -}): Promise { - const missingDepthConnectionIds = databaseConnectionsNeedingDepth(input.project); - if (missingDepthConnectionIds.length === 0) { - return input.project; - } - - const depth = await chooseSetupDatabaseContextDepth(input); - if (depth === 'back') { - return 'back'; - } - return await writeDatabaseContextDepths(input.project, missingDepthConnectionIds, depth); -} - -export async function applySetupDatabaseContextDepth(input: { - project: KtxLocalProject; - connection: KtxProjectConnectionConfig; - args: KtxSetupDatabaseContextDepthArgs; - prompts: KtxSetupDatabaseContextDepthPromptAdapter; -}): Promise { - if ( - !isDatabaseDriver(normalizeConnectionDriver(input.connection)) || - databaseContextDepth(input.connection) !== undefined - ) { - return input.connection; - } - - const depth = await chooseSetupDatabaseContextDepth(input); - if (depth === 'back') { - return 'back'; - } - return withDatabaseContextDepth(input.connection, depth); -} diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index 392c4761..002ead30 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -3,30 +3,62 @@ import { readFile, writeFile } from 'node:fs/promises'; import { delimiter, dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; +import { getDriverRegistration } from './context/connections/drivers.js'; +import { createLocalKtxLlmRuntimeFromConfig } from './context/llm/local-config.js'; +import type { KtxLlmRuntimePort } from './context/llm/runtime-port.js'; +import { queryHistoryDialectForConnection } from './context/ingest/adapters/historic-sql/connection-dialect.js'; +import { + proposeQueryHistoryServiceAccountFilters, + type ProposeQueryHistoryServiceAccountFiltersInput, + type QueryHistoryFilterProposal, +} from './context/ingest/adapters/historic-sql/query-history-filter-picker.js'; +import { resolveQueryHistoryScopeFloor } from './context/ingest/adapters/historic-sql/scope-floor.js'; import type { HistoricSqlDialect } from './context/ingest/adapters/historic-sql/types.js'; +import { + runHistoricSqlReadinessProbe, + type HistoricSqlProbeOutcome, + type HistoricSqlReadinessProbe, +} from './context/ingest/historic-sql-probes.js'; import { type KtxProjectConnectionConfig, serializeKtxProjectConfig } from './context/project/config.js'; import { loadKtxProject } from './context/project/project.js'; import { markKtxSetupStateStepComplete, setKtxSetupDatabaseConnectionIds } from './context/project/setup-config.js'; import type { KtxTableListEntry } from './context/scan/types.js'; -import type { KtxCliIo } from './cli-runtime.js'; +import { getKtxCliPackageInfo, type KtxCliIo } from './cli-runtime.js'; +import { + errorMessage, + flushPrefixedBufferedCommandOutput, + writePrefixedLines, +} from './clack.js'; import { runKtxConnection } from './connection.js'; +import { createBufferedCommandIo } from './io/buffered-command-io.js'; +import { + runConnectionSetupWithRecovery, + type ConfigureResult, + type RecoveryOutcome, + type ValidateResult, +} from './connection-recovery.js'; import { pickDatabaseScope as defaultPickDatabaseScope, type DatabaseScopePickResult, type PickDatabaseScopeArgs, } from './database-tree-picker.js'; import { withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js'; +import { createKtxCliHistoricSqlRuntime } from './local-adapters.js'; +import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; +import type { ManagedPythonCoreDaemonOptions } from './managed-python-http.js'; +import { queryHistoryPullConfig } from './public-ingest.js'; import { runKtxScan } from './scan.js'; -import { applySetupDatabaseContextDepth } from './setup-database-context-depth.js'; import { writeProjectLocalSecretReference } from './setup-secrets.js'; import { isDemoConnection } from './telemetry/demo-detect.js'; import { emitTelemetryEvent } from './telemetry/index.js'; import { createKtxSetupPromptAdapter, + createKtxSetupUiAdapter, type KtxSetupPromptOption, } from './setup-prompts.js'; const HISTORIC_SQL_WORK_UNIT_MAX_CONCURRENCY = 6; +const KTX_DEMO_START_URL = 'https://www.kaelio.com/start'; const execFileAsync = promisify(execFileCallback); export type KtxSetupDatabaseDriver = @@ -41,6 +73,10 @@ export type KtxSetupDatabaseDriver = export interface KtxSetupDatabasesArgs { projectDir: string; inputMode: 'auto' | 'disabled'; + debug?: boolean; + yes?: boolean; + cliVersion?: string; + runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; databaseDrivers?: KtxSetupDatabaseDriver[]; databaseConnectionIds?: string[]; databaseConnectionId?: string; @@ -56,7 +92,12 @@ export interface KtxSetupDatabasesArgs { } export type KtxSetupDatabasesResult = - | { status: 'ready'; projectDir: string; connectionIds: string[] } + | { + status: 'ready'; + projectDir: string; + connectionIds: string[]; + skipSources?: boolean; + } | { status: 'skipped'; projectDir: string } | { status: 'back'; projectDir: string } | { status: 'missing-input'; projectDir: string } @@ -84,19 +125,11 @@ export interface KtxSetupDatabasesPromptAdapter { cancel(message: string): void; } -interface KtxSetupHistoricSqlProbeInput { - projectDir: string; - connectionId: string; - dialect: HistoricSqlDialect; -} - interface KtxSetupHistoricSqlProbeResult { ok: boolean; lines: string[]; } -type KtxSetupHistoricSqlProbe = (input: KtxSetupHistoricSqlProbeInput) => Promise; - export interface KtxSetupDatabasesDeps { prompts?: KtxSetupDatabasesPromptAdapter; testConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise; @@ -105,17 +138,24 @@ export interface KtxSetupDatabasesDeps { listSchemas?: (projectDir: string, connectionId: string) => Promise; listTables?: (projectDir: string, connectionId: string, schemas?: string[]) => Promise; pickDatabaseScope?: (args: PickDatabaseScopeArgs, io: KtxCliIo) => Promise; - historicSqlProbe?: KtxSetupHistoricSqlProbe; + historicSqlReadinessProbe?: HistoricSqlReadinessProbe; + queryHistoryFilterPicker?: ( + input: ProposeQueryHistoryServiceAccountFiltersInput, + ) => Promise; + createQueryHistoryLlmRuntime?: ( + projectDir: string, + project: Awaited>, + ) => KtxLlmRuntimePort | null; } const DRIVER_OPTIONS: Array<{ value: KtxSetupDatabaseDriver; label: string }> = [ - { value: 'sqlite', label: 'SQLite' }, { value: 'postgres', label: 'PostgreSQL' }, + { value: 'bigquery', label: 'BigQuery' }, + { value: 'snowflake', label: 'Snowflake' }, { value: 'mysql', label: 'MySQL' }, { value: 'clickhouse', label: 'ClickHouse' }, { value: 'sqlserver', label: 'SQL Server' }, - { value: 'bigquery', label: 'BigQuery' }, - { value: 'snowflake', label: 'Snowflake' }, + { value: 'sqlite', label: 'SQLite' }, ]; const DRIVER_LABELS = Object.fromEntries(DRIVER_OPTIONS.map((option) => [option.value, option.label])) as Record< @@ -217,7 +257,6 @@ const SCOPE_DISCOVERY_SPECS: Partial; -type ConnectionSetupStatus = 'ready' | 'back' | 'failed'; const DRIVER_CONNECTION_DEFAULTS: Record = { postgres: { port: '5432' }, @@ -285,6 +324,13 @@ function numberConfigField(connection: KtxProjectConnectionConfig | undefined, f return typeof value === 'number' && Number.isFinite(value) ? value : undefined; } +function historicSqlConfigRecord(connection: KtxProjectConnectionConfig | undefined): Record | null { + const historicSql = connection?.historicSql; + return historicSql && typeof historicSql === 'object' && !Array.isArray(historicSql) + ? (historicSql as Record) + : null; +} + function contextRecord(connection: KtxProjectConnectionConfig | undefined): Record { const context = connection?.context; return context && typeof context === 'object' && !Array.isArray(context) ? (context as Record) : {}; @@ -297,12 +343,19 @@ function queryHistoryConfigRecord(connection: KtxProjectConnectionConfig | undef : null; } +function stripLegacyHistoricSql(connection: KtxProjectConnectionConfig): KtxProjectConnectionConfig { + const { historicSql: _historicSql, ...rest } = connection as KtxProjectConnectionConfig & { + historicSql?: unknown; + }; + return rest; +} + function withQueryHistoryConfig( connection: KtxProjectConnectionConfig, queryHistory: Record, ): KtxProjectConnectionConfig { return { - ...connection, + ...stripLegacyHistoricSql(connection), context: { ...contextRecord(connection), queryHistory, @@ -310,195 +363,52 @@ function withQueryHistoryConfig( }; } -function historicSqlProbeFailureLines(error: unknown): string[] { - if (error instanceof Error && error.name === 'HistoricSqlExtensionMissingError') { - return [ - ' FAIL pg_stat_statements extension is not installed in the connection database', - ' Fix: Run (against this database): CREATE EXTENSION pg_stat_statements;', - " Fix: Ensure shared_preload_libraries includes 'pg_stat_statements'.", - ]; +function migrateLegacyHistoricSqlConnection(connection: KtxProjectConnectionConfig): KtxProjectConnectionConfig { + const existingQueryHistory = queryHistoryConfigRecord(connection); + const legacy = historicSqlConfigRecord(connection); + if (existingQueryHistory || !legacy) { + return existingQueryHistory ? stripLegacyHistoricSql(connection) : connection; } - if (error instanceof Error && error.name === 'HistoricSqlGrantsMissingError') { - const dialect = (error as { dialect?: unknown }).dialect; - if (dialect === 'snowflake') { - return [ - ' FAIL Snowflake role cannot read SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY', - ' Fix: Run (as ACCOUNTADMIN): GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE ;', - ]; - } - return [ - ' FAIL Postgres connection role lacks pg_read_all_stats', - ' Fix: Run: GRANT pg_read_all_stats TO ;', - ]; - } - if (error instanceof Error && error.name === 'HistoricSqlVersionUnsupportedError') { - return [` FAIL ${error.message}`]; - } - return [` FAIL Query history probe failed: ${error instanceof Error ? error.message : String(error)}`]; + const { dialect: _dialect, ...queryHistory } = legacy; + return withQueryHistoryConfig(connection, queryHistory); } -async function defaultHistoricSqlProbe(input: KtxSetupHistoricSqlProbeInput): Promise { - if (input.dialect === 'postgres') { - return probePostgresHistoricSql(input); +function setupHistoricSqlProbeResult( + outcome: HistoricSqlProbeOutcome | null, +): KtxSetupHistoricSqlProbeResult { + if (!outcome) { + return { ok: true, lines: [] }; } - if (input.dialect === 'snowflake') { - return probeSnowflakeHistoricSql(input); - } - return { ok: true, lines: [] }; -} - -async function probePostgresHistoricSql( - input: KtxSetupHistoricSqlProbeInput, -): Promise { - const project = await loadKtxProject({ projectDir: input.projectDir }); - const connection = project.config.connections[input.connectionId]; - const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient }, { isKtxPostgresConnectionConfig }] = - await Promise.all([ - import('./context/ingest/adapters/historic-sql/postgres-pgss-reader.js'), - import('./connectors/postgres/historic-sql-query-client.js'), - import('./connectors/postgres/connector.js'), - ]); - - const postgresConnection = connection as Parameters[0]; - if (!isKtxPostgresConnectionConfig(postgresConnection)) { - return { - ok: false, - lines: [` FAIL Connection ${input.connectionId} is not a native Postgres connection.`], - }; - } - - const client = new KtxPostgresHistoricSqlQueryClient({ - connectionId: input.connectionId, - connection: postgresConnection, - }); - try { - const result = await new PostgresPgssReader().probe(client); + if (outcome.ok) { + const { detail, warnings } = outcome.runner.formatSuccessDetail(outcome.result); return { ok: true, - lines: [ - ` OK pg_stat_statements ready (${result.pgServerVersion})`, - ...result.warnings.map((warning: string) => ` ! ${warning}`), - ], - }; - } catch (error) { - return { ok: false, lines: historicSqlProbeFailureLines(error) }; - } finally { - await client.cleanup(); - } -} - -async function probeSnowflakeHistoricSql( - input: KtxSetupHistoricSqlProbeInput, -): Promise { - const project = await loadKtxProject({ projectDir: input.projectDir }); - const connection = project.config.connections[input.connectionId]; - const [{ SnowflakeHistoricSqlQueryHistoryReader }, { KtxSnowflakeHistoricSqlQueryClient }, { isKtxSnowflakeConnectionConfig }] = - await Promise.all([ - import('./context/ingest/adapters/historic-sql/snowflake-query-history-reader.js'), - import('./connectors/snowflake/historic-sql-query-client.js'), - import('./connectors/snowflake/connector.js'), - ]); - - if (!isKtxSnowflakeConnectionConfig(connection)) { - return { - ok: false, - lines: [` FAIL Connection ${input.connectionId} is not a native Snowflake connection.`], + lines: [` OK ${detail}`, ...warnings.map((warning) => ` ! ${warning}`)], }; } - - const client = new KtxSnowflakeHistoricSqlQueryClient({ - connectionId: input.connectionId, - connection, - projectDir: input.projectDir, - }); - try { - const result = await new SnowflakeHistoricSqlQueryHistoryReader().probe(client); - return { - ok: true, - lines: [ - ' OK SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY accessible', - ...result.warnings.map((warning: string) => ` ! ${warning}`), - ], - }; - } catch (error) { - return { ok: false, lines: historicSqlProbeFailureLines(error) }; - } finally { - await client.cleanup(); - } + const advice = outcome.runner.fixAdvice(outcome.error); + return { + ok: false, + lines: [` FAIL ${advice.failHeadline}`, ` Fix: ${advice.remediation}`], + }; } async function defaultListSchemas(projectDir: string, connectionId: string): Promise { const project = await loadKtxProject({ projectDir }); const connection = project.config.connections[connectionId]; const driver = normalizeDriver(connection?.driver); + const registration = driver ? getDriverRegistration(driver) : undefined; + if (!registration) return []; - if (driver === 'postgres') { - const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('./connectors/postgres/connector.js');; - if (!isKtxPostgresConnectionConfig(connection)) return []; - const connector = new KtxPostgresScanConnector({ connectionId, connection }); - try { - return await connector.listSchemas(); - } finally { - await connector.cleanup(); - } + const connectorModule = await registration.load(); + if (!connectorModule.isConnectionConfig(connection)) return []; + + const connector = connectorModule.createScanConnector({ connectionId, connection, projectDir }); + try { + return await connector.listSchemas(); + } finally { + await connector.cleanup?.(); } - - if (driver === 'sqlserver') { - const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('./connectors/sqlserver/connector.js');; - if (!isKtxSqlServerConnectionConfig(connection)) return []; - const connector = new KtxSqlServerScanConnector({ connectionId, connection }); - try { - return await connector.listSchemas(); - } finally { - await connector.cleanup(); - } - } - - if (driver === 'mysql') { - const { KtxMysqlScanConnector, isKtxMysqlConnectionConfig } = await import('./connectors/mysql/connector.js');; - if (!isKtxMysqlConnectionConfig(connection)) return []; - const connector = new KtxMysqlScanConnector({ connectionId, connection }); - try { - return await connector.listSchemas(); - } finally { - await connector.cleanup(); - } - } - - if (driver === 'clickhouse') { - const { KtxClickHouseScanConnector, isKtxClickHouseConnectionConfig } = await import('./connectors/clickhouse/connector.js');; - if (!isKtxClickHouseConnectionConfig(connection)) return []; - const connector = new KtxClickHouseScanConnector({ connectionId, connection }); - try { - return await connector.listSchemas(); - } finally { - await connector.cleanup(); - } - } - - if (driver === 'bigquery') { - const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('./connectors/bigquery/connector.js');; - if (!isKtxBigQueryConnectionConfig(connection)) return []; - const connector = new KtxBigQueryScanConnector({ connectionId, connection }); - try { - return await connector.listDatasets(); - } finally { - await connector.cleanup(); - } - } - - if (driver === 'snowflake') { - const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/connector.js');; - if (!isKtxSnowflakeConnectionConfig(connection)) return []; - const connector = new KtxSnowflakeScanConnector({ connectionId, connection, projectDir }); - try { - return await connector.listSchemas(); - } finally { - await connector.cleanup(); - } - } - - return []; } function configuredSchemas(connection: KtxProjectConnectionConfig | undefined, driver: KtxSetupDatabaseDriver): string[] | undefined { @@ -518,74 +428,18 @@ async function defaultListTables( const connection = project.config.connections[connectionId]; const driver = normalizeDriver(connection?.driver); const schemas = schemasOverride ?? (driver ? configuredSchemas(connection, driver) : undefined); + const registration = driver ? getDriverRegistration(driver) : undefined; + if (!registration) return []; - if (driver === 'postgres') { - const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('./connectors/postgres/connector.js');; - if (!isKtxPostgresConnectionConfig(connection)) return []; - const connector = new KtxPostgresScanConnector({ connectionId, connection }); - try { - return await connector.listTables(schemas); - } finally { - await connector.cleanup(); - } + const connectorModule = await registration.load(); + if (!connectorModule.isConnectionConfig(connection)) return []; + + const connector = connectorModule.createScanConnector({ connectionId, connection, projectDir }); + try { + return await connector.listTables(schemas); + } finally { + await connector.cleanup?.(); } - - if (driver === 'mysql') { - const { KtxMysqlScanConnector, isKtxMysqlConnectionConfig } = await import('./connectors/mysql/connector.js');; - if (!isKtxMysqlConnectionConfig(connection)) return []; - const connector = new KtxMysqlScanConnector({ connectionId, connection }); - try { - return await connector.listTables(schemas); - } finally { - await connector.cleanup(); - } - } - - if (driver === 'sqlserver') { - const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('./connectors/sqlserver/connector.js');; - if (!isKtxSqlServerConnectionConfig(connection)) return []; - const connector = new KtxSqlServerScanConnector({ connectionId, connection }); - try { - return await connector.listTables(schemas); - } finally { - await connector.cleanup(); - } - } - - if (driver === 'bigquery') { - const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('./connectors/bigquery/connector.js');; - if (!isKtxBigQueryConnectionConfig(connection)) return []; - const connector = new KtxBigQueryScanConnector({ connectionId, connection }); - try { - return await connector.listTables(schemas); - } finally { - await connector.cleanup(); - } - } - - if (driver === 'snowflake') { - const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/connector.js');; - if (!isKtxSnowflakeConnectionConfig(connection)) return []; - const connector = new KtxSnowflakeScanConnector({ connectionId, connection, projectDir }); - try { - return await connector.listTables(schemas); - } finally { - await connector.cleanup(); - } - } - - if (driver === 'clickhouse') { - const { KtxClickHouseScanConnector, isKtxClickHouseConnectionConfig } = await import('./connectors/clickhouse/connector.js');; - if (!isKtxClickHouseConnectionConfig(connection)) return []; - const connector = new KtxClickHouseScanConnector({ connectionId, connection }); - try { - return await connector.listTables(schemas); - } finally { - await connector.cleanup(); - } - } - - return []; } function existingConnectionIdsByDriver( @@ -635,6 +489,7 @@ function configuredPrimarySourcesPrompt(connectionIds: string[]): { message: `Databases configured: ${connectionIds.join(', ')}\nWhat would you like to do?`, options: [ { value: 'continue', label: 'Continue to context sources' }, + { value: 'skip-sources', label: 'Skip context sources' }, { value: 'edit', label: 'Edit an existing database' }, { value: 'add', label: 'Add another database' }, ], @@ -707,9 +562,9 @@ function scriptedScopeConfigForDriver( databaseSchemas: string[], ): Record { if (databaseSchemas.length === 0) return {}; - if (driver === 'bigquery') return { dataset_ids: databaseSchemas }; - if (driver === 'clickhouse') return { databases: databaseSchemas }; - return { schemas: databaseSchemas }; + const registration = getDriverRegistration(driver); + if (!registration?.scopeConfigKey) return {}; + return { [registration.scopeConfigKey]: databaseSchemas }; } function databaseNameFromLiteralUrl(url: string): string | undefined { @@ -1115,10 +970,14 @@ async function maybeApplyHistoricSqlConfig(input: { return withQueryHistoryConfig(input.connection, { ...existing, enabled: false }); } + const existingFilters = + existing.filters && typeof existing.filters === 'object' && !Array.isArray(existing.filters) + ? (existing.filters as Record) + : {}; const common: Record = { ...existing, enabled: true, - filters: historicSqlFiltersForSetup(input.args.queryHistoryServiceAccountPatterns), + filters: historicSqlFiltersForSetup(input.args.queryHistoryServiceAccountPatterns, existingFilters), }; if (dialect === 'postgres') { @@ -1135,9 +994,13 @@ async function maybeApplyHistoricSqlConfig(input: { }); } -function historicSqlFiltersForSetup(patterns: string[] | undefined) { +function historicSqlFiltersForSetup( + patterns: string[] | undefined, + existingFilters: Record = {}, +) { const serviceAccountPatterns = patterns ?? []; return { + ...existingFilters, dropTrivialProbes: true, ...(serviceAccountPatterns.length > 0 ? { @@ -1168,54 +1031,6 @@ async function defaultScanConnection(projectDir: string, connectionId: string, i ); } -interface BufferedCommandIo extends KtxCliIo { - stdoutText(): string; - stderrText(): string; -} - -function createBufferedCommandIo(): BufferedCommandIo { - let stdout = ''; - let stderr = ''; - return { - stdout: { - isTTY: false, - write(chunk: string) { - stdout += chunk; - }, - }, - stderr: { - write(chunk: string) { - stderr += chunk; - }, - }, - stdoutText() { - return stdout; - }, - stderrText() { - return stderr; - }, - }; -} - -function flushBufferedCommandOutput(io: KtxCliIo, bufferedIo: BufferedCommandIo): void { - const stdout = bufferedIo.stdoutText(); - const stderr = bufferedIo.stderrText(); - if (stdout.length > 0) { - io.stdout.write(stdout); - } - if (stderr.length > 0) { - io.stderr.write(stderr); - } -} - -function writePrefixedLines(write: (chunk: string) => void, output: string): void { - for (const line of output.split(/\r?\n/)) { - if (line.length > 0) { - write(`│ ${line}\n`); - } - } -} - function envWithCurrentNodeFirst(env: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv { return { ...env, @@ -1291,11 +1106,6 @@ async function defaultRebuildNativeSqlite(io: KtxCliIo): Promise { } } -function flushPrefixedBufferedCommandOutput(io: KtxCliIo, bufferedIo: BufferedCommandIo): void { - writePrefixedLines((chunk) => io.stdout.write(chunk), bufferedIo.stdoutText()); - writePrefixedLines((chunk) => io.stderr.write(chunk), bufferedIo.stderrText()); -} - function nativeSqliteAbiMismatchDetail(output: string): string | null { const mentionsBetterSqlite = /\bbetter-sqlite3\b|better_sqlite3/i.test(output); const mentionsAbiMismatch = /compiled against a different Node\.js version|NODE_MODULE_VERSION/i.test(output); @@ -1387,6 +1197,45 @@ async function writeConnectionConfig(input: { } } +async function disableConnectionQueryHistory(projectDir: string, connectionId: string): Promise { + const project = await loadKtxProject({ projectDir }); + const connection = project.config.connections[connectionId]; + if (!connection) { + return; + } + const existing = queryHistoryConfigRecord(connection) ?? historicSqlConfigRecord(connection) ?? {}; + await writeConnectionConfig({ + projectDir, + connectionId, + connection: withQueryHistoryConfig(connection, { ...existing, enabled: false }), + }); +} + +function okValidateResult(): ValidateResult { + return { status: 'ok' }; +} + +function backValidateResult(): ValidateResult { + return { status: 'back' }; +} + +function failedValidateResult(): ValidateResult { + return { status: 'failed' }; +} + +function queryHistoryUnavailableResult(projectDir: string, connectionId: string): ValidateResult { + return { + status: 'failed', + extraActions: [ + { + value: 'disable-query-history', + label: 'Disable query history and retry', + run: () => disableConnectionQueryHistory(projectDir, connectionId), + }, + ], + }; +} + async function createConnectionConfigRollback(projectDir: string, connectionId: string): Promise<() => Promise> { const project = await loadKtxProject({ projectDir }); const previousConnection = project.config.connections[connectionId]; @@ -1514,11 +1363,11 @@ async function maybeConfigureDatabaseScope(input: { io: KtxCliIo; prompts: KtxSetupDatabasesPromptAdapter; forcePrompt?: boolean; -}): Promise { +}): Promise { const project = await loadKtxProject({ projectDir: input.projectDir }); const connection = project.config.connections[input.connectionId]; const driver = normalizeDriver(connection?.driver); - if (!driver || driver === 'sqlite') return 'ready'; + if (!driver || driver === 'sqlite') return okValidateResult(); const spec = SCOPE_DISCOVERY_SPECS[driver]; const existingTables = connection?.enabled_tables; @@ -1527,7 +1376,7 @@ async function maybeConfigureDatabaseScope(input: { const hasExistingScope = !spec || existingScope.length > 0; if (hasExistingTables && hasExistingScope && input.forcePrompt !== true) { - return 'ready'; + return okValidateResult(); } const cliSchemas = input.args.databaseSchemas; @@ -1545,7 +1394,7 @@ async function maybeConfigureDatabaseScope(input: { input.io.stderr.write( `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; ${detail}\n`, ); - return 'ready'; + return okValidateResult(); } } if (scopeToWrite.length > 0) { @@ -1561,7 +1410,7 @@ async function maybeConfigureDatabaseScope(input: { ]); } } - return 'ready'; + return okValidateResult(); } if (spec && cliSchemas.length > 0) { @@ -1588,16 +1437,16 @@ async function maybeConfigureDatabaseScope(input: { input.connectionId, ); } catch (error) { - const detail = error instanceof Error ? error.message : String(error); - input.io.stderr.write( - `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; ${detail}\n`, + writePrefixedLines( + (chunk) => input.io.stderr.write(chunk), + `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; ${errorMessage(error)}`, ); const typed = await promptCommaSeparatedScope({ prompts: input.prompts, connectionId: input.connectionId, spec, }); - if (typed === undefined) return 'back'; + if (typed === undefined) return backValidateResult(); effectiveCliSchemas = typed; listedSchemas = typed; if (typed.length > 0) { @@ -1612,7 +1461,7 @@ async function maybeConfigureDatabaseScope(input: { } const schemas = unique(listedSchemas); if (spec && schemas.length === 0) { - return 'ready'; + return okValidateResult(); } const schemaSuggestion = effectiveCliSchemas.length > 0 @@ -1642,16 +1491,17 @@ async function maybeConfigureDatabaseScope(input: { input.io, ); } catch (error) { - const detail = error instanceof Error ? error.message : String(error); - input.io.stderr.write( + const detail = errorMessage(error); + writePrefixedLines( + (chunk) => input.io.stderr.write(chunk), input.forcePrompt === true - ? `Could not discover tables for ${input.connectionId}; edit was not saved. ${detail}\n` - : `Could not discover tables for ${input.connectionId}; continuing without table filter. ${detail}\n`, + ? `Could not discover tables for ${input.connectionId}; edit was not saved. ${detail}` + : `Could not discover tables for ${input.connectionId}; continuing without table filter. ${detail}`, ); - return input.forcePrompt === true ? 'failed' : 'ready'; + return input.forcePrompt === true ? failedValidateResult() : okValidateResult(); } if (pickResult.kind === 'back') { - return 'back'; + return backValidateResult(); } const enabledTables = pickResult.enabledTables; const activeSchemas = pickResult.activeSchemas; @@ -1666,7 +1516,7 @@ async function maybeConfigureDatabaseScope(input: { } const refreshedProject = await loadKtxProject({ projectDir: input.projectDir }); const currentConnection = refreshedProject.config.connections[input.connectionId]; - if (!currentConnection) return 'ready'; + if (!currentConnection) return okValidateResult(); await writeConnectionConfig({ projectDir: input.projectDir, connectionId: input.connectionId, @@ -1683,7 +1533,7 @@ async function maybeConfigureDatabaseScope(input: { writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [ `✓ ${enabledTables.length} tables enabled`, ]); - return 'ready'; + return okValidateResult(); } async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise { @@ -1713,7 +1563,18 @@ async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise { const project = await loadKtxProject({ projectDir }); - const config = setKtxSetupDatabaseConnectionIds(project.config, unique(connectionIds)); + const config = setKtxSetupDatabaseConnectionIds( + { + ...project.config, + connections: Object.fromEntries( + Object.entries(project.config.connections).map(([connectionId, connection]) => [ + connectionId, + migrateLegacyHistoricSqlConnection(connection), + ]), + ), + }, + unique(connectionIds), + ); await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); await markKtxSetupStateStepComplete(projectDir, 'databases'); } @@ -1723,33 +1584,249 @@ async function maybeRunHistoricSqlSetupProbe(input: { connectionId: string; io: KtxCliIo; deps: KtxSetupDatabasesDeps; -}): Promise { +}): Promise { const project = await loadKtxProject({ projectDir: input.projectDir }); const connection = project.config.connections[input.connectionId]; - const queryHistory = queryHistoryConfigRecord(connection); - const driver = normalizeDriver(connection?.driver); + const queryHistory = queryHistoryConfigRecord(connection) ?? historicSqlConfigRecord(connection); if (queryHistory?.enabled !== true) { - return; + return true; } - const dialect: 'postgres' | 'snowflake' | null = - driver === 'postgres' ? 'postgres' : driver === 'snowflake' ? 'snowflake' : null; + if (!connection) { + return true; + } + const dialect = queryHistoryDialectForConnection(connection); if (!dialect) { - return; + return true; } input.io.stdout.write('│ Query history probe...\n'); - const probe = input.deps.historicSqlProbe ?? defaultHistoricSqlProbe; - const result = await probe({ - projectDir: input.projectDir, - connectionId: input.connectionId, - dialect, - }); + const probe = input.deps.historicSqlReadinessProbe ?? runHistoricSqlReadinessProbe; + const result = setupHistoricSqlProbeResult( + await probe({ + projectDir: input.projectDir, + connectionId: input.connectionId, + connection, + env: process.env, + }), + ); for (const line of result.lines) { input.io.stdout.write(`│${line}\n`); } if (!result.ok) { input.io.stdout.write('│ Setup written; query history will be skipped until fixed.\n'); } + return result.ok; +} + +function hasServiceAccountsBlock(connection: KtxProjectConnectionConfig | undefined): boolean { + const queryHistory = queryHistoryConfigRecord(connection); + const filters = queryHistory?.filters; + if (!filters || typeof filters !== 'object' || Array.isArray(filters)) { + return false; + } + return 'serviceAccounts' in filters; +} + +function printQueryHistoryFilterProposal(io: KtxCliIo, proposal: QueryHistoryFilterProposal, debug = false): void { + if (debug && proposal.parseFailedTemplateIds.length > 0) { + io.stderr.write( + `[debug] query-history filter picker could not parse ${proposal.parseFailedTemplateIds.length} template(s): ${proposal.parseFailedTemplateIds.join(', ')}\n`, + ); + } + if (proposal.excludedRoles.length === 0) { + if (proposal.skipped?.reason === 'no-llm') { + io.stdout.write('│ Query-history filter picker skipped: no LLM is configured.\n'); + } else if (proposal.skipped?.reason === 'no-daemon') { + io.stdout.write('│ Query-history filter picker skipped: SQL analysis is unavailable.\n'); + } else if (proposal.skipped?.reason === 'no-in-scope-history') { + io.stdout.write('│ Query-history filter picker found no in-scope service-account exclusions.\n'); + } + if (proposal.parseFailedTemplateIds.length > 0) { + const count = proposal.parseFailedTemplateIds.length; + io.stdout.write( + `│ Skipped ${count} query template${count === 1 ? '' : 's'} ktx could not parse (run with --debug to list them).\n`, + ); + } + for (const warning of proposal.warnings) { + io.stdout.write(`│ ! ${warning}\n`); + } + return; + } + + io.stdout.write('│ Proposed query-history service-account filters:\n'); + for (const excluded of proposal.excludedRoles) { + io.stdout.write(`│ - ${excluded.role}: ${excluded.reason}\n`); + } +} + +async function shouldApplyQueryHistoryFilterProposal(input: { + args: KtxSetupDatabasesArgs; + prompts: KtxSetupDatabasesPromptAdapter; + proposal: QueryHistoryFilterProposal; +}): Promise { + if (input.proposal.excludedRoles.length === 0 || input.proposal.skipped?.reason === 'user-block-present') { + return false; + } + if (input.args.yes === true || input.args.inputMode === 'disabled') { + return true; + } + const choice = await input.prompts.select({ + message: `Apply ${input.proposal.excludedRoles.length} derived query-history service-account exclusion${ + input.proposal.excludedRoles.length === 1 ? '' : 's' + }?`, + options: [ + { value: 'apply', label: 'Apply derived filters (recommended)' }, + { value: 'skip', label: 'Leave query history filters unchanged' }, + ], + }); + return choice === 'apply'; +} + +function createSetupQueryHistoryLlmRuntime(input: { + projectDir: string; + project: Awaited>; + deps: KtxSetupDatabasesDeps; +}): KtxLlmRuntimePort | null { + try { + return ( + input.deps.createQueryHistoryLlmRuntime?.(input.projectDir, input.project) ?? + createLocalKtxLlmRuntimeFromConfig(input.project.config.llm, { + projectDir: input.projectDir, + }) + ); + } catch { + return null; + } +} + +/** @internal */ +export function managedDaemonOptionsForSetupQueryHistoryPicker(input: { + projectDir: string; + args: Pick; + io: KtxCliIo; +}): ManagedPythonCoreDaemonOptions { + return { + cliVersion: input.args.cliVersion ?? getKtxCliPackageInfo().version, + projectDir: input.projectDir, + installPolicy: input.args.runtimeInstallPolicy ?? (input.args.inputMode === 'disabled' ? 'never' : 'prompt'), + io: input.io, + }; +} + +async function maybeProposeQueryHistoryFilters(input: { + projectDir: string; + connectionId: string; + io: KtxCliIo; + deps: KtxSetupDatabasesDeps; + args: KtxSetupDatabasesArgs; + prompts: KtxSetupDatabasesPromptAdapter; +}): Promise { + const project = await loadKtxProject({ projectDir: input.projectDir }); + const connection = project.config.connections[input.connectionId]; + const queryHistory = queryHistoryConfigRecord(connection); + if (!connection || queryHistory?.enabled !== true) { + return; + } + const dialect = queryHistoryDialectForConnection(connection); + if (!dialect) { + return; + } + + const picker = input.deps.queryHistoryFilterPicker ?? proposeQueryHistoryServiceAccountFilters; + const llmRuntime = createSetupQueryHistoryLlmRuntime({ + projectDir: input.projectDir, + project, + deps: input.deps, + }); + if (!llmRuntime && !input.deps.queryHistoryFilterPicker) { + printQueryHistoryFilterProposal( + input.io, + { + excludedRoles: [], + consideredRoleCount: 0, + skipped: { reason: 'no-llm' }, + warnings: [], + parseFailedTemplateIds: [], + }, + input.args.debug === true, + ); + return; + } + + const runtime = createKtxCliHistoricSqlRuntime(project, input.connectionId, { + managedDaemon: managedDaemonOptionsForSetupQueryHistoryPicker({ + projectDir: input.projectDir, + args: input.args, + io: input.io, + }), + }); + if (!runtime) { + return; + } + const userServiceAccountsPresent = hasServiceAccountsBlock(connection); + const scopeFloor = await resolveQueryHistoryScopeFloor({ + projectDir: input.projectDir, + connectionId: input.connectionId, + driver: String(connection.driver ?? ''), + connection: connection as Record, + storedQueryHistory: queryHistory, + }); + const pullConfig = queryHistoryPullConfig({ + stored: queryHistory, + dialect, + enabledTables: scopeFloor.enabledTables, + enabledSchemas: scopeFloor.enabledSchemas, + modeledTableCatalog: scopeFloor.modeledTableCatalog, + scopeFloorWarnings: scopeFloor.warnings, + }); + const proposal = await picker({ + connectionId: input.connectionId, + dialect, + queryClient: runtime.queryClient, + reader: runtime.reader, + sqlAnalysis: runtime.sqlAnalysis, + llmRuntime, + pullConfig, + userServiceAccountsPresent, + }); + + printQueryHistoryFilterProposal(input.io, proposal, input.args.debug === true); + await emitTelemetryEvent({ + name: 'query_history_filter_completed', + projectDir: input.projectDir, + io: input.io, + fields: { + dialect, + consideredRoleCount: proposal.consideredRoleCount, + excludedRoleCount: proposal.excludedRoles.length, + parseFailedCount: proposal.parseFailedTemplateIds.length, + outcome: 'ok', + }, + }); + if (proposal.skipped?.reason === 'user-block-present') { + input.io.stdout.write('│ Existing query-history service-account filters left unchanged.\n'); + return; + } + if (!(await shouldApplyQueryHistoryFilterProposal({ args: input.args, prompts: input.prompts, proposal }))) { + return; + } + + await writeConnectionConfig({ + projectDir: input.projectDir, + connectionId: input.connectionId, + connection: withQueryHistoryConfig(connection, { + ...queryHistory, + filters: { + ...(queryHistory.filters && typeof queryHistory.filters === 'object' && !Array.isArray(queryHistory.filters) + ? queryHistory.filters + : {}), + serviceAccounts: { + mode: 'exclude', + patterns: proposal.excludedRoles.map((role) => role.pattern), + }, + }, + }), + }); } async function applyHistoricSqlConfigToExistingConnection(input: { @@ -1780,45 +1857,10 @@ async function applyHistoricSqlConfigToExistingConnection(input: { prompts: input.prompts, }); if (withHistoricSql === 'back') return 'back'; - const withContextDepth = await maybeApplyContextDepthConfig({ - projectDir: input.projectDir, - connectionId: input.connectionId, - connection: withHistoricSql, - args: input.args, - prompts: input.prompts, - }); - if (withContextDepth === 'back') return 'back'; await writeConnectionConfig({ projectDir: input.projectDir, connectionId: input.connectionId, - connection: withContextDepth, - }); -} - -async function maybeApplyContextDepthConfig(input: { - projectDir: string; - connectionId: string; - connection: KtxProjectConnectionConfig; - args: KtxSetupDatabasesArgs; - prompts: KtxSetupDatabasesPromptAdapter; -}): Promise { - const project = await loadKtxProject({ projectDir: input.projectDir }); - return await applySetupDatabaseContextDepth({ - project: { - ...project, - config: { - ...project.config, - connections: { - ...project.config.connections, - [input.connectionId]: input.connection, - }, - }, - }, - connection: input.connection, - args: { - inputMode: input.args.inputMode === 'disabled' || input.args.databaseUrl ? 'disabled' : input.args.inputMode, - }, - prompts: input.prompts, + connection: withHistoricSql, }); } @@ -1830,7 +1872,7 @@ async function validateAndScanConnection(input: { args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; forceScopeAndTables?: boolean; -}): Promise { +}): Promise { const testConnection = input.deps.testConnection ?? defaultTestConnection; const scanConnection = input.deps.scanConnection ?? defaultScanConnection; const project = await loadKtxProject({ projectDir: input.projectDir }); @@ -1839,9 +1881,12 @@ async function validateAndScanConnection(input: { const testIo = createBufferedCommandIo(); const testCode = await testConnection(input.projectDir, input.connectionId, testIo); if (testCode !== 0) { - flushBufferedCommandOutput(input.io, testIo); - input.io.stderr.write(`Connection test failed for ${input.connectionId}.\n`); - return 'failed'; + flushPrefixedBufferedCommandOutput(input.io, testIo); + writePrefixedLines( + (chunk) => input.io.stderr.write(chunk), + `Connection test failed for ${input.connectionId}.`, + ); + return failedValidateResult(); } const testOutput = testIo.stdoutText(); const outputDriver = normalizeDriver(readOutputValue(testOutput, 'Driver')); @@ -1850,18 +1895,18 @@ async function validateAndScanConnection(input: { writeSetupSection(input.io, `Testing ${input.connectionId}`, testLines); const scopeStatus = await maybeConfigureDatabaseScope({ ...input, forcePrompt: input.forceScopeAndTables }); - if (scopeStatus !== 'ready') { + if (scopeStatus.status !== 'ok') { return scopeStatus; } - await maybeRunHistoricSqlSetupProbe({ + const queryHistoryAvailable = await maybeRunHistoricSqlSetupProbe({ projectDir: input.projectDir, connectionId: input.connectionId, io: input.io, deps: input.deps, }); writeSetupSection(input.io, `Building schema context for ${input.connectionId}`, [ - 'Running fast database ingest…', + 'Running database scan…', ]); let scanIo = createBufferedCommandIo(); let scanCode = await scanConnection(input.projectDir, input.connectionId, scanIo); @@ -1871,7 +1916,7 @@ async function validateAndScanConnection(input: { writePrefixedLines( (chunk) => input.io.stderr.write(chunk), [ - `Fast database ingest failed for ${input.connectionId}.`, + `Database scan failed for ${input.connectionId}.`, 'Native SQLite is built for a different Node.js ABI.', `Detail: ${nativeSqliteDetail}`, 'Rebuilding Native SQLite with pnpm run native:rebuild…', @@ -1882,7 +1927,7 @@ async function validateAndScanConnection(input: { if (rebuildCode === 0) { writePrefixedLines( (chunk) => input.io.stderr.write(chunk), - 'Native SQLite rebuild complete. Retrying fast database ingest…', + 'Native SQLite rebuild complete. Retrying database scan…', ); const retryScanIo = createBufferedCommandIo(); scanCode = await scanConnection(input.projectDir, input.connectionId, retryScanIo); @@ -1893,10 +1938,10 @@ async function validateAndScanConnection(input: { (chunk) => input.io.stderr.write(chunk), [ rebuildCode === 0 - ? `Fast database ingest still failed for ${input.connectionId} after rebuilding Native SQLite.` + ? `Database scan still failed for ${input.connectionId} after rebuilding Native SQLite.` : `Native SQLite rebuild failed for ${input.connectionId}.`, 'Fix: pnpm run native:rebuild', - `Retry: ktx ingest ${input.connectionId} --project-dir ${input.projectDir} --fast`, + `Retry: ktx ingest ${input.connectionId} --project-dir ${input.projectDir}`, ].join('\n'), ); } @@ -1905,13 +1950,15 @@ async function validateAndScanConnection(input: { writePrefixedLines( (chunk) => input.io.stderr.write(chunk), [ - `Fast database ingest failed for ${input.connectionId}.`, - `Debug command: ktx ingest ${input.connectionId} --project-dir ${input.projectDir} --fast --debug`, + `Database scan failed for ${input.connectionId}.`, + `Debug command: ktx ingest ${input.connectionId} --project-dir ${input.projectDir} --debug`, ].join('\n'), ); } if (scanCode !== 0) { - return 'failed'; + return queryHistoryAvailable + ? failedValidateResult() + : queryHistoryUnavailableResult(input.projectDir, input.connectionId); } } const scanOutput = scanIo.stdoutText(); @@ -1920,10 +1967,20 @@ async function validateAndScanConnection(input: { `Schema context complete for ${input.connectionId}`, [`Changes: ${summarizeScanChanges(scanOutput)}`], ); + if (queryHistoryAvailable) { + await maybeProposeQueryHistoryFilters({ + projectDir: input.projectDir, + connectionId: input.connectionId, + io: input.io, + deps: input.deps, + args: input.args, + prompts: input.prompts, + }); + } writeSetupSection(input.io, 'Database ready', [ `${input.connectionId} · ${driverDisplay} · schema context complete`, ]); - return 'ready'; + return okValidateResult(); } async function chooseDrivers( @@ -1945,6 +2002,11 @@ async function chooseDrivers( return 'missing-input'; } const initialValues = unique(options?.initialDrivers ?? []); + createKtxSetupUiAdapter().note( + `Get demo credentials: ${KTX_DEMO_START_URL}`, + '🎁 Need a warehouse to play with?', + io, + ); const choices = await prompts.multiselect({ message: withMultiselectNavigation('Which databases should KTX connect to?'), options: [...DRIVER_OPTIONS], @@ -2041,6 +2103,149 @@ async function choosePrimarySourceToEdit(input: { return choice === 'back' ? 'back' : choice; } +async function configureDatabaseConnection(input: { + projectDir: string; + connectionId: string; + driver: KtxSetupDatabaseDriver; + args: KtxSetupDatabasesArgs; + prompts: KtxSetupDatabasesPromptAdapter; + io: KtxCliIo; + canReturnToDriverSelection: boolean; + editBaseline?: KtxProjectConnectionConfig; +}): Promise { + const project = await loadKtxProject({ projectDir: input.projectDir }); + const latestConnection = project.config.connections[input.connectionId]; + let connection = await buildConnectionConfig({ + driver: input.driver, + connectionId: input.connectionId, + args: input.args, + prompts: input.prompts, + existingConnection: latestConnection, + }); + + while (!connection && input.args.inputMode !== 'disabled') { + const action = await input.prompts.select( + missingConnectionDetailsPrompt(driverLabel(input.driver), input.canReturnToDriverSelection), + ); + if (action === 'back') { + return 'back'; + } + connection = await buildConnectionConfig({ + driver: input.driver, + connectionId: input.connectionId, + args: input.args, + prompts: input.prompts, + existingConnection: latestConnection, + }); + } + + if (connection === 'back') { + return 'back'; + } + if (!connection) { + input.io.stderr.write(`Missing connection details for ${driverLabel(input.driver)}.\n`); + return 'cancelled'; + } + + const withHistoricSql = await maybeApplyHistoricSqlConfig({ + connection, + driver: input.driver, + args: input.args, + prompts: input.prompts, + }); + if (withHistoricSql === 'back') { + return 'back'; + } + + await writeConnectionConfig({ + projectDir: input.projectDir, + connectionId: input.connectionId, + connection: input.editBaseline + ? withExistingPrimaryEditPromptDefaults({ + previous: input.editBaseline, + next: withHistoricSql, + driver: input.driver, + }) + : withHistoricSql, + io: input.io, + }); + return 'configured'; +} + +async function runDatabaseConnectionSetupWithRecovery(input: { + projectDir: string; + connectionId: string; + driver: KtxSetupDatabaseDriver; + args: KtxSetupDatabasesArgs; + prompts: KtxSetupDatabasesPromptAdapter; + io: KtxCliIo; + deps: KtxSetupDatabasesDeps; + canReturnToDriverSelection: boolean; + allowSkip: boolean; + interactive?: boolean; + forceScopeAndTables?: boolean; + editBaseline?: KtxProjectConnectionConfig; + reuseExistingOnFirstConfigure?: boolean; +}): Promise { + let configureCalls = 0; + // `configureDatabaseConnection` returns 'cancelled' only when required + // connection details are absent in non-interactive mode. The recovery + // primitive collapses that into 'failed', so we track it here to restore the + // distinct 'missing-input' outcome the surrounding step reports for + // incomplete flags (vs. a real connection/probe failure). + let sawMissingInput = false; + + const outcome = await runConnectionSetupWithRecovery({ + label: input.connectionId, + interactive: input.interactive ?? input.args.inputMode !== 'disabled', + allowSkip: input.allowSkip, + io: input.io, + prompts: input.prompts, + snapshot: () => createConnectionConfigRollback(input.projectDir, input.connectionId), + configure: async () => { + configureCalls += 1; + if (input.reuseExistingOnFirstConfigure && configureCalls === 1) { + const historicSqlResult = await applyHistoricSqlConfigToExistingConnection({ + projectDir: input.projectDir, + connectionId: input.connectionId, + args: input.args, + prompts: input.prompts, + }); + return historicSqlResult === 'back' ? 'back' : 'configured'; + } + const configured = await configureDatabaseConnection({ + projectDir: input.projectDir, + connectionId: input.connectionId, + driver: input.driver, + args: input.args, + prompts: input.prompts, + io: input.io, + canReturnToDriverSelection: input.canReturnToDriverSelection, + editBaseline: input.editBaseline, + }); + if (configured === 'cancelled') { + sawMissingInput = true; + } + return configured; + }, + validate: () => + validateAndScanConnection({ + projectDir: input.projectDir, + connectionId: input.connectionId, + io: input.io, + deps: input.deps, + args: input.args, + prompts: input.prompts, + forceScopeAndTables: input.forceScopeAndTables, + }), + }); + + if (outcome === 'failed' && sawMissingInput) { + return 'missing-input'; + } + return outcome; +} + async function runPrimarySourceFullEdit(input: { projectDir: string; connectionId: string; @@ -2048,68 +2253,33 @@ async function runPrimarySourceFullEdit(input: { prompts: KtxSetupDatabasesPromptAdapter; io: KtxCliIo; deps: KtxSetupDatabasesDeps; -}): Promise<'ready' | 'back' | 'failed'> { +}): Promise<'ready' | 'back' | 'failed' | 'missing-input'> { const project = await loadKtxProject({ projectDir: input.projectDir }); const existing = project.config.connections[input.connectionId]; const driver = normalizeDriver(existing?.driver); if (!existing || !driver) { - input.io.stderr.write(`Connection "${input.connectionId}" is not a configured database.\n`); + writePrefixedLines( + (chunk) => input.io.stderr.write(chunk), + `Connection "${input.connectionId}" is not a configured database.`, + ); return 'failed'; } - const rollback = await createConnectionConfigRollback(input.projectDir, input.connectionId); - const replacement = await buildConnectionConfig({ - driver, + const outcome = await runDatabaseConnectionSetupWithRecovery({ + projectDir: input.projectDir, connectionId: input.connectionId, - args: input.args, - prompts: input.prompts, - existingConnection: existing, - }); - if (replacement === 'back') { - await rollback(); - return 'back'; - } - if (!replacement) { - await rollback(); - return 'failed'; - } - - const withHistoricSql = await maybeApplyHistoricSqlConfig({ - connection: replacement, driver, args: input.args, prompts: input.prompts, - }); - if (withHistoricSql === 'back') { - await rollback(); - return 'back'; - } - - await writeConnectionConfig({ - projectDir: input.projectDir, - connectionId: input.connectionId, - connection: withExistingPrimaryEditPromptDefaults({ - previous: existing, - next: withHistoricSql, - driver, - }), - io: input.io, - }); - - const validated = await validateAndScanConnection({ - projectDir: input.projectDir, - connectionId: input.connectionId, io: input.io, deps: input.deps, - args: input.args, - prompts: input.prompts, + canReturnToDriverSelection: true, + allowSkip: false, forceScopeAndTables: true, + editBaseline: existing, }); - if (validated !== 'ready') { - await rollback(); - return validated; - } - return 'ready'; + + return outcome === 'skip' ? 'back' : outcome; } export async function runKtxSetupDatabasesStep( @@ -2127,28 +2297,37 @@ export async function runKtxSetupDatabasesStep( if (args.databaseConnectionIds && args.databaseConnectionIds.length > 0) { const selectedConnectionIds: string[] = []; for (const connectionId of unique(args.databaseConnectionIds)) { - const historicSqlResult = await applyHistoricSqlConfigToExistingConnection({ - projectDir: args.projectDir, - connectionId, - args, - prompts, - }); - if (historicSqlResult === 'back') return { status: 'back', projectDir: args.projectDir }; - const setupStatus = await validateAndScanConnection({ - projectDir: args.projectDir, - connectionId, - io, - deps, - args, - prompts, - }); - if (setupStatus === 'back') { - return { status: 'back', projectDir: args.projectDir }; - } - if (setupStatus === 'failed') { + const project = await loadKtxProject({ projectDir: args.projectDir }); + const driver = normalizeDriver(project.config.connections[connectionId]?.driver); + if (!driver) { + writePrefixedLines((chunk) => io.stderr.write(chunk), `Connection "${connectionId}" is not configured.`); return { status: 'failed', projectDir: args.projectDir }; } - selectedConnectionIds.push(connectionId); + const setupOutcome = await runDatabaseConnectionSetupWithRecovery({ + projectDir: args.projectDir, + connectionId, + driver, + args, + prompts, + io, + deps, + canReturnToDriverSelection: false, + allowSkip: false, + interactive: false, + reuseExistingOnFirstConfigure: true, + }); + if (setupOutcome === 'back') { + return { status: 'back', projectDir: args.projectDir }; + } + if (setupOutcome === 'missing-input') { + return { status: 'missing-input', projectDir: args.projectDir }; + } + if (setupOutcome === 'failed') { + return { status: 'failed', projectDir: args.projectDir }; + } + if (setupOutcome === 'ready') { + selectedConnectionIds.push(connectionId); + } } await markDatabasesComplete(args.projectDir, selectedConnectionIds); return { status: 'ready', projectDir: args.projectDir, connectionIds: selectedConnectionIds }; @@ -2169,6 +2348,15 @@ export async function runKtxSetupDatabasesStep( await markDatabasesComplete(args.projectDir, selectedConnectionIds); return { status: 'ready', projectDir: args.projectDir, connectionIds: selectedConnectionIds }; } + if (action === 'skip-sources') { + await markDatabasesComplete(args.projectDir, selectedConnectionIds); + return { + status: 'ready', + projectDir: args.projectDir, + connectionIds: selectedConnectionIds, + skipSources: true, + }; + } if (action === 'edit') { const connectionId = await choosePrimarySourceToEdit({ projectDir: args.projectDir, @@ -2191,6 +2379,9 @@ export async function runKtxSetupDatabasesStep( showConfiguredPrimaryMenu = true; continue; } + if (editResult === 'missing-input') { + return { status: 'missing-input', projectDir: args.projectDir }; + } if (editResult === 'failed') { return { status: 'failed', projectDir: args.projectDir }; } @@ -2233,7 +2424,7 @@ export async function runKtxSetupDatabasesStep( prompts, }); } catch (error) { - io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + writePrefixedLines((chunk) => io.stderr.write(chunk), errorMessage(error)); return { status: 'failed', projectDir: args.projectDir }; } if (connectionChoice === 'back') { @@ -2246,7 +2437,6 @@ export async function runKtxSetupDatabasesStep( return { status: 'missing-input', projectDir: args.projectDir }; } - let connectionAlreadyValidated = false; if (connectionChoice.kind === 'edit') { const editResult = await runPrimarySourceFullEdit({ projectDir: args.projectDir, @@ -2261,198 +2451,41 @@ export async function runKtxSetupDatabasesStep( returnToDriverSelection = true; break; } + if (editResult === 'missing-input') { + return { status: 'missing-input', projectDir: args.projectDir }; + } if (editResult === 'failed') { return { status: 'failed', projectDir: args.projectDir }; } - connectionAlreadyValidated = true; - } else if (connectionChoice.kind === 'new') { - let connection = await buildConnectionConfig({ - driver, + } else { + const setupOutcome = await runDatabaseConnectionSetupWithRecovery({ + projectDir: args.projectDir, connectionId: connectionChoice.connectionId, + driver, args, prompts, + io, + deps, + canReturnToDriverSelection, + allowSkip: true, + reuseExistingOnFirstConfigure: connectionChoice.kind === 'existing', }); - if (connection === 'back') { + if (setupOutcome === 'back') { if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; returnToDriverSelection = true; break; } - while (!connection && args.inputMode !== 'disabled') { - const label = driverLabel(driver); - const action = await prompts.select(missingConnectionDetailsPrompt(label, canReturnToDriverSelection)); - if (action === 'back') { - if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; - returnToDriverSelection = true; - break; - } - connection = await buildConnectionConfig({ - driver, - connectionId: connectionChoice.connectionId, - args, - prompts, - }); - if (connection === 'back') { - if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; - returnToDriverSelection = true; - break; - } - } - if (returnToDriverSelection) { - break; - } - if (connection === 'back') { - break; - } - if (!connection) { - io.stderr.write(`Missing connection details for ${driverLabel(driver)}.\n`); + if (setupOutcome === 'missing-input') { return { status: 'missing-input', projectDir: args.projectDir }; } - const withHistoricSql = await maybeApplyHistoricSqlConfig({ connection, driver, args, prompts }); - if (withHistoricSql === 'back') { - if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; - returnToDriverSelection = true; - break; + if (setupOutcome === 'failed') { + return { status: 'failed', projectDir: args.projectDir }; } - const withContextDepth = await maybeApplyContextDepthConfig({ - projectDir: args.projectDir, - connectionId: connectionChoice.connectionId, - connection: withHistoricSql, - args, - prompts, - }); - if (withContextDepth === 'back') { - if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; - returnToDriverSelection = true; - break; - } - await writeConnectionConfig({ - projectDir: args.projectDir, - connectionId: connectionChoice.connectionId, - connection: withContextDepth, - io, - }); - } else { - const existing = project.config.connections[connectionChoice.connectionId]; - const withHistoricSql = await maybeApplyHistoricSqlConfig({ connection: existing, driver, args, prompts }); - if (withHistoricSql === 'back') { - if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; - returnToDriverSelection = true; - break; - } - const withContextDepth = await maybeApplyContextDepthConfig({ - projectDir: args.projectDir, - connectionId: connectionChoice.connectionId, - connection: withHistoricSql, - args, - prompts, - }); - if (withContextDepth === 'back') { - if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; - returnToDriverSelection = true; - break; - } - await writeConnectionConfig({ - projectDir: args.projectDir, - connectionId: connectionChoice.connectionId, - connection: withContextDepth, - io, - }); - } - - let connectionSkipped = false; - let setupStatus: ConnectionSetupStatus = connectionAlreadyValidated - ? 'ready' - : await validateAndScanConnection({ - projectDir: args.projectDir, - connectionId: connectionChoice.connectionId, - io, - deps, - args, - prompts, - }); - while (!connectionAlreadyValidated && setupStatus !== 'ready') { - if (setupStatus === 'back') { - if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; - returnToDriverSelection = true; - break; - } - if (args.inputMode === 'disabled') return { status: 'failed', projectDir: args.projectDir }; - const action = await prompts.select({ - message: `Database setup failed for ${connectionChoice.connectionId}`, - options: [ - { value: 'retry', label: 'Retry connection test' }, - { value: 're-enter', label: 'Re-enter connection details' }, - { value: 'skip', label: 'Skip this database' }, - { value: 'back', label: 'Back' }, - ], - }); - if (action === 'back') { - if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; - returnToDriverSelection = true; - break; - } - if (action === 'skip') { - connectionSkipped = true; - break; - } - if (action === 'retry') { - setupStatus = await validateAndScanConnection({ - projectDir: args.projectDir, - connectionId: connectionChoice.connectionId, - io, - deps, - args, - prompts, - }); - } else if (action === 're-enter') { - const connection = await buildConnectionConfig({ - driver, - connectionId: connectionChoice.connectionId, - args, - prompts, - }); - if (connection === 'back') { - if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; - returnToDriverSelection = true; - break; - } - if (!connection) continue; - const withHistoricSql = await maybeApplyHistoricSqlConfig({ connection, driver, args, prompts }); - if (withHistoricSql === 'back') { - if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; - returnToDriverSelection = true; - break; - } - const withContextDepth = await maybeApplyContextDepthConfig({ - projectDir: args.projectDir, - connectionId: connectionChoice.connectionId, - connection: withHistoricSql, - args, - prompts, - }); - if (withContextDepth === 'back') { - if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; - returnToDriverSelection = true; - break; - } - await writeConnectionConfig({ - projectDir: args.projectDir, - connectionId: connectionChoice.connectionId, - connection: withContextDepth, - io, - }); - setupStatus = await validateAndScanConnection({ - projectDir: args.projectDir, - connectionId: connectionChoice.connectionId, - io, - deps, - args, - prompts, - }); + if (setupOutcome === 'skip') { + continue; } } if (returnToDriverSelection) break; - if (connectionSkipped) continue; pushUniqueConnectionId(selectedConnectionIds, connectionChoice.connectionId); } diff --git a/packages/cli/src/setup-embeddings.ts b/packages/cli/src/setup-embeddings.ts index 0aedd264..5d02e3e4 100644 --- a/packages/cli/src/setup-embeddings.ts +++ b/packages/cli/src/setup-embeddings.ts @@ -6,12 +6,13 @@ import { markKtxSetupStateStepComplete, readKtxSetupState } from './context/proj import type { KtxEmbeddingConfig } from './llm/types.js'; import { type KtxEmbeddingHealthCheckResult, runKtxEmbeddingHealthCheck } from './llm/embedding-health.js'; import type { KtxCliIo } from './cli-runtime.js'; -import { createStaticCliSpinner, type KtxCliSpinner } from './clack.js'; +import { createStaticCliSpinner, errorMessage, writePrefixedLines, type KtxCliSpinner } from './clack.js'; import { ensureManagedLocalEmbeddingsDaemon, managedLocalEmbeddingHealthConfig, type ManagedLocalEmbeddingsDaemon, } from './managed-local-embeddings.js'; +import { ManagedPythonDaemonStartError } from './managed-python-daemon.js'; import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; import { withTextInputNavigation } from './prompt-navigation.js'; import { envCredentialReference, writeProjectLocalSecretReference } from './setup-secrets.js'; @@ -221,8 +222,8 @@ async function chooseCredentialRef( const choice = await prompts.select({ message: `How should KTX find your ${embeddingBackendDisplayName(backend)} embedding API key?`, options: [ - { value: 'env', label: `Use ${defaultEnv} from the environment` }, { value: 'paste', label: 'Paste a key and save it as a local secret file' }, + { value: 'env', label: `Use ${defaultEnv} from the environment` }, { value: 'back', label: 'Back' }, ], }); @@ -419,7 +420,13 @@ export async function runKtxSetupEmbeddingsStep( io, }); } catch (error) { - io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + const write = (chunk: string) => io.stderr.write(chunk); + if (error instanceof ManagedPythonDaemonStartError) { + const tail = await readLocalEmbeddingDaemonStderrTail(error.stderrLog); + writePrefixedLines(write, localEmbeddingSetupMessage(error.detail, tail)); + } else { + writePrefixedLines(write, errorMessage(error)); + } return { status: 'failed', projectDir: args.projectDir }; } } diff --git a/packages/cli/src/setup-models.ts b/packages/cli/src/setup-models.ts index 041eef5c..e673cb99 100644 --- a/packages/cli/src/setup-models.ts +++ b/packages/cli/src/setup-models.ts @@ -3,6 +3,9 @@ import { writeFile } from 'node:fs/promises'; import { promisify } from 'node:util'; import { resolveLocalKtxLlmConfig } from './context/llm/local-config.js'; import { runClaudeCodeAuthProbe } from './context/llm/claude-code-runtime.js'; +import { formatCodexIsolationWarning } from './context/llm/codex-isolation.js'; +import { runCodexAuthProbe } from './context/llm/codex-runtime.js'; +import { DEFAULT_CODEX_MODEL } from './context/llm/codex-models.js'; import { resolveKtxConfigReference } from './context/core/config-reference.js'; import { type KtxProjectConfig, type KtxProjectLlmConfig, serializeKtxProjectConfig } from './context/project/config.js'; import { loadKtxProject } from './context/project/project.js'; @@ -56,7 +59,7 @@ export interface AnthropicModelChoice { recommended: boolean; } -export type KtxSetupLlmBackend = 'anthropic' | 'vertex' | 'claude-code'; +export type KtxSetupLlmBackend = 'anthropic' | 'vertex' | 'claude-code' | 'codex'; /** @internal */ export interface KtxSetupModelPromptAdapter { @@ -82,6 +85,7 @@ export interface KtxSetupModelDeps { model: string; env?: NodeJS.ProcessEnv; }) => Promise<{ ok: true } | { ok: false; message: string }>; + codexAuthProbe?: (input: { projectDir: string; model: string }) => Promise<{ ok: true } | { ok: false; message: string }>; readGcloudProject?: () => Promise; listGcloudProjects?: () => Promise; spinner?: () => KtxCliSpinner; @@ -110,6 +114,20 @@ const CLAUDE_CODE_MODELS: AnthropicModelChoice[] = [ { id: 'haiku', label: 'Claude Haiku', recommended: false }, ]; +// Curated Codex models from OpenAI's current lineup that work under both +// ChatGPT-account (subscription) and API-key auth. Intentionally omitted: +// the `*-codex` ids (e.g. gpt-5.3-codex, gpt-5.2-codex) are API-key-only and +// fail on ChatGPT-account auth, and gpt-5.3-codex-spark is a ChatGPT-Pro-only +// research preview. Codex resolves real availability per account at runtime +// (its binary remote-fetches the model list), so this is a convenience +// shortlist only — the manual-entry option accepts any id your account's +// `codex` picker exposes, and the auth probe reports an unsupported choice. +const CODEX_MODELS: AnthropicModelChoice[] = [ + { id: 'gpt-5.5', label: 'GPT-5.5', recommended: true }, + { id: 'gpt-5.4', label: 'GPT-5.4', recommended: false }, + { id: 'gpt-5.4-mini', label: 'GPT-5.4 mini', recommended: false }, +]; + const HIDDEN_ANTHROPIC_MODEL_PATTERNS = [ /^claude-sonnet-4$/i, /^claude-opus-4$/i, @@ -272,7 +290,12 @@ export function isKtxSetupLlmConfigReady(config: KtxProjectLlmConfig): boolean { return typeof resolved.vertex?.location === 'string' && resolved.vertex.location.trim().length > 0; } - return resolved.backend === 'anthropic' || resolved.backend === 'gateway' || resolved.backend === 'claude-code'; + return ( + resolved.backend === 'anthropic' || + resolved.backend === 'gateway' || + resolved.backend === 'claude-code' || + resolved.backend === 'codex' + ); } function hasUsableConfiguredLlm(config: KtxProjectConfig): boolean { @@ -284,7 +307,8 @@ function buildProjectLlmConfig( provider: | { backend: 'anthropic'; credentialRef: string } | { backend: 'vertex'; vertex: { project?: string; location: string } } - | { backend: 'claude-code' }, + | { backend: 'claude-code' } + | { backend: 'codex' }, model: string, ): KtxProjectLlmConfig { if (provider.backend === 'claude-code') { @@ -295,6 +319,14 @@ function buildProjectLlmConfig( }; } + if (provider.backend === 'codex') { + return { + provider: { backend: 'codex' }, + models: { ...existing.models, default: model }, + promptCaching: existing.promptCaching, + }; + } + if (provider.backend === 'vertex') { return { provider: { @@ -438,8 +470,8 @@ async function chooseCredentialRef( const choice = await prompts.select({ message: `How should KTX find your Anthropic API key?\n\n${ANTHROPIC_CREDENTIAL_PROMPT_CONTEXT}`, options: [ - { value: 'env', label: 'Use ANTHROPIC_API_KEY from the environment' }, { value: 'paste', label: 'Paste a key and save it as a local secret file' }, + { value: 'env', label: 'Use ANTHROPIC_API_KEY from the environment' }, { value: 'back', label: 'Back' }, ], }); @@ -515,6 +547,7 @@ async function chooseBackend( message: 'Which LLM provider should KTX use?', options: [ { value: 'claude-code', label: 'Claude subscription (Pro/Max)' }, + { value: 'codex', label: 'Codex subscription' }, { value: 'anthropic', label: 'Anthropic API key' }, { value: 'vertex', label: 'Google Vertex AI for Anthropic Claude' }, { value: 'back', label: 'Back' }, @@ -525,7 +558,7 @@ async function chooseBackend( } return { status: 'ready', - backend: choice === 'vertex' || choice === 'claude-code' ? choice : 'anthropic', + backend: choice === 'vertex' || choice === 'claude-code' || choice === 'codex' ? choice : 'anthropic', prompted: true, }; } @@ -884,12 +917,51 @@ async function chooseClaudeCodeModel(args: KtxSetupModelArgs, deps: KtxSetupMode return { status: 'ready', model: choice }; } +async function chooseCodexModel(args: KtxSetupModelArgs, deps: KtxSetupModelDeps): Promise { + const providedModel = requestedModel(args); + if (providedModel) { + return { status: 'ready', model: providedModel }; + } + if (args.inputMode === 'disabled') { + return { status: 'ready', model: DEFAULT_CODEX_MODEL }; + } + + const prompts = deps.prompts ?? createPromptAdapter(); + const choice = await prompts.select({ + message: `Which Codex model should KTX use?\n\n${ANTHROPIC_MODEL_PROMPT_CONTEXT}`, + options: [ + ...CODEX_MODELS.map((model) => ({ + value: model.id, + label: model.label, + ...(model.recommended ? { hint: 'recommended' } : {}), + })), + { value: 'manual', label: 'Enter a Codex model ID manually' }, + { value: 'back', label: 'Back' }, + ], + }); + if (choice === 'back') { + return { status: 'back' }; + } + if (choice === 'manual') { + const manual = await prompts.text({ + message: withTextInputNavigation('Codex model ID'), + placeholder: CODEX_MODELS.find((model) => model.recommended)?.id ?? CODEX_MODELS[0]?.id, + }); + if (manual === undefined) { + return { status: 'back' }; + } + return manual.trim() ? { status: 'ready', model: manual.trim() } : { status: 'missing-input' }; + } + return { status: 'ready', model: choice }; +} + async function persistLlmConfig( projectDir: string, provider: | { backend: 'anthropic'; credentialRef: string } | { backend: 'vertex'; vertex: { project?: string; location: string } } - | { backend: 'claude-code' }, + | { backend: 'claude-code' } + | { backend: 'codex' }, model: string, ): Promise { const project = await loadKtxProject({ projectDir }); @@ -1031,6 +1103,32 @@ export async function runKtxSetupAnthropicModelStep( return { status: 'ready', projectDir: args.projectDir }; } + if (backendChoice.backend === 'codex') { + const model = await chooseCodexModel(backendArgs, deps); + if (model.status === 'back' && backendChoice.prompted) { + attemptArgs = buildInteractiveRetryArgs(args); + continue; + } + if (model.status === 'invalid-credential') { + return { status: 'failed', projectDir: args.projectDir }; + } + if (model.status !== 'ready') { + return { status: model.status, projectDir: args.projectDir }; + } + const probe = deps.codexAuthProbe ?? runCodexAuthProbe; + const health = await probe({ projectDir: args.projectDir, model: model.model }); + if (!health.ok) { + io.stderr.write(`${health.message}\n`); + return { status: 'failed', projectDir: args.projectDir }; + } + // Prefix the clack gutter so the warning sits inside the setup frame + // instead of breaking out of it; kept on stderr for scripted runs. + io.stderr.write(`│ ${formatCodexIsolationWarning()}\n`); + await persistLlmConfig(args.projectDir, { backend: 'codex' }, model.model); + io.stdout.write(`│ LLM ready: yes (codex, ${model.model})\n`); + return { status: 'ready', projectDir: args.projectDir }; + } + const credential = await chooseCredentialRef(backendArgs, io, deps); if (credential.status === 'back' && backendChoice.prompted) { attemptArgs = buildInteractiveRetryArgs(args); diff --git a/packages/cli/src/setup-project.ts b/packages/cli/src/setup-project.ts index d7d189e1..08f935e6 100644 --- a/packages/cli/src/setup-project.ts +++ b/packages/cli/src/setup-project.ts @@ -24,17 +24,12 @@ export interface KtxSetupProjectArgs { allowBack?: boolean; } -export type KtxSetupCreatedProjectCleanup = - | { kind: 'remove-project-dir'; projectDir: string } - | { kind: 'remove-ktx-scaffold'; projectDir: string }; - export type KtxSetupProjectResult = | { status: 'ready'; projectDir: string; project: KtxLocalProject; confirmedCreation?: boolean; - createdProjectCleanup?: KtxSetupCreatedProjectCleanup; } | { status: 'back'; projectDir: string } | { status: 'cancelled'; projectDir: string } @@ -59,7 +54,6 @@ type PromptProjectDirResult = status: 'selected'; projectDir: string; confirmedCreation: boolean; - createdProjectCleanup?: KtxSetupCreatedProjectCleanup; } | { status: 'cancelled'; projectDir: string } | { status: 'missing-input'; projectDir: string } @@ -106,26 +100,12 @@ type ConfirmProjectDirResult = | { status: 'confirmed'; confirmedCreation: boolean; - createdProjectCleanup?: KtxSetupCreatedProjectCleanup; } | { status: 'choose-another' } | { status: 'back' } | { status: 'cancelled' } | { status: 'not-directory' }; -function cleanupForFolderState( - projectDir: string, - state: Awaited>, -): KtxSetupCreatedProjectCleanup | undefined { - if (state === 'missing') { - return { kind: 'remove-project-dir', projectDir }; - } - if (state === 'empty-directory') { - return { kind: 'remove-ktx-scaffold', projectDir }; - } - return undefined; -} - async function confirmProjectDir( selectedDir: string, io: KtxCliIo, @@ -165,7 +145,7 @@ async function confirmProjectDir( if (action === 'choose-another') return { status: 'choose-another' }; if (action === 'back') return { status: 'back' }; if (action !== 'create') return { status: 'cancelled' }; - return { status: 'confirmed', confirmedCreation: true, createdProjectCleanup: cleanupForFolderState(selectedDir, state) }; + return { status: 'confirmed', confirmedCreation: true }; } async function normalizeSetupGitignore(projectDir: string): Promise { @@ -252,24 +232,10 @@ async function promptForNewProjectDir( status: 'selected', projectDir: selectedDir, confirmedCreation: confirmed.confirmedCreation, - ...(confirmed.createdProjectCleanup ? { createdProjectCleanup: confirmed.createdProjectCleanup } : {}), }; } } -async function createProjectWithCleanup( - projectDir: string, - deps: KtxSetupProjectDeps, -): Promise<{ project: KtxLocalProject; createdProjectCleanup?: KtxSetupCreatedProjectCleanup }> { - const state = await existingFolderState(projectDir); - const project = await createProject(projectDir, deps); - const createdProjectCleanup = cleanupForFolderState(projectDir, state); - return { - project, - ...(createdProjectCleanup ? { createdProjectCleanup } : {}), - }; -} - export async function runKtxSetupProjectStep( args: KtxSetupProjectArgs, io: KtxCliIo, @@ -307,7 +273,6 @@ export async function runKtxSetupProjectStep( projectDir: selected.projectDir, project, confirmedCreation: selected.confirmedCreation, - ...(selected.createdProjectCleanup ? { createdProjectCleanup: selected.createdProjectCleanup } : {}), }; } @@ -322,13 +287,12 @@ export async function runKtxSetupProjectStep( io.stderr.write('Missing setup choice: pass --yes to create a project in non-interactive setup.\n'); return { status: 'missing-input', projectDir }; } - const { project, createdProjectCleanup } = await createProjectWithCleanup(projectDir, deps); + const project = await createProject(projectDir, deps); printProjectSummary(io, projectDir); return { status: 'ready', projectDir, project, - ...(createdProjectCleanup ? { createdProjectCleanup } : {}), }; } @@ -368,13 +332,12 @@ export async function runKtxSetupProjectStep( } if (choice === 'current') { - const { project, createdProjectCleanup } = await createProjectWithCleanup(projectDir, deps); + const project = await createProject(projectDir, deps); printProjectSummary(io, projectDir); return { status: 'ready', projectDir, project, - ...(createdProjectCleanup ? { createdProjectCleanup } : {}), }; } @@ -390,7 +353,6 @@ export async function runKtxSetupProjectStep( projectDir: defaultProjectDir, project, confirmedCreation: confirmed.confirmedCreation, - ...(confirmed.createdProjectCleanup ? { createdProjectCleanup: confirmed.createdProjectCleanup } : {}), }; } @@ -419,7 +381,6 @@ export async function runKtxSetupProjectStep( projectDir: customDir, project, confirmedCreation: confirmed.confirmedCreation, - ...(confirmed.createdProjectCleanup ? { createdProjectCleanup: confirmed.createdProjectCleanup } : {}), }; } diff --git a/packages/cli/src/setup-prompts.ts b/packages/cli/src/setup-prompts.ts index 1609bd76..e508d8ff 100644 --- a/packages/cli/src/setup-prompts.ts +++ b/packages/cli/src/setup-prompts.ts @@ -9,12 +9,12 @@ import { log, multiselect, note, - password, select, text, } from '@clack/prompts'; import type { KtxCliIo } from './cli-runtime.js'; import { withMenuOptionsSpacing, withTextInputNavigation } from './prompt-navigation.js'; +import { revealPassword } from './reveal-password-prompt.js'; import { withSetupInterruptConfirmation } from './setup-interrupt.js'; export interface KtxSetupPromptOption { @@ -189,7 +189,7 @@ export function createKtxSetupPromptAdapter(options: KtxSetupPromptAdapterOption }, async password(promptOptions) { const value = await withSetupInterruptConfirmation(() => - password({ ...promptOptions, message: withTextInputNavigation(promptOptions.message) }), + revealPassword({ ...promptOptions, message: withTextInputNavigation(promptOptions.message) }), ); return isCancel(value) ? undefined : String(value); }, diff --git a/packages/cli/src/setup-ready-menu.test.ts b/packages/cli/src/setup-ready-menu.test.ts deleted file mode 100644 index 028b94ee..00000000 --- a/packages/cli/src/setup-ready-menu.test.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { isKtxPreAgentSetupReady, isKtxSetupReady, runKtxSetupReadyChangeMenu } from './setup-ready-menu.js'; -import type { KtxSetupStatus } from './setup.js'; - -const readyStatus: KtxSetupStatus = { - project: { path: '/tmp/revenue', ready: true }, - llm: { backend: 'anthropic', ready: true, model: 'claude-sonnet-4-6' }, - embeddings: { backend: 'openai', ready: true, model: 'text-embedding-3-small', dimensions: 1536 }, - databases: [{ connectionId: 'warehouse', ready: true }], - sources: [], - runtime: { required: false, ready: true, features: [] }, - context: { ready: true, status: 'completed' }, - agents: [{ target: 'codex', scope: 'project', ready: true }], -}; - -describe('setup ready menu', () => { - it('recognizes a ready setup only when required sections are ready', () => { - expect(isKtxSetupReady(readyStatus)).toBe(true); - expect(isKtxSetupReady({ ...readyStatus, embeddings: { ready: false } })).toBe(false); - expect(isKtxSetupReady({ ...readyStatus, runtime: { required: true, ready: false, features: ['core'] } })).toBe(false); - expect(isKtxSetupReady({ ...readyStatus, context: { ready: false, status: 'not_started' } })).toBe(false); - expect(isKtxSetupReady({ ...readyStatus, agents: [] })).toBe(false); - }); - - it('recognizes pre-agent readiness without requiring agents', () => { - expect(isKtxPreAgentSetupReady(readyStatus)).toBe(true); - expect(isKtxPreAgentSetupReady({ ...readyStatus, agents: [] })).toBe(true); - expect(isKtxPreAgentSetupReady({ ...readyStatus, embeddings: { ready: false } })).toBe(false); - expect(isKtxPreAgentSetupReady({ ...readyStatus, runtime: { required: true, ready: false, features: ['core'] } })).toBe( - false, - ); - expect(isKtxPreAgentSetupReady({ ...readyStatus, context: { ready: false, status: 'not_started' } })).toBe(false); - }); - - it('maps ready-project menu choices to setup sections', async () => { - const prompts = { select: vi.fn(async () => 'agents'), cancel: vi.fn() }; - - await expect(runKtxSetupReadyChangeMenu(readyStatus, { prompts })).resolves.toEqual({ action: 'agents' }); - - expect(prompts.select).toHaveBeenCalledWith({ - message: 'KTX is already set up for /tmp/revenue. What would you like to change?', - options: [ - { value: 'models', label: 'Models' }, - { value: 'embeddings', label: 'Embeddings' }, - { value: 'databases', label: 'Databases' }, - { value: 'sources', label: 'Context sources' }, - { value: 'context', label: 'Rebuild KTX context' }, - { value: 'agents', label: 'Agent integration' }, - { value: 'exit', label: 'Exit' }, - ], - }); - }); -}); diff --git a/packages/cli/src/setup-ready-menu.ts b/packages/cli/src/setup-ready-menu.ts index f1f736e4..de0f5a45 100644 --- a/packages/cli/src/setup-ready-menu.ts +++ b/packages/cli/src/setup-ready-menu.ts @@ -14,6 +14,12 @@ export type KtxSetupReadyAction = | 'agents' | 'exit'; +/** + * Where a project stands once its `ktx.yaml` exists. Single source of truth for the + * end-of-setup interception: each state maps to exactly one obvious next action. + */ +export type KtxSetupCompletion = 'incomplete' | 'needs-context' | 'needs-agents' | 'ready'; + interface KtxSetupReadyMenuPromptAdapter { select(options: { message: string; options: KtxSetupPromptOption[] }): Promise; cancel(message: string): void; @@ -23,7 +29,11 @@ export interface KtxSetupReadyMenuDeps { prompts?: KtxSetupReadyMenuPromptAdapter; } -export function isKtxPreAgentSetupReady(status: KtxSetupStatus): boolean { +export function setupHasContextTargets(status: KtxSetupStatus): boolean { + return status.databases.length > 0 || status.sources.length > 0; +} + +function setupConfigReady(status: KtxSetupStatus): boolean { return ( status.project.ready && status.llm.ready && @@ -31,25 +41,58 @@ export function isKtxPreAgentSetupReady(status: KtxSetupStatus): boolean { status.databases.every((database) => database.ready) && status.sources.every((source) => source.ready) && status.runtime.ready && - status.context.ready + setupHasContextTargets(status) ); } -export function isKtxSetupReady(status: KtxSetupStatus): boolean { - return isKtxPreAgentSetupReady(status) && status.agents.some((agent) => agent.ready); +export function classifyKtxSetupCompletion(status: KtxSetupStatus): KtxSetupCompletion { + if (!setupConfigReady(status)) { + return 'incomplete'; + } + if (!status.context.ready) { + return 'needs-context'; + } + if (!status.agents.some((agent) => agent.ready)) { + return 'needs-agents'; + } + return 'ready'; } function createPromptAdapter(): KtxSetupReadyMenuPromptAdapter { return createKtxSetupPromptAdapter({ selectCancelValue: 'exit' }); } +/** + * Shown when a returning user re-runs `ktx setup` on a fully-ready project. Leads with + * "you're done" (the readiness note is printed by the caller first) and keeps the + * section editor one explicit step away rather than defaulting into it. + */ +export async function runKtxSetupReadyMenu( + status: KtxSetupStatus, + deps: KtxSetupReadyMenuDeps = {}, +): Promise<{ action: KtxSetupReadyAction }> { + const prompts = deps.prompts ?? createPromptAdapter(); + const choice = await prompts.select({ + message: 'Anything else?', + options: [ + { value: 'done', label: "Done — I'll start using ktx" }, + { value: 'change', label: 'Change a setting' }, + ], + }); + if (choice !== 'change') { + return { action: 'exit' }; + } + return runKtxSetupReadyChangeMenu(status, { prompts }); +} + +/** @internal Reached only through {@link runKtxSetupReadyMenu}; exported for unit tests. */ export async function runKtxSetupReadyChangeMenu( status: KtxSetupStatus, deps: KtxSetupReadyMenuDeps = {}, ): Promise<{ action: KtxSetupReadyAction }> { const prompts = deps.prompts ?? createPromptAdapter(); const action = (await prompts.select({ - message: `KTX is already set up for ${status.project.name ?? status.project.path}. What would you like to change?`, + message: 'What would you like to change?', options: [ { value: 'models', label: 'Models' }, { value: 'embeddings', label: 'Embeddings' }, diff --git a/packages/cli/src/setup-runtime.ts b/packages/cli/src/setup-runtime.ts index 25612065..19f09a53 100644 --- a/packages/cli/src/setup-runtime.ts +++ b/packages/cli/src/setup-runtime.ts @@ -1,6 +1,7 @@ import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; import { markKtxSetupStateStepComplete } from './context/project/setup-config.js'; import type { KtxCliIo } from './cli-runtime.js'; +import { errorMessage, writePrefixedLines } from './clack.js'; import { ensureManagedLocalEmbeddingsDaemon, type ManagedLocalEmbeddingsDaemon, @@ -88,7 +89,7 @@ export async function runKtxSetupRuntimeStep( }); } } catch (error) { - io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + writePrefixedLines((chunk) => io.stderr.write(chunk), errorMessage(error)); return { status: 'failed', projectDir: args.projectDir, requirements }; } diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index 410de812..25552fbf 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -17,8 +17,15 @@ import { type KtxProjectConfig, type KtxProjectConnectionConfig, serializeKtxPro import { loadKtxProject } from './context/project/project.js'; import { markKtxSetupStateStepComplete } from './context/project/setup-config.js'; import type { KtxCliIo } from './cli-runtime.js'; +import { errorMessage, writePrefixedLines } from './clack.js'; import { pickNotionRootPages } from './notion-page-picker.js'; import { runKtxSourceMapping } from './source-mapping.js'; +import { + runConnectionSetupWithRecovery, + type ConfigureResult, + type RecoveryOutcome, + type ValidateResult, +} from './connection-recovery.js'; import { withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js'; import { runKtxPublicIngest } from './public-ingest.js'; import { writeProjectLocalSecretReference } from './setup-secrets.js'; @@ -112,11 +119,11 @@ export interface KtxSetupSourcesDeps { const SOURCE_OPTIONS: Array<{ value: KtxSetupSourceType; label: string }> = [ { value: 'dbt', label: 'dbt' }, - { value: 'metricflow', label: 'MetricFlow' }, { value: 'metabase', label: 'Metabase' }, + { value: 'notion', label: 'Notion' }, + { value: 'metricflow', label: 'MetricFlow' }, { value: 'looker', label: 'Looker' }, { value: 'lookml', label: 'LookML' }, - { value: 'notion', label: 'Notion' }, ]; const SOURCE_LABELS = Object.fromEntries(SOURCE_OPTIONS.map((option) => [option.value, option.label])) as Record< @@ -216,6 +223,39 @@ function credentialRef(value: string | undefined, label: string): string { return ref; } +type SourceCredentialFlag = { + field: 'sourceAuthTokenRef' | 'sourceApiKeyRef' | 'sourceClientSecretRef'; + flag: string; +}; + +// Each connector reads exactly one credential ref; the flag name mirrors the +// ktx.yaml field it writes (auth_token_ref / api_key_ref / client_secret_ref). +const SOURCE_CREDENTIAL_FLAG: Record = { + dbt: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' }, + metricflow: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' }, + lookml: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' }, + notion: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' }, + metabase: { field: 'sourceApiKeyRef', flag: '--source-api-key-ref' }, + looker: { field: 'sourceClientSecretRef', flag: '--source-client-secret-ref' }, +}; + +const ALL_SOURCE_CREDENTIAL_FLAGS: SourceCredentialFlag[] = [ + { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' }, + { field: 'sourceApiKeyRef', flag: '--source-api-key-ref' }, + { field: 'sourceClientSecretRef', flag: '--source-client-secret-ref' }, +]; + +// Reject a credential ref flag the chosen source does not read, so a wrong flag +// fails loudly instead of being silently dropped (KLO-724). +function assertSourceCredentialFlags(source: KtxSetupSourceType, args: KtxSetupSourcesArgs): void { + const allowed = SOURCE_CREDENTIAL_FLAG[source]; + for (const { field, flag } of ALL_SOURCE_CREDENTIAL_FLAGS) { + if (args[field] && field !== allowed.field) { + throw new Error(`${flag} does not apply to --source ${source}; use ${allowed.flag}.`); + } + } +} + async function chooseSourceCredentialRef(input: { prompts: KtxSetupSourcesPromptAdapter; projectDir: string; @@ -229,8 +269,8 @@ async function chooseSourceCredentialRef(input: { message: `How should KTX find your ${input.label}?`, options: [ ...(input.existingRef ? [{ value: 'keep', label: 'Keep existing credential' }] : []), - { value: 'env', label: `Use ${input.envName} from the environment` }, { value: 'paste', label: 'Paste a key and save it as a local secret file' }, + { value: 'env', label: `Use ${input.envName} from the environment` }, { value: 'back', label: 'Back' }, ], }); @@ -267,8 +307,8 @@ async function chooseGitAuthCredentialRef(input: { message: `${label} repo requires authentication.`, options: [ ...(input.existingRef ? [{ value: 'keep', label: 'Keep existing credential' }] : []), - { value: 'env', label: 'Use GITHUB_TOKEN from the environment' }, { value: 'paste', label: 'Paste a token and save it as a local secret file' }, + { value: 'env', label: 'Use GITHUB_TOKEN from the environment' }, { value: 'skip', label: 'Skip — try without authentication' }, { value: 'back', label: 'Back' }, ], @@ -514,7 +554,7 @@ function buildNotionConnection(args: KtxSetupSourcesArgs): KtxProjectConnectionC } return { driver: 'notion', - auth_token_ref: credentialRef(args.sourceApiKeyRef, 'Notion token ref'), + auth_token_ref: credentialRef(args.sourceAuthTokenRef, 'Notion token ref'), crawl_mode: crawlMode, ...(rootPageIds.length > 0 ? { root_page_ids: rootPageIds } : {}), root_database_ids: [], @@ -832,8 +872,7 @@ type InteractiveSourceConnectionChoice = type SourceSetupChoiceResult = | { status: 'ready'; connectionId: string } - | { status: 'back' } - | { status: 'failed' }; + | { status: Exclude }; async function runSourcePromptSteps( initialState: SourcePromptState, @@ -1024,8 +1063,8 @@ async function promptForInteractiveSource( const selectedLocation = await prompts.select({ message: `${source} source location`, options: [ - { value: 'path', label: 'Local path' }, { value: 'git', label: 'Git URL' }, + { value: 'path', label: 'Local path' }, { value: 'back', label: 'Back' }, ], }); @@ -1294,18 +1333,18 @@ async function promptForInteractiveSource( label: 'Notion integration token', envName: 'NOTION_TOKEN', secretFileName: `${currentState.sourceConnectionId ?? 'notion-main'}-token`, - existingRef: currentState.sourceApiKeyRef, + existingRef: currentState.sourceAuthTokenRef, }); if (ref === 'back') return 'back'; - currentState.sourceApiKeyRef = ref; + currentState.sourceAuthTokenRef = ref; return 'next'; }, async (currentState) => { const crawlMode = await prompts.select({ message: 'Which Notion pages should KTX ingest?', options: [ - { value: 'selected_roots', label: 'Specific pages and their subpages (choose them in a picker)' }, { value: 'all_accessible', label: 'All pages the integration can access' }, + { value: 'selected_roots', label: 'Specific pages and their subpages (choose them in a picker)' }, { value: 'back', label: 'Back' }, ], }); @@ -1325,7 +1364,7 @@ async function promptForInteractiveSource( connectionId, connection: { driver: 'notion', - auth_token_ref: credentialRef(currentState.sourceApiKeyRef, 'Notion token ref'), + auth_token_ref: credentialRef(currentState.sourceAuthTokenRef, 'Notion token ref'), crawl_mode: 'selected_roots', root_page_ids: currentState.notionRootPageIds ?? [], root_database_ids: [], @@ -1515,7 +1554,7 @@ function sourceArgsFromExistingConnection(input: { return sourceArgs; } - sourceArgs.sourceApiKeyRef = stringField(input.connection.auth_token_ref); + sourceArgs.sourceAuthTokenRef = stringField(input.connection.auth_token_ref); sourceArgs.notionCrawlMode = input.connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots'; if (Array.isArray(input.connection.root_page_ids)) { @@ -1724,6 +1763,58 @@ async function validateSource( return await (deps.validateNotion ?? defaultValidateNotion)(args.connection); } +async function createSourceSetupRollback(projectDir: string): Promise<() => Promise> { + const project = await loadKtxProject({ projectDir }); + const previousConfig = project.config; + const configPath = project.configPath; + return async () => { + await writeFile(configPath, serializeKtxProjectConfig(previousConfig), 'utf-8'); + }; +} + +function sourceConnectionId(input: { + source: KtxSetupSourceType; + sourceChoice: Exclude; +}): string { + return input.sourceChoice.kind === 'existing' || input.sourceChoice.kind === 'edited' + ? input.sourceChoice.connectionId + : (input.sourceChoice.args.sourceConnectionId ?? `${input.source}-main`); +} + +async function validateSourceConnectionAndMapping(input: { + args: KtxSetupSourcesArgs; + source: KtxSetupSourceType; + connectionId: string; + connection: KtxProjectConnectionConfig; + prompts: KtxSetupSourcesPromptAdapter; + io: KtxCliIo; + deps: KtxSetupSourcesDeps; +}): Promise { + const validation = await validateSource( + input.source, + { projectDir: input.args.projectDir, connectionId: input.connectionId, connection: input.connection }, + input.deps, + ); + if (!validation.ok) { + input.io.stderr.write(`${validation.message}\n`); + return { status: 'failed' }; + } + + if (input.source === 'metabase' || input.source === 'looker') { + input.prompts.log?.(`Validating ${sourceLabel(input.source)} mapping...`); + const mappingCode = await (input.deps.runMapping ?? defaultRunMapping)( + input.args.projectDir, + input.connectionId, + createSetupPrefixedIo(input.io), + ); + if (mappingCode !== 0) { + return { status: 'failed' }; + } + } + + return { status: 'ok' }; +} + async function saveValidateAndMaybeBuildSource(input: { args: KtxSetupSourcesArgs; source: KtxSetupSourceType; @@ -1732,76 +1823,121 @@ async function saveValidateAndMaybeBuildSource(input: { io: KtxCliIo; deps: KtxSetupSourcesDeps; }): Promise { - const connectionId = - input.sourceChoice.kind === 'existing' - ? input.sourceChoice.connectionId - : input.sourceChoice.kind === 'edited' - ? input.sourceChoice.connectionId - : (input.sourceChoice.args.sourceConnectionId ?? `${input.source}-main`); - const connection = - input.sourceChoice.kind === 'existing' - ? input.sourceChoice.connection - : buildConnection(input.source, input.sourceChoice.args); - const rollback = - input.sourceChoice.kind === 'existing' - ? undefined - : await writeSourceConnection( - input.args.projectDir, - connectionId, - connection, - sourceAdapter(input.source), - input.io, - ); + let latestChoice = input.sourceChoice; + let latestConnectionId = sourceConnectionId({ source: input.source, sourceChoice: latestChoice }); + let latestConnection = + latestChoice.kind === 'existing' + ? latestChoice.connection + : buildConnection(input.source, latestChoice.args); + let configureCount = 0; + let rollbackAfterConfigure: (() => Promise) | undefined; - if (input.sourceChoice.kind === 'existing') { - await ensureSourceAdapterEnabled(input.args.projectDir, input.source); - } + const outcome = await runConnectionSetupWithRecovery({ + label: latestConnectionId, + interactive: input.args.inputMode !== 'disabled', + allowSkip: true, + io: input.io, + prompts: input.prompts, + snapshot: async () => { + rollbackAfterConfigure = await createSourceSetupRollback(input.args.projectDir); + return rollbackAfterConfigure; + }, + configure: async (): Promise => { + configureCount += 1; + if (latestChoice.kind === 'existing' && configureCount === 1) { + await ensureSourceAdapterEnabled(input.args.projectDir, input.source); + return 'configured'; + } - const validation = await validateSource( - input.source, - { projectDir: input.args.projectDir, connectionId, connection }, - input.deps, - ); - if (!validation.ok) { - await rollback?.(); - input.io.stderr.write(`${validation.message}\n`); - return { status: 'failed' }; - } + const project = await loadKtxProject({ projectDir: input.args.projectDir }); + const currentConnection = project.config.connections[latestConnectionId] ?? latestConnection; + const useAlreadyPromptedArgs = configureCount === 1 && latestChoice.kind !== 'existing'; + const sourceArgs = + useAlreadyPromptedArgs && latestChoice.kind !== 'existing' + ? latestChoice.args + : input.args.inputMode === 'disabled' + ? sourceArgsFromExistingConnection({ + args: input.args, + source: input.source, + connectionId: latestConnectionId, + connection: currentConnection, + }) + : await promptForInteractiveSource( + sourceArgsFromExistingConnection({ + args: input.args, + source: input.source, + connectionId: latestConnectionId, + connection: currentConnection, + }), + input.source, + input.prompts, + input.io, + { + pickNotionRootPages: input.deps.pickNotionRootPages, + discoverMetabaseDatabases: input.deps.discoverMetabaseDatabases, + }, + latestConnectionId, + input.deps.testGitRepo, + input.deps.discoverMetabaseDatabases, + ); - if (input.source === 'metabase' || input.source === 'looker') { - input.prompts.log?.(`Validating ${sourceLabel(input.source)} mapping…`); - const mappingCode = await (input.deps.runMapping ?? defaultRunMapping)( - input.args.projectDir, - connectionId, - createSetupPrefixedIo(input.io), - ); - if (mappingCode !== 0) { - await rollback?.(); - return { status: 'failed' }; - } + if (sourceArgs === 'back') { + return 'back'; + } + + latestConnectionId = sourceArgs.sourceConnectionId ?? latestConnectionId; + latestConnection = buildConnection(input.source, sourceArgs); + latestChoice = + latestChoice.kind === 'new' + ? { kind: 'new', args: sourceArgs } + : { kind: 'edited', connectionId: latestConnectionId, args: sourceArgs }; + + await writeSourceConnection( + input.args.projectDir, + latestConnectionId, + latestConnection, + sourceAdapter(input.source), + input.io, + ); + return 'configured'; + }, + validate: () => + validateSourceConnectionAndMapping({ + args: input.args, + source: input.source, + connectionId: latestConnectionId, + connection: latestConnection, + prompts: input.prompts, + io: input.io, + deps: input.deps, + }), + }); + + if (outcome !== 'ready') { + return { status: outcome }; } if (input.args.runInitialSourceIngest) { const ingestResult = await runInitialSourceIngestWithRecovery({ args: input.args, - connectionId, + connectionId: latestConnectionId, io: input.io, prompts: input.prompts, deps: input.deps, }); if (ingestResult === 'failed') { - await rollback?.(); + await rollbackAfterConfigure?.(); return { status: 'failed' }; } if (ingestResult === 'back') { - await rollback?.(); + await rollbackAfterConfigure?.(); return { status: 'back' }; } } else { - input.io.stdout.write(`│ Context source ${connectionId} saved. It will be built during the context build step.\n`); + input.io.stdout.write(`│ Context source ${latestConnectionId} saved. It will be built during the context build step.\n`); } - return { status: 'ready', connectionId }; + return { status: 'ready', connectionId: latestConnectionId }; } export async function runKtxSetupSourcesStep( @@ -1816,6 +1952,10 @@ export async function runKtxSetupSourcesStep( return { status: 'skipped', projectDir: args.projectDir }; } + if (args.source) { + assertSourceCredentialFlags(args.source, args); + } + const prompts = deps.prompts ?? createPromptAdapter(); const project = await loadKtxProject({ projectDir: args.projectDir }); if (!hasPrimarySource(project.config)) { @@ -1904,8 +2044,13 @@ export async function runKtxSetupSourcesStep( returnToSourceSelection = true; break; } - if (!readyConnectionIds.includes(choiceResult.connectionId)) { - readyConnectionIds.push(choiceResult.connectionId); + if (choiceResult.status === 'skip') { + continue; + } + if (choiceResult.status === 'ready') { + if (!readyConnectionIds.includes(choiceResult.connectionId)) { + readyConnectionIds.push(choiceResult.connectionId); + } } } @@ -1919,7 +2064,7 @@ export async function runKtxSetupSourcesStep( const addMore = await prompts.select({ message: `${readyConnectionIds.length} context source${readyConnectionIds.length > 1 ? 's' : ''} configured (${readyConnectionIds.join(', ')}). Add another?`, options: [ - { value: 'done', label: 'Done — continue to context build' }, + { value: 'done', label: 'Done adding context sources' }, { value: 'edit', label: 'Edit an existing context source' }, { value: 'add', label: 'Add another context source' }, ], @@ -1967,8 +2112,13 @@ export async function runKtxSetupSourcesStep( if (choiceResult.status === 'back') { continue; } - if (!readyConnectionIds.includes(choiceResult.connectionId)) { - readyConnectionIds.push(choiceResult.connectionId); + if (choiceResult.status === 'skip') { + continue; + } + if (choiceResult.status === 'ready') { + if (!readyConnectionIds.includes(choiceResult.connectionId)) { + readyConnectionIds.push(choiceResult.connectionId); + } } continue; } @@ -1983,7 +2133,7 @@ export async function runKtxSetupSourcesStep( return { status: 'ready', projectDir: args.projectDir, connectionIds: readyConnectionIds }; } } catch (error) { - io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + writePrefixedLines((chunk) => io.stderr.write(chunk), errorMessage(error)); return { status: 'failed', projectDir: args.projectDir }; } } diff --git a/packages/cli/src/setup.ts b/packages/cli/src/setup.ts index 825170c0..fc45abb3 100644 --- a/packages/cli/src/setup.ts +++ b/packages/cli/src/setup.ts @@ -1,13 +1,12 @@ import { existsSync } from 'node:fs'; -import { rm } from 'node:fs/promises'; import { basename, join, resolve } from 'node:path'; import { getLatestLocalIngestStatus } from './context/ingest/local-ingest.js'; -import { savedMemoryCountsForReport } from './context/ingest/reports.js'; +import { ingestReportOutcome, savedMemoryCountsForReport } from './context/ingest/reports.js'; import { ktxLocalStateDbPath } from './context/project/local-state-db.js'; import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; import { readKtxSetupState } from './context/project/setup-config.js'; import { getKtxCliPackageInfo, type KtxCliIo } from './cli-runtime.js'; -import { formatSetupNextStepLines } from './next-steps.js'; +import { formatNextStepLines, formatSetupNextStepLines } from './next-steps.js'; import { runtimeInstallPolicyFromFlags } from './managed-python-command.js'; import { readManagedPythonRuntimeStatus } from './managed-python-runtime.js'; import { resolveProjectRuntimeRequirements } from './runtime-requirements.js'; @@ -32,16 +31,12 @@ import { isKtxSetupLlmConfigReady, runKtxSetupAnthropicModelStep, } from './setup-models.js'; +import { type KtxSetupProjectDeps, runKtxSetupProjectStep } from './setup-project.js'; import { - type KtxSetupCreatedProjectCleanup, - type KtxSetupProjectDeps, - runKtxSetupProjectStep, -} from './setup-project.js'; -import { - isKtxPreAgentSetupReady, - isKtxSetupReady, + classifyKtxSetupCompletion, type KtxSetupReadyMenuDeps, - runKtxSetupReadyChangeMenu, + runKtxSetupReadyMenu, + setupHasContextTargets, } from './setup-ready-menu.js'; import { type KtxSetupSourcesDeps, type KtxSetupSourceType, runKtxSetupSourcesStep } from './setup-sources.js'; import { @@ -85,6 +80,7 @@ export type KtxSetupArgs = agentScope?: KtxAgentScope; skipAgents?: boolean; inputMode: 'auto' | 'disabled'; + debug?: boolean; yes: boolean; cliVersion: string; llmBackend?: KtxSetupLlmBackend; @@ -170,6 +166,7 @@ export interface KtxSetupDeps { } const SOURCE_DRIVERS = new Set(['dbt', 'metricflow', 'metabase', 'looker', 'lookml', 'notion']); +const KTX_DOCS_URL = 'https://docs.kaelio.com/ktx'; type KtxSetupEntryAction = 'setup' | 'new-project' | 'agents' | 'status' | 'demo' | 'exit'; type KtxSetupFlowStep = 'models' | 'embeddings' | 'databases' | 'sources' | 'runtime' | 'context' | 'agents'; @@ -221,6 +218,7 @@ async function recordSetupStep(input: { startedAt: number; io: KtxCliIo; cliVersion?: string; + errorDetail?: string; }): Promise { const { emitTelemetryEvent } = await import('./telemetry/index.js'); await emitTelemetryEvent({ @@ -232,6 +230,7 @@ async function recordSetupStep(input: { step: input.step, outcome: setupTelemetryOutcome(input.status), durationMs: Math.max(0, performance.now() - input.startedAt), + ...(input.errorDetail ? { errorDetail: input.errorDetail } : {}), }, }); } @@ -310,7 +309,7 @@ function sourceConnections(config: Awaited>['c type LocalIngestStatusReport = NonNullable>>; function reportHasSavedContext(report: LocalIngestStatusReport): boolean { - if (report.body.failedWorkUnits.length > 0) { + if (ingestReportOutcome(report) === 'error') { return false; } const counts = savedMemoryCountsForReport(report); @@ -531,10 +530,6 @@ function setupStatusReady(status: KtxSetupStatus): boolean { ); } -function setupHasContextTargets(status: KtxSetupStatus): boolean { - return status.databases.length > 0 || status.sources.length > 0; -} - function setupContextReady(status: KtxSetupStatus): boolean { return status.context.ready; } @@ -555,23 +550,6 @@ async function commitSetupConfigChanges(projectDir: string): Promise { await project.git.commitFile('ktx.yaml', 'setup: update KTX project config', 'ktx setup', 'setup@ktx.local'); } -const KTX_SETUP_SCAFFOLD_PATHS = ['ktx.yaml', '.ktx', 'wiki', 'semantic-layer', 'raw-sources', '.git']; - -async function cleanupCreatedProjectScaffold(cleanup: KtxSetupCreatedProjectCleanup | undefined): Promise { - if (!cleanup) { - return; - } - if (cleanup.kind === 'remove-project-dir') { - await rm(cleanup.projectDir, { recursive: true, force: true }); - return; - } - await Promise.all( - KTX_SETUP_SCAFFOLD_PATHS.map((relativePath) => - rm(join(cleanup.projectDir, relativePath), { recursive: true, force: true }), - ), - ); -} - export async function runKtxSetup(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetupDeps = {}): Promise { try { return await runKtxSetupInner(args, io, deps); @@ -586,6 +564,7 @@ export async function runKtxSetup(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSet async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetupDeps = {}): Promise { const setupUi = deps.setupUi ?? createKtxSetupUiAdapter(); setupUi.intro('KTX setup', io); + setupUi.note(KTX_DOCS_URL, '📚 Docs', io); let entryAction: KtxSetupEntryAction | undefined; let projectResult: Awaited>; let agentNextActions: string | undefined; @@ -648,12 +627,19 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup let readyAction: string | undefined; if (args.inputMode !== 'disabled' && !agentsRequested) { - if (isKtxSetupReady(currentStatus)) { - readyAction = (await runKtxSetupReadyChangeMenu(currentStatus, deps.readyMenuDeps)).action; - if (readyAction === 'exit') return 0; - } else if (isKtxPreAgentSetupReady(currentStatus)) { + const completion = classifyKtxSetupCompletion(currentStatus); + if (completion === 'ready') { + setupUi.note(formatNextStepLines().join('\n'), 'ktx is ready', io); + const choice = (await runKtxSetupReadyMenu(currentStatus, deps.readyMenuDeps)).action; + if (choice === 'exit') return 0; + readyAction = choice; + } else if (completion === 'needs-context') { + // Config is done; skip the re-walk and land straight on the build prompt. + readyAction = 'context'; + } else if (completion === 'needs-agents') { readyAction = 'agents'; } + // 'incomplete' → readyAction stays undefined → run the full setup walk. } const runOnly = readyAction; @@ -667,6 +653,7 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup const shouldRunContext = !agentOnlySetup && (!runOnly || runOnly === 'context'); const shouldRunAgents = agentsRequested || !runOnly || runOnly === 'agents'; const showPromptInstructions = projectResult.confirmedCreation !== true; + let skipSourcesFromDatabaseMenu = false; const setupSteps: KtxSetupFlowStep[] = agentOnlySetup ? [] @@ -680,7 +667,9 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup if (step === 'models') return !args.skipLlm && shouldRunModels; if (step === 'embeddings') return !args.skipEmbeddings && shouldRunEmbeddings; if (step === 'databases') return !args.skipDatabases && shouldRunDatabases; - if (step === 'sources') return args.skipSources !== true && shouldRunSources; + if (step === 'sources') { + return args.skipSources !== true && !skipSourcesFromDatabaseMenu && shouldRunSources; + } if (step === 'runtime') return shouldRunRuntime; if (step === 'context') return shouldRunContext; return shouldRunAgents && args.skipAgents !== true; @@ -700,7 +689,7 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup if (!step) break; const stepStartedAt = performance.now(); - let stepResult: { status: KtxSetupFlowStatus }; + let stepResult: { status: KtxSetupFlowStatus; errorDetail?: string }; if (step === 'models') { const modelRunner = deps.model ?? ((modelArgs, modelIo) => runKtxSetupAnthropicModelStep(modelArgs, modelIo, deps.modelDeps)); @@ -743,10 +732,14 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup const databasesRunner = deps.databases ?? ((databaseArgs, databaseIo) => runKtxSetupDatabasesStep(databaseArgs, databaseIo, deps.databasesDeps)); - stepResult = await databasesRunner( + const databaseResult = await databasesRunner( { projectDir: projectResult.projectDir, inputMode: args.inputMode, + ...(args.debug !== undefined ? { debug: args.debug } : {}), + yes: args.yes, + cliVersion: args.cliVersion, + runtimeInstallPolicy: setupRuntimeInstallPolicy(args), ...(args.databaseDrivers ? { databaseDrivers: args.databaseDrivers } : {}), ...(args.databaseConnectionIds ? { databaseConnectionIds: args.databaseConnectionIds } : {}), ...(args.databaseConnectionId ? { databaseConnectionId: args.databaseConnectionId } : {}), @@ -768,6 +761,8 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup }, io, ); + skipSourcesFromDatabaseMenu = databaseResult.status === 'ready' && databaseResult.skipSources === true; + stepResult = databaseResult; } else if (step === 'sources') { const sourcesRunner = deps.sources ?? ((sourceArgs, sourceIo) => runKtxSetupSourcesStep(sourceArgs, sourceIo, deps.sourcesDeps)); @@ -794,7 +789,7 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup ...(args.notionCrawlMode ? { notionCrawlMode: args.notionCrawlMode } : {}), ...(args.notionRootPageIds ? { notionRootPageIds: args.notionRootPageIds } : {}), runInitialSourceIngest: args.runInitialSourceIngest ?? false, - skipSources: args.skipSources === true || !shouldRunSources, + skipSources: args.skipSources === true || !shouldRunSources || skipSourcesFromDatabaseMenu, }, io, ); @@ -859,10 +854,10 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup startedAt: stepStartedAt, io, cliVersion: args.cliVersion, + ...(stepResult.errorDetail ? { errorDetail: stepResult.errorDetail } : {}), }); if (stepResult.status === 'failed') { - await cleanupCreatedProjectScaffold(projectResult.createdProjectCleanup); return 1; } if (stepResult.status === 'missing-input') { @@ -885,7 +880,9 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup } if (step === 'context' && stepResult.status !== 'ready') { if (shouldRunAgents && args.skipAgents !== true) { - return 0; + // Context isn't built, so skip agent install — but still reach the + // completion screen, which states readiness and points at `ktx ingest`. + break setupLoop; } } diff --git a/packages/cli/src/skills/analytics/SKILL.md b/packages/cli/src/skills/analytics/SKILL.md index e4aa86d2..e6857e56 100644 --- a/packages/cli/src/skills/analytics/SKILL.md +++ b/packages/cli/src/skills/analytics/SKILL.md @@ -28,7 +28,12 @@ You have access to KTX MCP tools for data discovery, semantic-layer analysis, ra - Read entity details before writing SQL against an unfamiliar table. Do not assume column names. - Treat `sql_execution` as read-only. Writes are rejected by the server. - Validate value mentions with `dictionary_search` instead of guessing case or spelling. Treat a `dictionary_search` miss as non-authoritative. The index is built from profile-sampled values, so a missing value may simply have been outside the sample. Follow up with `sql_execution` against the most plausible columns before concluding the value is absent. -- When `connection_list` shows multiple connections, pass an explicit `connectionId` to every tool that takes one and where user intent pins a specific warehouse. Required: `entity_details`, `sl_read_source`, and `sql_execution`. Required when user intent is warehouse-specific, including wording like "in our warehouse" or "this warehouse": `memory_ingest`; without `connectionId`, the memory agent cannot update the semantic layer and the knowledge lands as wiki-only. Pass `connectionId` when intent pins a warehouse, otherwise omit for unscoped discovery: `sl_query`, `discover_data`, and `dictionary_search`. Never pass `connectionId` to `connection_list`, `wiki_search`, `wiki_read`, or `memory_ingest_status`. If intent is ambiguous for a required-or-scoped tool, ask the user which warehouse before calling. +- `connectionId` scoping when `connection_list` shows multiple connections: + - Always pass it: `entity_details`, `sl_read_source`, `sql_execution`. + - Pass it when intent pins a warehouse, otherwise omit for unscoped discovery: `sl_query`, `discover_data`, `dictionary_search`. + - `memory_ingest`: pass it for warehouse-specific knowledge (e.g. "in our warehouse"); without it the memory lands as wiki-only and cannot update the semantic layer. + - Never pass it: `connection_list`, `wiki_search`, `wiki_read`, `memory_ingest_status`. + - If scoping is required but intent is ambiguous, ask which warehouse before calling. - Show compact result tables for small outputs. For broad results, summarize the top findings and mention the applied limit. - Ask a concise clarification only when the metric, date range, entity, or grain is genuinely ambiguous and cannot be inferred from context. diff --git a/packages/cli/src/skills/historic_sql_patterns/SKILL.md b/packages/cli/src/skills/historic_sql_patterns/SKILL.md index 057a7c78..fc7096a1 100644 --- a/packages/cli/src/skills/historic_sql_patterns/SKILL.md +++ b/packages/cli/src/skills/historic_sql_patterns/SKILL.md @@ -15,8 +15,7 @@ Use this skill when the WorkUnit raw file is a `patterns-input/part-0001.json` s 3. Call `read_raw_file` for that exact raw file path. 4. Identify recurring analytical intents that span at least two tables and have repeated usage signal. 5. Emit one `pattern` evidence object per durable cross-table intent by calling `emit_historic_sql_evidence`. -6. Set each evidence object's `rawPath` to the exact raw file path read in step 3. -7. Stop after all pattern evidence has been emitted. +6. Stop after all pattern evidence has been emitted. Every join column mentioned in pattern descriptions must be verified via entity_details for both sides of the join. @@ -56,7 +55,6 @@ Each call to `emit_historic_sql_evidence` must use this shape: ```json { "kind": "pattern", - "rawPath": "patterns-input/part-0001.json", "pattern": { "slug": "order-lifecycle-analysis", "title": "Order Lifecycle Analysis", diff --git a/packages/cli/src/skills/historic_sql_table_digest/SKILL.md b/packages/cli/src/skills/historic_sql_table_digest/SKILL.md index 99cf6936..1710f21c 100644 --- a/packages/cli/src/skills/historic_sql_table_digest/SKILL.md +++ b/packages/cli/src/skills/historic_sql_table_digest/SKILL.md @@ -53,7 +53,6 @@ Call `emit_historic_sql_evidence` with this shape: { "kind": "table_usage", "table": "public.orders", - "rawPath": "tables/public.orders.json", "usage": { "narrative": "Orders are repeatedly queried for paid/refunded lifecycle analysis and customer-level rollups.", "frequencyTier": "high", diff --git a/packages/cli/src/skills/sl/SKILL.md b/packages/cli/src/skills/sl/SKILL.md index d5f334fe..53e1bd22 100644 --- a/packages/cli/src/skills/sl/SKILL.md +++ b/packages/cli/src/skills/sl/SKILL.md @@ -124,7 +124,7 @@ Every standalone column requires `name` and `type`. Overlays have computed colum ### Grain -`grain: [col_a, col_b]` - the set of columns that uniquely identify one row. The query engine uses grain to prevent fan-out in joins. Overlays inherit grain from the manifest unless they override. +`grain: [col_a, col_b]` - the set of columns that uniquely identify one row. The query engine uses grain to prevent fanout in joins. Overlays inherit grain from the manifest unless they override. ### Joins @@ -177,7 +177,7 @@ The reverse edge (wiki pages that cite this source) is derived automatically fro ## Part 2 - Querying via `sl_query` -The `sl_query` tool generates correct SQL from a structured query. It handles joins, fan-out prevention, aggregation correctness, and filter classification automatically. Prefer it over writing raw SQL whenever the SL has the relevant sources. +The `sl_query` tool generates correct SQL from a structured query. It handles joins, fanout prevention, aggregation correctness, and filter classification automatically. Prefer it over writing raw SQL whenever the SL has the relevant sources. ### When to prefer sl_query over raw SQL diff --git a/packages/cli/src/sl.ts b/packages/cli/src/sl.ts index 76e1092a..dcf5e460 100644 --- a/packages/cli/src/sl.ts +++ b/packages/cli/src/sl.ts @@ -7,7 +7,14 @@ import type { KtxEmbeddingPort } from './context/core/embedding.js'; import type { KtxSemanticLayerComputePort } from './context/daemon/semantic-layer-compute.js'; import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; import { compileLocalSlQuery } from './context/sl/local-query.js'; -import { listLocalSlSources, readLocalSlSource, searchLocalSlSources as defaultSearchLocalSlSources, validateLocalSlSource, type LocalSlSourceSearchResult, type LocalSlSourceSummary } from './context/sl/local-sl.js'; +import { + listLocalSlSources, + resolveLocalSlSource, + searchLocalSlSources as defaultSearchLocalSlSources, + validateLocalSlSource, + type LocalSlSourceSearchResult, + type LocalSlSourceSummary, +} from './context/sl/local-sl.js'; import type { SemanticLayerQueryInput } from './context/sl/types.js'; import { resolveProjectEmbeddingProvider, @@ -19,7 +26,8 @@ import { type KtxManagedPythonInstallPolicy, } from './managed-python-command.js'; import { profileMark } from './startup-profile.js'; -import { emitTelemetryEvent } from './telemetry/index.js'; +import { emitTelemetryEvent, reportException } from './telemetry/index.js'; +import { collectTelemetryRedactionSecrets } from './telemetry/redaction-secrets.js'; import { scrubErrorClass } from './telemetry/scrubber.js'; profileMark('module:sl'); @@ -45,7 +53,8 @@ export type KtxSlArgs = json?: boolean; cliVersion: string; } - | { command: 'validate'; projectDir: string; connectionId: string; sourceName: string } + | { command: 'read'; projectDir: string; connectionId?: string; sourceName: string } + | { command: 'validate'; projectDir: string; connectionId?: string; sourceName: string } | { command: 'query'; projectDir: string; @@ -185,11 +194,18 @@ async function readSlQueryFile(path: string): Promise { return parsed as SemanticLayerQueryInput; } +function ambiguousSourceMessage(sourceName: string, connectionIds: readonly string[]): string { + return `Source '${sourceName}' exists in multiple connections: ${connectionIds.join( + ', ', + )}. Re-run with --connection-id .`; +} + export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: KtxSlDeps = {}): Promise { const startedAt = performance.now(); let queryForTelemetry: SemanticLayerQueryInput | undefined; + let project: KtxLocalProject | undefined; try { - const project = await (deps.loadProject ?? loadKtxProject)({ projectDir: args.projectDir }); + project = await (deps.loadProject ?? loadKtxProject)({ projectDir: args.projectDir }); if (args.command === 'list') { const sources = await listLocalSlSources(project, { connectionId: args.connectionId }); await printSlSources({ @@ -232,25 +248,50 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx }); return 0; } - if (args.command === 'validate') { - const source = await readLocalSlSource(project, { + if (args.command === 'read') { + const resolved = await resolveLocalSlSource(project, { connectionId: args.connectionId, sourceName: args.sourceName, }); - if (!source) { - throw new Error(`Semantic-layer source "${args.connectionId}/${args.sourceName}" was not found`); + if (resolved.kind === 'not-found') { + throw new Error( + args.connectionId !== undefined + ? `No semantic-layer source '${args.sourceName}' for connection '${args.connectionId}'` + : `No semantic-layer source '${args.sourceName}'`, + ); } - const result = await validateLocalSlSource(source.yaml, { - project, + if (resolved.kind === 'ambiguous') { + throw new Error(ambiguousSourceMessage(args.sourceName, resolved.connectionIds)); + } + io.stdout.write(resolved.source.yaml); + return 0; + } + if (args.command === 'validate') { + const resolved = await resolveLocalSlSource(project, { connectionId: args.connectionId, sourceName: args.sourceName, }); + if (resolved.kind === 'not-found') { + throw new Error( + args.connectionId !== undefined + ? `Semantic-layer source "${args.connectionId}/${args.sourceName}" was not found` + : `Semantic-layer source "${args.sourceName}" was not found`, + ); + } + if (resolved.kind === 'ambiguous') { + throw new Error(ambiguousSourceMessage(args.sourceName, resolved.connectionIds)); + } + const result = await validateLocalSlSource(resolved.source.yaml, { + project, + connectionId: resolved.source.connectionId, + sourceName: args.sourceName, + }); await emitTelemetryEvent({ name: 'sl_validate_completed', projectDir: args.projectDir, io, fields: { - sourceCount: source ? 1 : 0, + sourceCount: 1, modelCount: 0, validationErrorCount: result.valid ? 0 : result.errors.length, outcome: result.valid ? 'ok' : 'error', @@ -263,7 +304,7 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx } return 1; } - io.stdout.write(`Valid semantic-layer source: ${args.connectionId}/${args.sourceName}\n`); + io.stdout.write(`Valid semantic-layer source: ${resolved.source.connectionId}/${args.sourceName}\n`); return 0; } if (args.command === 'query') { @@ -281,7 +322,7 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx projectDir: args.projectDir, }); const queryExecutor = args.execute ? (deps.createQueryExecutor ?? createDefaultLocalQueryExecutor)() : undefined; - const result = await compileLocalSlQuery(project as KtxLocalProject, { + const result = await compileLocalSlQuery(project, { connectionId: args.connectionId, query, compute, @@ -312,6 +353,20 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx const _exhaustive: never = args; throw new Error(`Unsupported sl command: ${JSON.stringify(_exhaustive)}`); } catch (error) { + await reportException({ + error, + context: { source: `sl ${args.command}`, handled: true, fatal: false }, + projectDir: args.projectDir, + io, + redactionSecrets: await collectTelemetryRedactionSecrets({ + project, + projectDir: args.projectDir, + connectionId: args.connectionId, + includeLlm: args.command === 'query', + includeEmbeddings: args.command === 'search' || args.command === 'query', + env: process.env, + }), + }); if (args.command === 'validate') { const errorClass = scrubErrorClass(error); await emitTelemetryEvent({ diff --git a/packages/cli/src/sql.ts b/packages/cli/src/sql.ts index 1b15f92e..d3eb6a81 100644 --- a/packages/cli/src/sql.ts +++ b/packages/cli/src/sql.ts @@ -7,7 +7,8 @@ import { createKtxCliScanConnector } from './local-scan-connectors.js'; import { createManagedDaemonSqlAnalysisPort } from './managed-python-http.js'; import { profileMark } from './startup-profile.js'; import { isDemoConnection } from './telemetry/demo-detect.js'; -import { emitTelemetryEvent } from './telemetry/index.js'; +import { emitTelemetryEvent, reportException } from './telemetry/index.js'; +import { collectTelemetryRedactionSecrets } from './telemetry/redaction-secrets.js'; import { scrubErrorClass } from './telemetry/scrubber.js'; profileMark('module:sql'); @@ -43,16 +44,12 @@ function sqlAnalysisDialectForDriver(driver: string | undefined): SqlAnalysisDia const normalized = String(driver ?? '').trim().toLowerCase(); const map: Record = { postgres: 'postgres', - postgresql: 'postgres', bigquery: 'bigquery', snowflake: 'snowflake', mysql: 'mysql', sqlserver: 'tsql', - mssql: 'tsql', sqlite: 'sqlite', - sqlite3: 'sqlite', clickhouse: 'clickhouse', - redshift: 'redshift', }; return map[normalized] ?? 'postgres'; } @@ -146,8 +143,9 @@ export async function runKtxSql(args: KtxSqlArgs, io: KtxCliIo = process, deps: const startedAt = performance.now(); let driver = 'unknown'; let demoConnection = false; + let project: KtxLocalProject | undefined; try { - const project = await (deps.loadProject ?? loadKtxProject)({ projectDir: args.projectDir }); + project = await (deps.loadProject ?? loadKtxProject)({ projectDir: args.projectDir }); const connection = project.config.connections[args.connectionId]; if (!connection) { throw new Error(`Connection "${args.connectionId}" is not configured in ktx.yaml`); @@ -175,7 +173,7 @@ export async function runKtxSql(args: KtxSqlArgs, io: KtxCliIo = process, deps: const createScanConnector = deps.createScanConnector ?? createKtxCliScanConnector; let connector: KtxScanConnector | null = null; try { - connector = await createScanConnector(project as KtxLocalProject, args.connectionId); + connector = await createScanConnector(project, args.connectionId); if (!connector.capabilities.readOnlySql || !connector.executeReadOnly) { throw new Error(`Connection "${args.connectionId}" does not support read-only SQL execution.`); } @@ -222,6 +220,20 @@ export async function runKtxSql(args: KtxSqlArgs, io: KtxCliIo = process, deps: ...(errorClass ? { errorClass } : {}), }, }); + await reportException({ + error, + context: { source: 'sql run', handled: true, fatal: false }, + projectDir: args.projectDir, + io, + redactionSecrets: await collectTelemetryRedactionSecrets({ + project, + projectDir: args.projectDir, + connectionId: args.connectionId, + includeLlm: false, + includeEmbeddings: false, + env: process.env, + }), + }); io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); return 1; } diff --git a/packages/cli/src/status-project.ts b/packages/cli/src/status-project.ts index aaecff27..ff7b98f4 100644 --- a/packages/cli/src/status-project.ts +++ b/packages/cli/src/status-project.ts @@ -1,14 +1,23 @@ import { stat as statAsync, readdir as readdirAsync } from 'node:fs/promises'; import { basename, join } from 'node:path'; import { runClaudeCodeAuthProbe } from './context/llm/claude-code-runtime.js'; +import { + CODEX_ISOLATION_WARNING, + CODEX_ISOLATION_WARNING_FIX, +} from './context/llm/codex-isolation.js'; +import { runCodexAuthProbe } from './context/llm/codex-runtime.js'; import type { KtxConfigIssue, KtxProjectConfig, KtxProjectConnectionConfig, KtxProjectEmbeddingConfig, KtxProjectLlmConfig } from './context/project/config.js'; import type { KtxLocalProject } from './context/project/project.js'; import { ktxLocalStateDbPath } from './context/project/local-state-db.js'; -import type { PostgresPgssProbeResult } from './context/ingest/adapters/historic-sql/types.js'; import { isQueryHistoryEnabled, queryHistoryDialectForConnection, } from './context/ingest/adapters/historic-sql/connection-dialect.js'; +import { + historicSqlProbeCatalogName, + runHistoricSqlReadinessProbe, + type HistoricSqlReadinessProbe, +} from './context/ingest/historic-sql-probes.js'; import { formatClaudeCodePromptCachingFix, formatClaudeCodePromptCachingWarning, @@ -90,11 +99,12 @@ type ClaudeCodeAuthProbe = (input: { env?: NodeJS.ProcessEnv; }) => Promise<{ ok: true } | { ok: false; message: string }>; -const PROJECT_READY_COMMANDS = KTX_NEXT_STEP_DIRECT_COMMANDS.map((step) => step.command); +type CodexAuthProbe = (input: { + projectDir: string; + model: string; +}) => Promise<{ ok: true } | { ok: false; message: string; fix: string }>; -function hasOwnField(value: Record, key: string): boolean { - return Object.prototype.hasOwnProperty.call(value, key); -} +const PROJECT_READY_COMMANDS = KTX_NEXT_STEP_DIRECT_COMMANDS.map((step) => step.command); interface LocalStatsIngestPerConnection { connectionId: string; @@ -174,6 +184,13 @@ function resolveRef(value: unknown, env: NodeJS.ProcessEnv): { resolved: string; return { resolved: trimmed, via: 'literal' }; } +function failureDetail(error: unknown): string { + if (error instanceof Error && error.message.trim().length > 0) { + return error.message.trim().split('\n')[0] ?? error.message.trim(); + } + return String(error); +} + function envHint(value: unknown): string | undefined { if (typeof value === 'string' && value.trim().startsWith('env:')) { return value.trim().slice(4).trim(); @@ -187,6 +204,7 @@ async function buildLlmStatus( projectDir: string; env: NodeJS.ProcessEnv; claudeCodeAuthProbe?: ClaudeCodeAuthProbe; + codexAuthProbe?: CodexAuthProbe; fast?: boolean; useSpinner?: boolean; }, @@ -203,6 +221,18 @@ async function buildLlmStatus( fix: 'Run: ktx setup (choose an LLM provider)', }; } + // The runtime (resolveModelSlots) hard-requires llm.models.default for every + // non-none backend; without it ingest/scan/memory throw. Report that here so + // status never marks a project ready that the runtime would refuse to run. + if (!model || model.trim().length === 0) { + return { + backend, + model, + status: 'fail', + detail: `llm.models.default is required for backend "${backend}"`, + fix: 'Set llm.models.default in ktx.yaml, then rerun `ktx status` (or rerun `ktx setup`).', + }; + } if (backend === 'anthropic') { const ref = config.provider.anthropic?.api_key; const resolved = resolveRef(ref, env); @@ -244,7 +274,7 @@ async function buildLlmStatus( }; } if (backend === 'claude-code') { - const modelName = model ?? 'sonnet'; + const modelName = model; if (options.fast === true) { return { backend, @@ -273,6 +303,36 @@ async function buildLlmStatus( fix: 'Authenticate Claude Code locally with the Claude Code CLI, then rerun `ktx status`.', }; } + if (backend === 'codex') { + const modelName = model; + if (options.fast === true) { + return { + backend, + model: modelName, + status: 'skipped', + detail: 'auth probe skipped (--fast)', + }; + } + const probe = options.codexAuthProbe ?? runCodexAuthProbe; + const auth = await withSpinner(options.useSpinner === true, 'Probing Codex authentication', () => + probe({ projectDir: options.projectDir, model: modelName }), + ); + if (auth.ok) { + return { + backend, + model: modelName, + status: 'ok', + detail: 'local Codex session authenticated', + }; + } + return { + backend, + model: modelName, + status: 'fail', + detail: auth.message, + fix: auth.fix, + }; + } return { backend, model, status: 'warn', detail: 'unknown LLM backend' }; } @@ -332,7 +392,6 @@ function buildConnectionStatus( switch (driver) { case 'postgres': - case 'postgresql': case 'mysql': case 'clickhouse': case 'sqlserver': { @@ -397,232 +456,6 @@ function buildConnectionStatus( } } -interface QueryHistoryProbeInput { - projectDir: string; - connectionId: string; - connection: KtxProjectConnectionConfig; - env: NodeJS.ProcessEnv; -} - -interface GenericProbeResult { - warnings: string[]; - info?: string[]; -} - -type PostgresQueryHistoryProbe = (input: QueryHistoryProbeInput) => Promise; -type SnowflakeQueryHistoryProbe = (input: QueryHistoryProbeInput) => Promise; -type BigQueryQueryHistoryProbe = (input: QueryHistoryProbeInput) => Promise; - -function failureDetail(error: unknown): string { - if (error instanceof Error && error.message.trim().length > 0) { - return error.message.trim().split('\n')[0] ?? error.message.trim(); - } - return String(error); -} - -function postgresReadinessDetail(result: PostgresPgssProbeResult): string { - const warningText = result.warnings.length > 0 ? ` with warnings: ${result.warnings.join('; ')}` : ''; - const info = result.info ?? []; - const infoText = info.length > 0 ? `; info: ${info.join('; ')}` : ''; - return `pg_stat_statements ready (${result.pgServerVersion})${warningText}${infoText}`; -} - -function genericReadinessDetail(label: string, result: GenericProbeResult): string { - const warningText = result.warnings.length > 0 ? ` with warnings: ${result.warnings.join('; ')}` : ''; - const info = result.info ?? []; - const infoText = info.length > 0 ? `; info: ${info.join('; ')}` : ''; - return `${label} ready${warningText}${infoText}`; -} - -function probeFailureFix(error: unknown, dialect: string, connectionId: string, projectDir: string): string { - if (error instanceof Error && error.name === 'HistoricSqlExtensionMissingError' && 'remediation' in error) { - return String(error.remediation); - } - if (error instanceof Error && error.name === 'HistoricSqlGrantsMissingError' && 'remediation' in error) { - return String(error.remediation); - } - if (error instanceof Error && error.name === 'HistoricSqlVersionUnsupportedError') { - return 'Use PostgreSQL 14 or newer, or disable query history for this connection'; - } - return `Fix connections.${connectionId} ${dialect} settings, then rerun \`ktx status --project-dir ${projectDir}\``; -} - -async function defaultPostgresQueryHistoryProbe( - input: QueryHistoryProbeInput, -): Promise { - const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient }, { isKtxPostgresConnectionConfig }] = - await Promise.all([ - import('./context/ingest/adapters/historic-sql/postgres-pgss-reader.js'), - import('./connectors/postgres/historic-sql-query-client.js'), - import('./connectors/postgres/connector.js'), - ]); - - const inputDriver = input.connection.driver ?? 'unknown'; - if (!isKtxPostgresConnectionConfig(input.connection)) { - throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`); - } - - const client = new KtxPostgresHistoricSqlQueryClient({ - connectionId: input.connectionId, - connection: input.connection, - env: input.env, - }); - try { - return await new PostgresPgssReader().probe(client); - } finally { - await client.cleanup(); - } -} - -async function defaultSnowflakeQueryHistoryProbe( - input: QueryHistoryProbeInput, -): Promise { - const [{ SnowflakeHistoricSqlQueryHistoryReader }, { KtxSnowflakeHistoricSqlQueryClient }, { isKtxSnowflakeConnectionConfig }] = - await Promise.all([ - import('./context/ingest/adapters/historic-sql/snowflake-query-history-reader.js'), - import('./connectors/snowflake/historic-sql-query-client.js'), - import('./connectors/snowflake/connector.js'), - ]); - - const inputDriver = input.connection.driver ?? 'unknown'; - if (!isKtxSnowflakeConnectionConfig(input.connection)) { - throw new Error(`Native Snowflake connector cannot run driver "${inputDriver}"`); - } - - const client = new KtxSnowflakeHistoricSqlQueryClient({ - connectionId: input.connectionId, - connection: input.connection, - projectDir: input.projectDir, - env: input.env, - }); - try { - return await new SnowflakeHistoricSqlQueryHistoryReader().probe(client); - } finally { - await client.cleanup(); - } -} - -async function defaultBigQueryQueryHistoryProbe( - input: QueryHistoryProbeInput, -): Promise { - const [ - { BigQueryHistoricSqlQueryHistoryReader }, - { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig }, - { resolveKtxConfigReference }, - ] = await Promise.all([ - import('./context/ingest/adapters/historic-sql/bigquery-query-history-reader.js'), - import('./connectors/bigquery/connector.js'), - import('./context/core/config-reference.js'), - ]); - - const inputDriver = input.connection.driver ?? 'unknown'; - if (!isKtxBigQueryConnectionConfig(input.connection)) { - throw new Error(`Native BigQuery connector cannot run driver "${inputDriver}"`); - } - - const rawCredentials = typeof input.connection.credentials_json === 'string' ? input.connection.credentials_json : ''; - const resolvedCredentials = resolveKtxConfigReference(rawCredentials, input.env); - if (!resolvedCredentials) { - throw new Error(`Query history BigQuery connection ${input.connectionId} requires credentials_json`); - } - const parsed = JSON.parse(resolvedCredentials) as { project_id?: unknown }; - if (typeof parsed.project_id !== 'string' || parsed.project_id.trim().length === 0) { - throw new Error(`Query history BigQuery connection ${input.connectionId} requires credentials_json.project_id`); - } - const region = - typeof input.connection.location === 'string' && input.connection.location.trim().length > 0 - ? input.connection.location.trim() - : 'us'; - - const connector = new KtxBigQueryScanConnector({ - connectionId: input.connectionId, - connection: input.connection, - }); - try { - return await new BigQueryHistoricSqlQueryHistoryReader({ - projectId: parsed.project_id, - region, - }).probe({ - async executeQuery(sql: string) { - const result = await connector.executeReadOnly({ connectionId: input.connectionId, sql }, {} as never); - return { - headers: result.headers, - rows: result.rows, - totalRows: result.totalRows, - }; - }, - }); - } finally { - await connector.cleanup(); - } -} - -interface DispatchedProbe { - label: string; - spinnerLabel: string; - fastSkipDetail: string; - run: () => Promise<{ status: ProjectStatusLevel; detail: string; fix?: string }>; -} - -function postgresProbeDispatch( - input: QueryHistoryProbeInput, - probe: PostgresQueryHistoryProbe, -): DispatchedProbe { - return { - label: 'postgres', - spinnerLabel: `Probing pg_stat_statements on ${input.connectionId}`, - fastSkipDetail: 'pg_stat_statements probe skipped (--fast)', - run: async () => { - const result = await probe(input); - return { - status: result.warnings.length > 0 ? 'warn' : 'ok', - detail: postgresReadinessDetail(result), - ...(result.warnings.length > 0 - ? { - fix: `Update the Postgres parameter group or config, then rerun \`ktx status --project-dir ${input.projectDir}\``, - } - : {}), - }; - }, - }; -} - -function snowflakeProbeDispatch( - input: QueryHistoryProbeInput, - probe: SnowflakeQueryHistoryProbe, -): DispatchedProbe { - return { - label: 'snowflake', - spinnerLabel: `Probing SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY on ${input.connectionId}`, - fastSkipDetail: 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY probe skipped (--fast)', - run: async () => { - const result = await probe(input); - return { - status: result.warnings.length > 0 ? 'warn' : 'ok', - detail: genericReadinessDetail('SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY', result), - }; - }, - }; -} - -function bigqueryProbeDispatch( - input: QueryHistoryProbeInput, - probe: BigQueryQueryHistoryProbe, -): DispatchedProbe { - return { - label: 'bigquery', - spinnerLabel: `Probing INFORMATION_SCHEMA.JOBS_BY_PROJECT on ${input.connectionId}`, - fastSkipDetail: 'INFORMATION_SCHEMA.JOBS_BY_PROJECT probe skipped (--fast)', - run: async () => { - const result = await probe(input); - return { - status: result.warnings.length > 0 ? 'warn' : 'ok', - detail: genericReadinessDetail('INFORMATION_SCHEMA.JOBS_BY_PROJECT', result), - }; - }, - }; -} - async function buildQueryHistoryStatus( project: KtxLocalProject, options: BuildProjectStatusOptions, @@ -631,9 +464,7 @@ async function buildQueryHistoryStatus( .filter(([, connection]) => isQueryHistoryEnabled(connection)) .sort(([left], [right]) => left.localeCompare(right)); - const postgresProbe = options.postgresQueryHistoryProbe ?? defaultPostgresQueryHistoryProbe; - const snowflakeProbe = options.snowflakeQueryHistoryProbe ?? defaultSnowflakeQueryHistoryProbe; - const bigqueryProbe = options.bigqueryQueryHistoryProbe ?? defaultBigQueryQueryHistoryProbe; + const probe = options.queryHistoryReadinessProbe ?? runHistoricSqlReadinessProbe; const env = options.env ?? process.env; const statuses: QueryHistoryStatus[] = []; @@ -653,18 +484,7 @@ async function buildQueryHistoryStatus( continue; } - const probeInput: QueryHistoryProbeInput = { - projectDir: project.projectDir, - connectionId, - connection, - env, - }; - const dispatched = - dialect === 'postgres' - ? postgresProbeDispatch(probeInput, postgresProbe) - : dialect === 'snowflake' - ? snowflakeProbeDispatch(probeInput, snowflakeProbe) - : bigqueryProbeDispatch(probeInput, bigqueryProbe); + const catalogName = historicSqlProbeCatalogName(dialect); if (options.fast === true) { statuses.push({ @@ -672,36 +492,68 @@ async function buildQueryHistoryStatus( driver, dialect, status: 'skipped', - detail: dispatched.fastSkipDetail, + detail: `${catalogName} probe skipped (--fast)`, }); continue; } - try { - const outcome = await withSpinner(options.useSpinner === true, dispatched.spinnerLabel, dispatched.run); + const outcome = await withSpinner( + options.useSpinner === true, + `Probing ${catalogName} on ${connectionId}`, + () => + probe({ + projectDir: project.projectDir, + connectionId, + connection, + env, + }), + ); + + if (!outcome) { statuses.push({ connection: connectionId, driver, - dialect, - ...outcome, - }); - } catch (error) { - statuses.push({ - connection: connectionId, - driver, - dialect, + dialect: driver, status: 'fail', - detail: failureDetail(error), - fix: probeFailureFix(error, dispatched.label, connectionId, project.projectDir), + detail: `query history is not supported for driver "${driver}"`, + fix: `Disable connections.${connectionId}.context.queryHistory, or use a postgres, snowflake, or bigquery connection`, }); + continue; } + + if (outcome.ok) { + const { detail, warnings } = outcome.runner.formatSuccessDetail(outcome.result); + statuses.push({ + connection: connectionId, + driver, + dialect, + status: warnings.length > 0 ? 'warn' : 'ok', + detail, + ...(dialect === 'postgres' && warnings.length > 0 + ? { + fix: `Update the Postgres parameter group or config, then rerun \`ktx status --project-dir ${project.projectDir}\``, + } + : {}), + }); + continue; + } + + const advice = outcome.runner.fixAdvice(outcome.error); + statuses.push({ + connection: connectionId, + driver, + dialect, + status: 'fail', + detail: advice.failHeadline, + fix: advice.remediation, + }); } return statuses; } const ADAPTER_DRIVER_REQUIREMENT: Record = { - 'live-database': ['postgres', 'postgresql', 'mysql', 'snowflake', 'bigquery', 'clickhouse', 'sqlite', 'sqlserver'], + 'live-database': ['postgres', 'mysql', 'snowflake', 'bigquery', 'clickhouse', 'sqlite', 'sqlserver'], dbt: ['dbt', 'dbt-core', 'dbt-cloud'], notion: ['notion'], metabase: ['metabase'], @@ -740,30 +592,6 @@ function buildWarnings( ): WarningItem[] { const warnings: WarningItem[] = []; - for (const [connectionId, connection] of Object.entries(config.connections)) { - const driver = String(connection.driver ?? '').toLowerCase(); - if (hasOwnField(connection, 'readonly')) { - warnings.push({ - message: `connections.${connectionId}.readonly is no longer used.`, - fix: `Remove connections.${connectionId}.readonly from ktx.yaml.`, - }); - } - - if ((driver === 'sqlite' || driver === 'sqlite3') && hasOwnField(connection, 'file_path')) { - warnings.push({ - message: `connections.${connectionId}.file_path was removed.`, - fix: `Rename connections.${connectionId}.file_path to path.`, - }); - } - - if (driver === 'notion' && hasOwnField(connection, 'last_successful_cursor')) { - warnings.push({ - message: `connections.${connectionId}.last_successful_cursor is local sync state.`, - fix: 'Remove it from ktx.yaml. KTX stores the Notion cursor in .ktx/db.sqlite.', - }); - } - } - for (const adapter of config.ingest.adapters) { const requiredDrivers = ADAPTER_DRIVER_REQUIREMENT[adapter]; if (!requiredDrivers) continue; @@ -797,6 +625,13 @@ function buildWarnings( }); } + if (llm.backend === 'codex') { + warnings.push({ + message: CODEX_ISOLATION_WARNING, + fix: CODEX_ISOLATION_WARNING_FIX, + }); + } + return warnings; } @@ -857,10 +692,9 @@ function buildVerdict( export interface BuildProjectStatusOptions { env?: NodeJS.ProcessEnv; - postgresQueryHistoryProbe?: PostgresQueryHistoryProbe; - snowflakeQueryHistoryProbe?: SnowflakeQueryHistoryProbe; - bigqueryQueryHistoryProbe?: BigQueryQueryHistoryProbe; + queryHistoryReadinessProbe?: HistoricSqlReadinessProbe; claudeCodeAuthProbe?: ClaudeCodeAuthProbe; + codexAuthProbe?: CodexAuthProbe; configIssues?: KtxConfigIssue[]; fast?: boolean; useSpinner?: boolean; @@ -1109,6 +943,7 @@ export async function buildProjectStatus(project: KtxLocalProject, options: Buil projectDir: project.projectDir, env, claudeCodeAuthProbe: options.claudeCodeAuthProbe, + codexAuthProbe: options.codexAuthProbe, fast: options.fast, useSpinner: options.useSpinner, }); diff --git a/packages/cli/src/telemetry/command-hook.ts b/packages/cli/src/telemetry/command-hook.ts index e4f003d7..99f8723e 100644 --- a/packages/cli/src/telemetry/command-hook.ts +++ b/packages/cli/src/telemetry/command-hook.ts @@ -1,4 +1,4 @@ -import { scrubErrorClass } from './scrubber.js'; +import { formatErrorDetail, scrubErrorClass } from './scrubber.js'; export type CommandOutcome = 'ok' | 'error' | 'aborted'; @@ -16,6 +16,7 @@ export interface CompletedCommandSpan { durationMs: number; outcome: CommandOutcome; errorClass?: string; + errorDetail?: string; flagsPresent: Record; hasProject: boolean; projectDir?: string; @@ -40,12 +41,14 @@ export function completeCommandSpan(input: { } const errorClass = input.error ? scrubErrorClass(input.error) : undefined; + const errorDetail = input.error ? formatErrorDetail(input.error) : undefined; return { commandPath: span.commandPath, durationMs: Math.max(0, input.completedAt - span.startedAt), outcome: input.outcome, ...(errorClass ? { errorClass } : {}), + ...(errorDetail ? { errorDetail } : {}), flagsPresent: span.flagsPresent, hasProject: span.hasProject, projectDir: span.projectDir, diff --git a/packages/cli/src/telemetry/emitter.ts b/packages/cli/src/telemetry/emitter.ts index 435a122b..12453262 100644 --- a/packages/cli/src/telemetry/emitter.ts +++ b/packages/cli/src/telemetry/emitter.ts @@ -16,6 +16,16 @@ type PostHogClient = { properties: Record; groups?: Record; }): void; + captureException( + error: unknown, + distinctId?: string, + additionalProperties?: Record, + ): void; + captureExceptionImmediate( + error: unknown, + distinctId?: string, + additionalProperties?: Record, + ): Promise; shutdown(): Promise | void; }; @@ -44,7 +54,7 @@ async function getPostHogClient(projectApiKey: string, host: string): Promise new PostHog(projectApiKey, { host, flushAt: 1, flushInterval: 0 })) + .then(({ PostHog }) => new PostHog(projectApiKey, { host, flushAt: 1, flushInterval: 0, disableGeoip: false })) .catch(() => null); return await clientPromise; @@ -105,6 +115,57 @@ export async function trackTelemetryEvent(input: { } } +function writeDebugExceptionPayload(input: { + error: Error; + distinctId: string; + properties: Record; + stderr: TelemetrySink; +}): void { + input.stderr.write( + `[telemetry-exception] ${JSON.stringify({ + distinctId: input.distinctId, + message: input.error.message, + name: input.error.name, + properties: input.properties, + })}\n`, + ); +} + +export async function trackTelemetryException(input: { + error: Error; + distinctId: string; + properties: Record; + env?: TelemetryEmitterEnv; + stderr: TelemetrySink; + projectApiKey?: string; + host?: string; + immediate?: boolean; +}): Promise { + const env = input.env ?? process.env; + + if (debugEnabled(env)) { + writeDebugExceptionPayload(input); + return; + } + + const projectApiKey = telemetryProjectApiKey(input.projectApiKey); + const host = telemetryHost(env, input.host); + const client = await getPostHogClient(projectApiKey, host); + if (!client) { + return; + } + + try { + if (input.immediate) { + await client.captureExceptionImmediate(input.error, input.distinctId, input.properties); + return; + } + client.captureException(input.error, input.distinctId, input.properties); + } catch { + return; + } +} + export async function shutdownTelemetryEmitter(): Promise { const client = await clientPromise; if (!client) { diff --git a/packages/cli/src/telemetry/events.schema.json b/packages/cli/src/telemetry/events.schema.json index 13642c49..c6c3d6f8 100644 --- a/packages/cli/src/telemetry/events.schema.json +++ b/packages/cli/src/telemetry/events.schema.json @@ -26,6 +26,7 @@ "durationMs", "outcome", "errorClass", + "errorDetail", "flagsPresent", "hasProject", "projectGroupAttached" @@ -37,7 +38,8 @@ "fields": [ "step", "outcome", - "durationMs" + "durationMs", + "errorDetail" ] }, { @@ -56,6 +58,7 @@ "isDemoConnection", "outcome", "errorClass", + "errorDetail", "durationMs", "serverVersion" ] @@ -84,7 +87,8 @@ "rowsBucket", "durationMs", "outcome", - "errorClass" + "errorClass", + "errorDetail" ] }, { @@ -98,7 +102,8 @@ "declaredFkCount", "durationMs", "outcome", - "errorClass" + "errorClass", + "errorDetail" ] }, { @@ -157,7 +162,9 @@ "outcome", "durationMs", "errorClass", - "sampleRate" + "sampleRate", + "mcpClientName", + "mcpClientVersion" ] }, { @@ -199,6 +206,17 @@ "errorClass", "durationMs" ] + }, + { + "name": "query_history_filter_completed", + "description": "Emitted after the setup query-history service-account filter picker runs.", + "fields": [ + "dialect", + "consideredRoleCount", + "excludedRoleCount", + "parseFailedCount", + "outcome" + ] } ], "$defs": { @@ -294,6 +312,10 @@ "errorClass": { "type": "string" }, + "errorDetail": { + "type": "string", + "maxLength": 1000 + }, "flagsPresent": { "type": "object", "propertyNames": { @@ -365,7 +387,6 @@ "embeddings", "secrets", "databases", - "database-context-depth", "sources", "context", "agents", @@ -383,6 +404,10 @@ "durationMs": { "type": "number", "minimum": 0 + }, + "errorDetail": { + "type": "string", + "maxLength": 1000 } }, "required": [ @@ -493,6 +518,10 @@ "errorClass": { "type": "string" }, + "errorDetail": { + "type": "string", + "maxLength": 1000 + }, "durationMs": { "type": "number", "minimum": 0 @@ -672,6 +701,10 @@ }, "errorClass": { "type": "string" + }, + "errorDetail": { + "type": "string", + "maxLength": 1000 } }, "required": [ @@ -758,6 +791,10 @@ }, "errorClass": { "type": "string" + }, + "errorDetail": { + "type": "string", + "maxLength": 1000 } }, "required": [ @@ -1132,7 +1169,13 @@ }, "sampleRate": { "type": "number", - "const": 0.1 + "const": 1 + }, + "mcpClientName": { + "type": "string" + }, + "mcpClientVersion": { + "type": "string" } }, "required": [ @@ -1402,6 +1445,77 @@ "durationMs" ], "additionalProperties": false + }, + "query_history_filter_completed": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "cliVersion": { + "type": "string" + }, + "nodeVersion": { + "type": "string" + }, + "osPlatform": { + "type": "string" + }, + "osRelease": { + "type": "string" + }, + "arch": { + "type": "string" + }, + "runtime": { + "type": "string", + "enum": [ + "node", + "daemon-py" + ] + }, + "isCi": { + "type": "boolean" + }, + "dialect": { + "type": "string" + }, + "consideredRoleCount": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "excludedRoleCount": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "parseFailedCount": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "outcome": { + "type": "string", + "enum": [ + "ok", + "error" + ] + } + }, + "required": [ + "cliVersion", + "nodeVersion", + "osPlatform", + "osRelease", + "arch", + "runtime", + "isCi", + "dialect", + "consideredRoleCount", + "excludedRoleCount", + "parseFailedCount", + "outcome" + ], + "additionalProperties": false } } } diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts index e73001ed..cf650492 100644 --- a/packages/cli/src/telemetry/events.ts +++ b/packages/cli/src/telemetry/events.ts @@ -21,6 +21,7 @@ const commandSchema = telemetryCommonEnvelopeSchema durationMs: z.number().nonnegative(), outcome: z.enum(['ok', 'error', 'aborted']), errorClass: z.string().optional(), + errorDetail: z.string().max(1000).optional(), flagsPresent: z.record(z.string(), z.boolean()), hasProject: z.boolean(), projectGroupAttached: z.boolean(), @@ -38,7 +39,6 @@ const setupStepSchema = telemetryCommonEnvelopeSchema 'embeddings', 'secrets', 'databases', - 'database-context-depth', 'sources', 'context', 'agents', @@ -46,6 +46,7 @@ const setupStepSchema = telemetryCommonEnvelopeSchema ]), outcome: z.enum(['completed', 'skipped', 'abandoned']), durationMs: z.number().nonnegative(), + errorDetail: z.string().max(1000).optional(), }) .strict(); @@ -62,6 +63,7 @@ const connectionTestSchema = telemetryCommonEnvelopeSchema isDemoConnection: z.boolean(), outcome: outcomeSchema, errorClass: z.string().optional(), + errorDetail: z.string().max(1000).optional(), durationMs: z.number().nonnegative(), serverVersion: z.string().optional(), }) @@ -91,6 +93,7 @@ const ingestCompletedSchema = telemetryCommonEnvelopeSchema durationMs: z.number().nonnegative(), outcome: outcomeSchema, errorClass: z.string().optional(), + errorDetail: z.string().max(1000).optional(), }) .strict(); @@ -104,6 +107,7 @@ const scanCompletedSchema = telemetryCommonEnvelopeSchema durationMs: z.number().nonnegative(), outcome: outcomeSchema, errorClass: z.string().optional(), + errorDetail: z.string().max(1000).optional(), }) .strict(); @@ -157,7 +161,12 @@ const mcpRequestCompletedSchema = telemetryCommonEnvelopeSchema outcome: outcomeSchema, durationMs: z.number().nonnegative(), errorClass: z.string().optional(), - sampleRate: z.literal(0.1), + sampleRate: z.literal(1), + // Raw, client-tool-controlled identity from the MCP initialize handshake + // (clientInfo.name/version). Optional: clients may omit clientInfo. Stored + // verbatim — normalize the free-form names at query time, not at write time. + mcpClientName: z.string().optional(), + mcpClientVersion: z.string().optional(), }) .strict(); @@ -197,6 +206,16 @@ const sqlGenCompletedSchema = telemetryCommonEnvelopeSchema }) .strict(); +const queryHistoryFilterCompletedSchema = telemetryCommonEnvelopeSchema + .extend({ + dialect: z.string(), + consideredRoleCount: z.number().int().nonnegative(), + excludedRoleCount: z.number().int().nonnegative(), + parseFailedCount: z.number().int().nonnegative(), + outcome: outcomeSchema, + }) + .strict(); + /** @internal */ export const telemetryEventSchemas = { install_first_run: installFirstRunSchema, @@ -216,6 +235,7 @@ export const telemetryEventSchemas = { daemon_stopped: daemonStoppedSchema, sl_plan_completed: slPlanCompletedSchema, sql_gen_completed: sqlGenCompletedSchema, + query_history_filter_completed: queryHistoryFilterCompletedSchema, } as const; /** @internal */ @@ -233,6 +253,7 @@ export const telemetryEventCatalog = [ 'durationMs', 'outcome', 'errorClass', + 'errorDetail', 'flagsPresent', 'hasProject', 'projectGroupAttached', @@ -241,7 +262,7 @@ export const telemetryEventCatalog = [ { name: 'setup_step', description: 'Emitted after an interactive setup step completes, skips, or aborts.', - fields: ['step', 'outcome', 'durationMs'], + fields: ['step', 'outcome', 'durationMs', 'errorDetail'], }, { name: 'connection_added', @@ -251,7 +272,7 @@ export const telemetryEventCatalog = [ { name: 'connection_test', description: 'Emitted after ktx connection test completes.', - fields: ['driver', 'isDemoConnection', 'outcome', 'errorClass', 'durationMs', 'serverVersion'], + fields: ['driver', 'isDemoConnection', 'outcome', 'errorClass', 'errorDetail', 'durationMs', 'serverVersion'], }, { name: 'project_stack_snapshot', @@ -271,6 +292,7 @@ export const telemetryEventCatalog = [ 'durationMs', 'outcome', 'errorClass', + 'errorDetail', ], }, { @@ -285,6 +307,7 @@ export const telemetryEventCatalog = [ 'durationMs', 'outcome', 'errorClass', + 'errorDetail', ], }, { @@ -326,7 +349,7 @@ export const telemetryEventCatalog = [ { name: 'mcp_request_completed', description: 'Emitted for sampled MCP tool requests.', - fields: ['toolName', 'outcome', 'durationMs', 'errorClass', 'sampleRate'], + fields: ['toolName', 'outcome', 'durationMs', 'errorClass', 'sampleRate', 'mcpClientName', 'mcpClientVersion'], }, { name: 'daemon_started', @@ -348,6 +371,11 @@ export const telemetryEventCatalog = [ description: 'Emitted after daemon SQL generation completes.', fields: ['outcome', 'dialect', 'errorClass', 'durationMs'], }, + { + name: 'query_history_filter_completed', + description: 'Emitted after the setup query-history service-account filter picker runs.', + fields: ['dialect', 'consideredRoleCount', 'excludedRoleCount', 'parseFailedCount', 'outcome'], + }, ] as const; export type TelemetryEventName = keyof typeof telemetryEventSchemas; diff --git a/packages/cli/src/telemetry/exception.ts b/packages/cli/src/telemetry/exception.ts new file mode 100644 index 00000000..0ce81244 --- /dev/null +++ b/packages/cli/src/telemetry/exception.ts @@ -0,0 +1,201 @@ +import { inspect } from 'node:util'; + +import { getKtxCliPackageInfo, type KtxCliIo, type KtxCliPackageInfo } from '../cli-runtime.js'; +import { buildCommonEnvelope } from './events.js'; +import { trackTelemetryException } from './emitter.js'; +import { computeTelemetryProjectId, loadTelemetryIdentity } from './identity.js'; + +export interface ExceptionContext { + source: string; + handled: boolean; + fatal: boolean; + extra?: Record; +} + +type AnyObject = object; + +const reportedObjects = new WeakSet(); +const recentHandledPrimitives: string[] = []; +const RECENT_PRIMITIVE_LIMIT = 128; + +function primitiveKey(value: unknown): string { + return `${typeof value}:${String(value)}`; +} + +function rememberHandledPrimitive(value: unknown): void { + recentHandledPrimitives.push(primitiveKey(value)); + if (recentHandledPrimitives.length > RECENT_PRIMITIVE_LIMIT) { + recentHandledPrimitives.splice(0, recentHandledPrimitives.length - RECENT_PRIMITIVE_LIMIT); + } +} + +function consumeHandledPrimitive(value: unknown): boolean { + const key = primitiveKey(value); + const index = recentHandledPrimitives.indexOf(key); + if (index < 0) { + return false; + } + recentHandledPrimitives.splice(index, 1); + return true; +} + +function shouldSkipAsAlreadyReported(error: unknown, handled: boolean): boolean { + if ((typeof error === 'object' || typeof error === 'function') && error !== null) { + if (reportedObjects.has(error)) { + return true; + } + reportedObjects.add(error); + return false; + } + + if (handled) { + rememberHandledPrimitive(error); + return false; + } + + return consumeHandledPrimitive(error); +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function redactStaticPatterns(value: string): string { + return value + .replace(/([a-z][a-z0-9+.-]*:\/\/[^:\s/@]+:)([^@\s/]+)(@)/gi, '$1[redacted]$3') + .replace(/\b(password|pwd)=([^;&\s]+)/gi, '$1=[redacted]') + .replace(/\bAuthorization\s*:\s*[^\r\n,;]+/gi, 'Authorization: [redacted]') + .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, 'Bearer [redacted]') + .replace(/\b(api[_-]?key)\s*[:=]\s*([^\s,;]+)/gi, '$1=[redacted]') + .replace(/\b(KTX_[A-Z0-9_]*|[A-Z0-9_]*(?:TOKEN|SECRET))\s*[:=]\s*([^\s,;]+)/g, '$1=[redacted]') + .replace(/([?&](?:X-Amz-Signature|X-Goog-Signature|sig)=)[^&\s]+/gi, '$1[redacted]'); +} + +function redactText(value: string, secrets: ReadonlyArray): string { + let redacted = value; + for (const secret of secrets) { + if (secret) { + redacted = redacted.replace(new RegExp(escapeRegExp(secret), 'g'), '[redacted]'); + } + } + return redactStaticPatterns(redacted); +} + +const FORBIDDEN_EXTRA_PROPERTY_KEYS = new Set([ + 'argv', + 'args', + 'env', + 'environment', + 'sql', + 'query', + 'prompt', + 'mcparguments', + 'mcpargs', + 'tablename', + 'schemaname', + 'columnname', + 'databaseurl', + 'connectionstring', + 'url', + 'password', + 'token', + 'apikey', + 'api_key', + 'authorization', +]); + +function safeExtraProperties( + extra: Record | undefined, +): Record { + const safe: Record = {}; + for (const [key, value] of Object.entries(extra ?? {})) { + if (!FORBIDDEN_EXTRA_PROPERTY_KEYS.has(key.replace(/[^a-z0-9_]/gi, '').toLowerCase())) { + safe[key] = value; + } + } + return safe; +} + +function toMessage(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + if (typeof error === 'string') { + return error; + } + return inspect(error, { depth: 4, breakLength: 120 }); +} + +function sanitizedError(error: unknown, secrets: ReadonlyArray): Error { + if (error instanceof Error) { + const cause = 'cause' in error ? (error as Error & { cause?: unknown }).cause : undefined; + const clone = new Error(redactText(error.message, secrets), { + ...(cause !== undefined ? { cause: sanitizedError(cause, secrets) } : {}), + }); + clone.name = error.name; + if (error.stack) { + clone.stack = redactText(error.stack, secrets); + } + return clone; + } + return new Error(redactText(toMessage(error), secrets)); +} + +export async function reportException(input: { + error: unknown; + context: ExceptionContext; + io: KtxCliIo; + packageInfo?: KtxCliPackageInfo; + projectDir?: string; + immediate?: boolean; + redactionSecrets?: ReadonlyArray; +}): Promise { + try { + if (shouldSkipAsAlreadyReported(input.error, input.context.handled)) { + return; + } + + const debug = process.env.KTX_TELEMETRY_DEBUG === '1'; + const identity = await loadTelemetryIdentity({ + stderr: input.io.stderr, + env: process.env, + }); + + if ((!identity.enabled || !identity.installId) && !debug) { + return; + } + + const packageInfo = input.packageInfo ?? getKtxCliPackageInfo(); + const installId = identity.installId ?? 'debug'; + const projectId = input.projectDir ? computeTelemetryProjectId(installId, input.projectDir) : undefined; + const safeError = sanitizedError(input.error, input.redactionSecrets ?? []); + const properties: Record = { + ...buildCommonEnvelope({ + cliVersion: packageInfo.version, + isCi: Boolean(process.env.CI), + }), + source: input.context.source, + handled: input.context.handled, + fatal: input.context.fatal, + ...(projectId ? { projectId } : {}), + ...safeExtraProperties(input.context.extra), + }; + + delete properties.$groups; + await trackTelemetryException({ + error: safeError, + distinctId: installId, + properties, + env: process.env, + stderr: input.io.stderr, + immediate: input.immediate, + }); + } catch { + return; + } +} + +/** @internal */ +export function __resetTelemetryExceptionStateForTests(): void { + recentHandledPrimitives.length = 0; +} diff --git a/packages/cli/src/telemetry/identity.ts b/packages/cli/src/telemetry/identity.ts index 4d46307c..ee5f7a39 100644 --- a/packages/cli/src/telemetry/identity.ts +++ b/packages/cli/src/telemetry/identity.ts @@ -6,7 +6,7 @@ import { z } from 'zod'; /** @internal */ export const TELEMETRY_NOTICE = - 'ktx collects anonymous usage data to improve the product. Opt out: set KTX_TELEMETRY_DISABLED=1.'; + 'ktx collects usage data to improve the product. Opt out: set KTX_TELEMETRY_DISABLED=1.'; const NOTICE_VERSION = 1; @@ -37,7 +37,6 @@ function styleNotice(notice: string, env: TelemetryIdentityEnv): string { export interface LoadTelemetryIdentityOptions { homeDir?: string; env?: TelemetryIdentityEnv; - stdoutIsTTY: boolean; stderr: { write(chunk: string): void }; now?: () => Date; } @@ -75,17 +74,14 @@ export async function loadTelemetryIdentity(options: LoadTelemetryIdentityOption const env = options.env ?? process.env; const path = telemetryPath(options.homeDir ?? homedir()); - if (envDisablesTelemetry(env) || options.stdoutIsTTY !== true) { - const existing = await readTelemetryFile(path); - return { - installId: existing?.installId, - enabled: false, - createdFile: false, - noticeShown: false, - path, - }; + if (envDisablesTelemetry(env)) { + return { enabled: false, createdFile: false, noticeShown: false, path }; } + // Honor an already-consented identity regardless of the current surface. + // Telemetry enablement follows the persisted decision and opt-out env vars, + // not whether this invocation happens to own a TTY — MCP servers always run + // headless (stdio stubs stdout; the HTTP server runs detached). const existing = await readTelemetryFile(path); if (existing) { return { @@ -97,6 +93,12 @@ export async function loadTelemetryIdentity(options: LoadTelemetryIdentityOption }; } + // No identity yet → mint one regardless of surface. Telemetry is opt-out, so + // a fresh install is counted even when its first run is headless (an + // IDE-launched `ktx mcp stdio`, a scripted invocation); otherwise those + // installs would be permanently invisible. Opt-out env vars are honored + // above. The one-time notice is written to stderr — safe even under MCP + // stdio, which reserves stdout for its JSON-RPC protocol. const timestamp = (options.now ?? (() => new Date()))().toISOString(); const next = { installId: randomUUID(), diff --git a/packages/cli/src/telemetry/index.ts b/packages/cli/src/telemetry/index.ts index 10637a3d..e3716060 100644 --- a/packages/cli/src/telemetry/index.ts +++ b/packages/cli/src/telemetry/index.ts @@ -7,6 +7,7 @@ import { type CompletedCommandSpan, } from './command-hook.js'; import { shutdownTelemetryEmitter, trackTelemetryEvent } from './emitter.js'; +import { reportException, type ExceptionContext } from './exception.js'; import { buildCommonEnvelope, buildTelemetryEvent, @@ -17,12 +18,11 @@ import { import { computeTelemetryProjectId, loadTelemetryIdentity } from './identity.js'; import { buildProjectStackSnapshotFields } from './project-snapshot.js'; -export { beginCommandSpan, completeCommandSpan, shutdownTelemetryEmitter }; -export type { CommandOutcome, CompletedCommandSpan }; +export { beginCommandSpan, completeCommandSpan, reportException, shutdownTelemetryEmitter }; +export type { CommandOutcome, CompletedCommandSpan, ExceptionContext }; export async function showTelemetryNoticeIfNeeded(io: KtxCliIo, packageInfo: KtxCliPackageInfo): Promise { const identity = await loadTelemetryIdentity({ - stdoutIsTTY: io.stdout.isTTY === true, stderr: io.stderr, env: process.env, }); @@ -52,15 +52,23 @@ type TelemetryEventFields = Omit< >; const emittedProjectSnapshots = new Set(); -const MCP_SAMPLE_RATE = 0.1 as const; +// MCP tool calls are captured at full rate while ktx is early-stage: at current +// install counts any sampling below 1.0 yields too few events to be useful, and +// the recorded sampleRate lets us dial this down (and reweight history) once +// per-session call volume justifies it. +const MCP_SAMPLE_RATE = 1 as const; let mcpSampled: boolean | undefined; +function telemetryDebugEnabled(): boolean { + return process.env.KTX_TELEMETRY_DEBUG === '1'; +} + export function shouldEmitMcpTelemetry(): boolean { mcpSampled ??= Math.random() < MCP_SAMPLE_RATE; return mcpSampled; } -export function mcpTelemetrySampleRate(): 0.1 { +export function mcpTelemetrySampleRate(): 1 { return MCP_SAMPLE_RATE; } @@ -71,19 +79,20 @@ export async function emitTelemetryEvent(input: packageInfo?: KtxCliPackageInfo; projectDir?: string; }): Promise { + const debug = telemetryDebugEnabled(); const identity = await loadTelemetryIdentity({ - stdoutIsTTY: input.io.stdout.isTTY === true, stderr: input.io.stderr, env: process.env, }); - if (!identity.enabled || !identity.installId) { + if ((!identity.enabled || !identity.installId) && !debug) { return; } const packageInfo = input.packageInfo ?? getKtxCliPackageInfo(); + const installId = identity.installId ?? 'debug'; - const projectId = input.projectDir ? computeTelemetryProjectId(identity.installId, input.projectDir) : undefined; + const projectId = input.projectDir ? computeTelemetryProjectId(installId, input.projectDir) : undefined; await trackTelemetryEvent({ event: buildTelemetryEvent( input.name, @@ -93,7 +102,7 @@ export async function emitTelemetryEvent(input: }), input.fields, ), - distinctId: identity.installId, + distinctId: installId, projectId, env: process.env, stderr: input.io.stderr, @@ -144,3 +153,20 @@ export async function emitCompletedCommand(input: { packageInfo: input.packageInfo, }); } + +/** + * Flush telemetry when the process is interrupted (Ctrl-C / kill). The normal + * `command` emit + flush lives in a `finally` that a signal skips, so without + * this an interrupted long-running command (ingest, `mcp stdio`) loses its + * `command` event and any queued events. Marks the active command span as + * `aborted`, emits it, and drains the emitter. Best-effort and idempotent: if + * the span was already completed (normal exit racing a signal) the emit no-ops. + */ +export async function emitAbortedCommandAndShutdown(input: { + packageInfo: KtxCliPackageInfo; + io: KtxCliIo; +}): Promise { + const completed = completeCommandSpan({ completedAt: performance.now(), outcome: 'aborted' }); + await emitCompletedCommand({ completed, packageInfo: input.packageInfo, io: input.io }); + await shutdownTelemetryEmitter(); +} diff --git a/packages/cli/src/telemetry/redaction-secrets.ts b/packages/cli/src/telemetry/redaction-secrets.ts new file mode 100644 index 00000000..2bf7a863 --- /dev/null +++ b/packages/cli/src/telemetry/redaction-secrets.ts @@ -0,0 +1,117 @@ +import { resolveKtxConfigReference } from '../context/core/config-reference.js'; +import { loadKtxProject, type KtxLocalProject } from '../context/project/project.js'; + +const SENSITIVE_KEY = + /(password|secret|token|api[_-]?key|auth[_-]?token|auth_token_ref|private[_-]?key|passphrase|credential|authorization|url)$/i; + +type TelemetryRedactionProject = Pick; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function addSecret(values: string[], value: string | undefined): void { + const trimmed = value?.trim(); + if (trimmed && !values.includes(trimmed)) { + values.push(trimmed); + } +} + +function tryResolve(value: string, env: NodeJS.ProcessEnv): string | undefined { + try { + return resolveKtxConfigReference(value, env); + } catch { + return undefined; + } +} + +function addUrlCredentials(values: string[], value: string): void { + try { + const parsed = new URL(value); + addSecret(values, parsed.password ? decodeURIComponent(parsed.password) : undefined); + addSecret(values, parsed.username ? decodeURIComponent(parsed.username) : undefined); + } catch { + return; + } +} + +function collectFromRecord(input: unknown, env: NodeJS.ProcessEnv, values: string[]): void { + if (Array.isArray(input)) { + for (const item of input) { + collectFromRecord(item, env, values); + } + return; + } + + if (!isRecord(input)) { + return; + } + + for (const [key, raw] of Object.entries(input)) { + if (isRecord(raw) || Array.isArray(raw)) { + collectFromRecord(raw, env, values); + continue; + } + if (typeof raw !== 'string' || !SENSITIVE_KEY.test(key)) { + continue; + } + const resolved = tryResolve(raw, env); + addSecret(values, resolved); + if (resolved) { + addUrlCredentials(values, resolved); + } + } +} + +function collectLlmSecrets(project: TelemetryRedactionProject, env: NodeJS.ProcessEnv, values: string[]): void { + collectFromRecord(project.config.llm.provider, env, values); +} + +function collectEmbeddingSecrets(project: TelemetryRedactionProject, env: NodeJS.ProcessEnv, values: string[]): void { + collectFromRecord(project.config.ingest.embeddings, env, values); + collectFromRecord(project.config.scan.enrichment.embeddings, env, values); +} + +function collectConnectionSecrets( + project: TelemetryRedactionProject, + connectionId: string | undefined, + env: NodeJS.ProcessEnv, + values: string[], +): void { + if (!connectionId) { + return; + } + collectFromRecord(project.config.connections[connectionId], env, values); +} + +export async function collectTelemetryRedactionSecrets(input: { + project?: TelemetryRedactionProject; + projectDir?: string; + connectionId?: string; + includeLlm?: boolean; + includeEmbeddings?: boolean; + env?: NodeJS.ProcessEnv; +}): Promise { + const env = input.env ?? process.env; + let project = input.project; + if (!project && input.projectDir) { + try { + project = await loadKtxProject({ projectDir: input.projectDir }); + } catch { + project = undefined; + } + } + if (!project) { + return []; + } + + const values: string[] = []; + if (input.includeLlm) { + collectLlmSecrets(project, env, values); + } + if (input.includeEmbeddings) { + collectEmbeddingSecrets(project, env, values); + } + collectConnectionSecrets(project, input.connectionId, env, values); + return values; +} diff --git a/packages/cli/src/telemetry/scrubber.test.ts b/packages/cli/src/telemetry/scrubber.test.ts deleted file mode 100644 index 87eb74d4..00000000 --- a/packages/cli/src/telemetry/scrubber.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { scrubErrorClass } from './scrubber.js'; - -class KtxProjectMissingAbortError extends Error {} - -describe('scrubErrorClass', () => { - it('keeps normal JavaScript class names', () => { - expect(scrubErrorClass(new KtxProjectMissingAbortError('missing'))).toBe('KtxProjectMissingAbortError'); - }); - - it('drops path-like, URL-like, email-like, and long values', () => { - expect(scrubErrorClass({ constructor: { name: '/Users/alice/project' } })).toBeUndefined(); - expect(scrubErrorClass({ constructor: { name: 'https://example.test/error' } })).toBeUndefined(); - expect(scrubErrorClass({ constructor: { name: 'alice@example.test' } })).toBeUndefined(); - expect(scrubErrorClass({ constructor: { name: 'A'.repeat(81) } })).toBeUndefined(); - }); - - it('drops lowercase, spaced, and non-error-like values', () => { - expect(scrubErrorClass({ constructor: { name: 'lowercaseError' } })).toBeUndefined(); - expect(scrubErrorClass({ constructor: { name: 'Bad Error' } })).toBeUndefined(); - expect(scrubErrorClass('plain string')).toBeUndefined(); - expect(scrubErrorClass(null)).toBeUndefined(); - }); -}); diff --git a/packages/cli/src/telemetry/scrubber.ts b/packages/cli/src/telemetry/scrubber.ts index 27e41f87..a7b8c393 100644 --- a/packages/cli/src/telemetry/scrubber.ts +++ b/packages/cli/src/telemetry/scrubber.ts @@ -26,3 +26,27 @@ export function scrubErrorClass(error: unknown): string | undefined { return constructorName; } + +const MAX_ERROR_DETAIL_LENGTH = 1000; + +/** + * Human-readable failure detail for telemetry: the error's `.code` (when + * present) prefixed onto its `message`, collapsed to a single line and + * length-capped. Captures the message only — never the stack. + * + * This intentionally forwards raw error text, which can include identifiers from + * the user's environment (table/column names, hostnames, usernames), so that + * funnel failures are diagnosable. Callers must gate it to the failure path. + */ +export function formatErrorDetail(error: unknown): string | undefined { + if (error === undefined || error === null) { + return undefined; + } + + const code = (error as { code?: unknown }).code; + const message = error instanceof Error ? error.message : String(error); + const prefix = typeof code === 'string' || typeof code === 'number' ? `${code}: ` : ''; + const detail = `${prefix}${message}`.replace(/\s+/g, ' ').trim(); + + return detail.length > 0 ? detail.slice(0, MAX_ERROR_DETAIL_LENGTH) : undefined; +} diff --git a/packages/cli/src/tree-picker-state.ts b/packages/cli/src/tree-picker-state.ts index 3e96e096..2392ef68 100644 --- a/packages/cli/src/tree-picker-state.ts +++ b/packages/cli/src/tree-picker-state.ts @@ -25,7 +25,8 @@ export interface PickerState { expanded: Set; checked: Set; cursorId: string; - search: { editing: boolean; query: string }; + search: { query: string }; + isNavigating: boolean; pendingConfirm: PendingConfirmKind | null; preLoadWarnings: string[]; transientHint: { text: string; expiresAt: number } | null; @@ -47,9 +48,7 @@ export type PickerCommand = | 'toggle-select-all-visible' | 'select-none' | 'clear-transient-hint' - | 'search-start' - | 'search-cancel' - | 'search-submit' + | 'search-clear' | 'search-backspace' | { type: 'search-input'; value: string } | 'save-request' @@ -464,7 +463,8 @@ export function buildInitialState(args: { expanded, checked: minimalChecked, cursorId: args.tree[0]?.id ?? '', - search: { editing: false, query: '' }, + search: { query: '' }, + isNavigating: false, pendingConfirm: null, preLoadWarnings, transientHint: null, @@ -473,6 +473,14 @@ export function buildInitialState(args: { }; } +function refocusVisibleCursor(state: PickerState): PickerState { + const ids = visibleNodeIds(state); + if (ids.length === 0 || ids.includes(state.cursorId)) { + return state; + } + return cloneState(state, { cursorId: ids[0]! }); +} + export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now()): { next: PickerState; effect: PickerEffect } { if (state.pendingConfirm) { if (cmd === 'save-confirm') { @@ -491,13 +499,13 @@ export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now() switch (cmd) { case 'cursor-up': - return { next: moveCursor(state, 'up'), effect: null }; + return { next: cloneState(moveCursor(state, 'up'), { isNavigating: true }), effect: null }; case 'cursor-down': - return { next: moveCursor(state, 'down'), effect: null }; + return { next: cloneState(moveCursor(state, 'down'), { isNavigating: true }), effect: null }; case 'cursor-left': - return { next: moveCursor(state, 'left'), effect: null }; + return { next: cloneState(moveCursor(state, 'left'), { isNavigating: true }), effect: null }; case 'cursor-right': - return { next: moveCursor(state, 'right'), effect: null }; + return { next: cloneState(moveCursor(state, 'right'), { isNavigating: true }), effect: null }; case 'expand': return { next: setExpanded(state, state.cursorId, 'toggle'), effect: null }; case 'collapse': @@ -521,15 +529,19 @@ export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now() return { next: selectNone(state), effect: null }; case 'clear-transient-hint': return { next: clearExpiredTransientHint(state, now), effect: null }; - case 'search-start': - return { next: cloneState(state, { search: { ...state.search, editing: true } }), effect: null }; - case 'search-cancel': - return { next: cloneState(state, { search: { editing: false, query: '' } }), effect: null }; - case 'search-submit': - return { next: cloneState(state, { search: { ...state.search, editing: false } }), effect: null }; + case 'search-clear': + return { + next: cloneState(state, { search: { query: '' }, isNavigating: false }), + effect: null, + }; case 'search-backspace': return { - next: cloneState(state, { search: { ...state.search, query: state.search.query.slice(0, -1) } }), + next: refocusVisibleCursor( + cloneState(state, { + search: { query: state.search.query.slice(0, -1) }, + isNavigating: false, + }), + ), effect: null, }; case 'save-request': @@ -546,6 +558,14 @@ export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now() case 'quit': return { next: state, effect: 'quit-without-save' }; default: - return { next: cloneState(state, { search: { ...state.search, query: state.search.query + cmd.value } }), effect: null }; + return { + next: refocusVisibleCursor( + cloneState(state, { + search: { query: state.search.query + cmd.value }, + isNavigating: false, + }), + ), + effect: null, + }; } } diff --git a/packages/cli/src/tree-picker-tui.tsx b/packages/cli/src/tree-picker-tui.tsx index 57525270..94fc0dd6 100644 --- a/packages/cli/src/tree-picker-tui.tsx +++ b/packages/cli/src/tree-picker-tui.tsx @@ -32,7 +32,7 @@ const NO_COLOR_THEME = { type TreePickerTheme = Record; const DEFAULT_TREE_PICKER_HELP_TEXT = - 'Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape to go back, or Ctrl+C to exit.'; + 'Up/Down to move, Right/Left to expand or collapse, Tab to select, Type to search, Enter to confirm, Escape to clear search or go back, Ctrl+C to exit.'; const DEFAULT_SKIP_EMPTY_MESSAGE = 'Nothing selected. Skip this step? Press Enter to skip or Escape to go back.'; @@ -50,6 +50,8 @@ interface InkKey { return?: boolean; escape?: boolean; ctrl?: boolean; + tab?: boolean; + shift?: boolean; backspace?: boolean; delete?: boolean; } @@ -147,35 +149,27 @@ function truncateText(value: string, width: number): string { export function treePickerCommandForInkInput( input: string, key: InkKey, - search: PickerState['search'], - pendingConfirm: PickerState['pendingConfirm'], + state: Pick, ): PickerCommand | null { - if (pendingConfirm) { + if (state.pendingConfirm) { if (input === 'y' || key.return) return 'save-confirm'; if (input === 'n' || key.escape) return 'save-cancel'; if (key.ctrl === true && input === 'c') return 'quit'; return null; } - if (search.editing) { - if (key.escape) return 'search-cancel'; - if (key.return) return 'search-submit'; - if (key.backspace || key.delete) return 'search-backspace'; - if (key.downArrow) return 'cursor-down'; - if (key.upArrow) return 'cursor-up'; - if (input.length === 1 && input >= ' ' && input !== '') return { type: 'search-input', value: input }; - return null; - } if (key.ctrl === true && input === 'c') return 'quit'; + if (key.ctrl === true && input === 'a') return 'toggle-select-all-visible'; + if (key.ctrl === true && input === 'n') return 'select-none'; + if (key.return) return 'save-request'; if (key.upArrow) return 'cursor-up'; if (key.downArrow) return 'cursor-down'; if (key.leftArrow) return 'cursor-left'; if (key.rightArrow) return 'cursor-right'; - if (key.return) return 'save-request'; - if (input === ' ') return 'toggle-check'; - if (input === '/') return 'search-start'; - if (input === 'a') return 'toggle-select-all-visible'; - if (input === 'n') return 'select-none'; - if (key.escape) return 'quit'; + if (key.tab) return 'toggle-check'; + if (input === ' ' && state.isNavigating) return 'toggle-check'; + if (key.backspace || key.delete) return 'search-backspace'; + if (key.escape) return state.search.query.length > 0 ? 'search-clear' : 'quit'; + if (input.length === 1 && input >= ' ' && input !== '') return { type: 'search-input', value: input }; return null; } @@ -220,14 +214,13 @@ export function TreePickerApp(props: TreePickerAppProps): ReactNode { const theme = useMemo(() => resolveTheme(props.env), [props.env]); const visibleIds = visibleNodeIds(state); const selectedIndex = Math.max(0, visibleIds.indexOf(state.cursorId)); - const reservedRows = state.pendingConfirm === 'save-confirm' ? 10 : 9; + const reservedRows = state.pendingConfirm === 'save-confirm' ? 11 : 10; const visibleRows = Math.max(5, Math.min(12, (props.terminalRows ?? 24) - reservedRows)); const rows = windowItems(visibleIds, selectedIndex, visibleRows); const hiddenAbove = rows.offset; const hiddenBelow = Math.max(0, visibleIds.length - rows.offset - rows.items.length); const searchMatchCount = filterTree(state).visibleIds.size; const width = resolveTreePickerWidth(props.terminalWidth); - const showSearch = state.search.editing || state.search.query.trim().length > 0; const helpText = props.chrome.helpText ?? DEFAULT_TREE_PICKER_HELP_TEXT; const skipEmptyMessage = props.chrome.skipEmptyMessage ?? DEFAULT_SKIP_EMPTY_MESSAGE; @@ -258,7 +251,7 @@ export function TreePickerApp(props: TreePickerAppProps): ReactNode { }, [state.transientHint?.expiresAt]); useInput((input, key) => { - const command = treePickerCommandForInkInput(input, key, stateRef.current.search, stateRef.current.pendingConfirm); + const command = treePickerCommandForInkInput(input, key, stateRef.current); if (!command) { return; } @@ -308,16 +301,18 @@ export function TreePickerApp(props: TreePickerAppProps): ReactNode { {warning} ))} - {showSearch ? ( - - / + + Search: + {state.isNavigating ? ( + {state.search.query || '(type to filter)'} + ) : ( {state.search.query} - {state.search.editing ? '█' : ''} + - ({searchMatchCount} matches) - - ) : null} + )} + ({searchMatchCount} match{searchMatchCount === 1 ? '' : 'es'}) + {hiddenAbove > 0 ? ↑ {hiddenAbove} more : null} {rows.items.map((nodeId) => ( diff --git a/packages/cli/src/update-check/cache.ts b/packages/cli/src/update-check/cache.ts new file mode 100644 index 00000000..19ebf07a --- /dev/null +++ b/packages/cli/src/update-check/cache.ts @@ -0,0 +1,45 @@ +import { renameSync, writeFileSync } from 'node:fs'; +import { mkdir, readFile } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { z } from 'zod'; + +const updateCheckCacheSchema = z + .object({ + checkedAt: z.string(), + channel: z.enum(['latest', 'next']), + installedVersion: z.string(), + latestForChannel: z.string(), + lastNoticeAt: z.string().optional(), + }) + .strict(); + +export type UpdateCheckCache = z.infer; + +/** @internal */ +export function updateCheckCachePath(homeDir = homedir()): string { + return join(homeDir, '.ktx', 'update-check.json'); +} + +export async function readUpdateCheckCache(options: { homeDir?: string } = {}): Promise { + try { + return updateCheckCacheSchema.parse(JSON.parse(await readFile(updateCheckCachePath(options.homeDir), 'utf-8'))); + } catch { + return null; + } +} + +export async function writeUpdateCheckCache( + value: UpdateCheckCache, + options: { homeDir?: string } = {}, +): Promise { + try { + const path = updateCheckCachePath(options.homeDir); + await mkdir(dirname(path), { recursive: true }); + const tempPath = `${path}.${process.pid}.${Date.now()}.tmp`; + writeFileSync(tempPath, `${JSON.stringify(value, null, 2)}\n`, 'utf-8'); + renameSync(tempPath, path); + } catch { + return; + } +} diff --git a/packages/cli/src/update-check/channel.ts b/packages/cli/src/update-check/channel.ts new file mode 100644 index 00000000..d8251021 --- /dev/null +++ b/packages/cli/src/update-check/channel.ts @@ -0,0 +1,43 @@ +import semver from 'semver'; + +export type UpdateChannel = 'latest' | 'next'; + +export type UpdateDecision = + | { status: 'skip' } + | { status: 'upToDate'; channel: UpdateChannel; target: string } + | { status: 'available'; channel: UpdateChannel; target: string }; + +/** @internal */ +export function inferUpdateChannel(installed: string): UpdateChannel | null { + const parsed = semver.parse(installed); + if (!parsed || installed === '0.0.0') { + return null; + } + + const [prereleaseId] = parsed.prerelease; + if (prereleaseId === undefined) { + return 'latest'; + } + if (prereleaseId === 'rc') { + return 'next'; + } + return null; +} + +export function decideUpdate(installed: string, distTags: Record): UpdateDecision { + const channel = inferUpdateChannel(installed); + if (!channel || !semver.valid(installed)) { + return { status: 'skip' }; + } + + const target = distTags[channel]; + if (!target || !semver.valid(target)) { + return { status: 'skip' }; + } + + if (semver.gt(target, installed)) { + return { status: 'available', channel, target }; + } + + return { status: 'upToDate', channel, target }; +} diff --git a/packages/cli/src/update-check/registry.ts b/packages/cli/src/update-check/registry.ts new file mode 100644 index 00000000..f0934933 --- /dev/null +++ b/packages/cli/src/update-check/registry.ts @@ -0,0 +1,52 @@ +import { request as httpsRequest } from 'node:https'; +import { URL } from 'node:url'; +import { z } from 'zod'; + +const DIST_TAGS_URL = new URL('https://registry.npmjs.org/-/package/@kaelio/ktx/dist-tags'); +const distTagsSchema = z.record(z.string(), z.string()); + +function parseDistTags(raw: string): Record { + return distTagsSchema.parse(JSON.parse(raw)); +} + +export function fetchDistTags(): Promise> { + return new Promise((resolve, reject) => { + const request = httpsRequest( + DIST_TAGS_URL, + { + method: 'GET', + headers: { + accept: 'application/json', + }, + }, + (response) => { + const chunks: Buffer[] = []; + response.on('data', (chunk: Buffer | string) => { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + }); + response.on('end', () => { + const text = Buffer.concat(chunks).toString('utf8'); + const statusCode = response.statusCode ?? 0; + if (statusCode < 200 || statusCode >= 300) { + reject(new Error(`npm dist-tags request failed with ${statusCode}: ${text}`)); + return; + } + try { + resolve(parseDistTags(text)); + } catch (error) { + reject(error); + } + }); + }, + ); + + request.on('socket', (socket) => { + socket.unref(); + }); + request.on('error', reject); + request.setTimeout(5000, () => { + request.destroy(new Error('npm dist-tags request timed out')); + }); + request.end(); + }); +} diff --git a/packages/cli/src/update-check/update-check.ts b/packages/cli/src/update-check/update-check.ts new file mode 100644 index 00000000..611a43a3 --- /dev/null +++ b/packages/cli/src/update-check/update-check.ts @@ -0,0 +1,187 @@ +import type { KtxCliIo } from '../cli-runtime.js'; +import { cyan, dim, type CliStyleEnv } from '../clack.js'; +import { resolveOutputMode } from '../io/mode.js'; +import { type UpdateCheckCache, readUpdateCheckCache, writeUpdateCheckCache } from './cache.js'; +import { decideUpdate, inferUpdateChannel, type UpdateChannel } from './channel.js'; +import { fetchDistTags as defaultFetchDistTags } from './registry.js'; + +const DAY_MS = 24 * 60 * 60 * 1000; + +/** @internal */ +export interface UpdateCheckEnv extends NodeJS.ProcessEnv, CliStyleEnv { + CI?: string; + DO_NOT_TRACK?: string; + KTX_NO_UPDATE_CHECK?: string; + KTX_OUTPUT?: string; + NO_UPDATE_NOTIFIER?: string; +} + +/** @internal */ +export interface UpdateCheckCommandOptions { + format?: unknown; + json?: unknown; + output?: unknown; +} + +export interface PrepareUpdateCheckNoticeOptions { + commandOptions?: UpdateCheckCommandOptions; + env?: UpdateCheckEnv; + fetchDistTags?: () => Promise>; + homeDir?: string; + installedVersion: string; + io: KtxCliIo; + now?: () => Date; +} + +export interface PreparedUpdateCheckNotice { + notice: string | null; +} + +function truthy(value: string | undefined): boolean { + return value !== undefined && value !== '' && value !== '0' && value !== 'false'; +} + +function commandRequestsJson(options: UpdateCheckCommandOptions | undefined): boolean { + return options?.json === true || options?.output === 'json' || options?.format === 'json'; +} + +/** @internal */ +export function shouldSuppressUpdateCheck(args: { + commandOptions?: UpdateCheckCommandOptions; + env?: UpdateCheckEnv; + io: KtxCliIo; +}): boolean { + const env = args.env ?? process.env; + if (truthy(env.KTX_NO_UPDATE_CHECK) || truthy(env.NO_UPDATE_NOTIFIER) || truthy(env.DO_NOT_TRACK)) { + return true; + } + + if (commandRequestsJson(args.commandOptions) || truthy(env.CI) || args.io.stdout.isTTY !== true) { + return true; + } + + try { + const mode = resolveOutputMode({ + json: false, + io: args.io, + env, + }); + return mode !== 'pretty'; + } catch { + return true; + } +} + +/** @internal */ +export function renderUpdateNotice(args: { + channel: UpdateChannel; + env?: CliStyleEnv; + installedVersion: string; + targetVersion: string; +}): string { + const command = args.channel === 'next' ? 'npm i -g @kaelio/ktx@next' : 'npm i -g @kaelio/ktx'; + return `${cyan('↑', args.env)} Update available: ktx ${args.installedVersion} → ${args.targetVersion}\n ${dim(command, args.env)}\n`; +} + +function timestampMs(value: string | undefined): number | null { + if (!value) { + return null; + } + const parsed = Date.parse(value); + return Number.isNaN(parsed) ? null : parsed; +} + +function elapsedAtLeast(value: string | undefined, now: Date, intervalMs: number): boolean { + const previous = timestampMs(value); + if (previous === null) { + return true; + } + return now.getTime() - previous >= intervalMs; +} + +function shouldRefreshCache(cache: UpdateCheckCache | null, installedVersion: string, now: Date): boolean { + if (!cache || cache.installedVersion !== installedVersion) { + return true; + } + return elapsedAtLeast(cache.checkedAt, now, DAY_MS); +} + +async function refreshUpdateCache(args: { + cache: UpdateCheckCache | null; + fetchDistTags: () => Promise>; + homeDir?: string; + installedVersion: string; + now: Date; +}): Promise { + const distTags = await args.fetchDistTags(); + const decision = decideUpdate(args.installedVersion, distTags); + if (decision.status === 'skip') { + return; + } + + await writeUpdateCheckCache( + { + checkedAt: args.now.toISOString(), + channel: decision.channel, + installedVersion: args.installedVersion, + latestForChannel: decision.target, + ...(args.cache?.installedVersion === args.installedVersion && args.cache.channel === decision.channel + ? { lastNoticeAt: args.cache.lastNoticeAt } + : {}), + }, + { homeDir: args.homeDir }, + ); +} + +export async function prepareUpdateCheckNotice( + options: PrepareUpdateCheckNoticeOptions, +): Promise { + const env = options.env ?? process.env; + const now = (options.now ?? (() => new Date()))(); + const fetchDistTags = options.fetchDistTags ?? defaultFetchDistTags; + + if ( + shouldSuppressUpdateCheck({ + commandOptions: options.commandOptions, + env, + io: options.io, + }) + ) { + return { notice: null }; + } + + if (!inferUpdateChannel(options.installedVersion)) { + return { notice: null }; + } + + let cache = await readUpdateCheckCache({ homeDir: options.homeDir }); + let notice: string | null = null; + + if (cache?.installedVersion === options.installedVersion) { + const decision = decideUpdate(options.installedVersion, { + [cache.channel]: cache.latestForChannel, + }); + if (decision.status === 'available' && elapsedAtLeast(cache.lastNoticeAt, now, DAY_MS)) { + notice = renderUpdateNotice({ + channel: decision.channel, + env, + installedVersion: options.installedVersion, + targetVersion: decision.target, + }); + cache = { ...cache, lastNoticeAt: now.toISOString() }; + await writeUpdateCheckCache(cache, { homeDir: options.homeDir }); + } + } + + if (shouldRefreshCache(cache, options.installedVersion, now)) { + void refreshUpdateCache({ + cache, + fetchDistTags, + homeDir: options.homeDir, + installedVersion: options.installedVersion, + now, + }).catch(() => {}); + } + + return { notice }; +} diff --git a/packages/cli/src/admin-reindex.test.ts b/packages/cli/test/admin-reindex.test.ts similarity index 96% rename from packages/cli/src/admin-reindex.test.ts rename to packages/cli/test/admin-reindex.test.ts index dace420f..bdfc5a09 100644 --- a/packages/cli/src/admin-reindex.test.ts +++ b/packages/cli/test/admin-reindex.test.ts @@ -1,9 +1,9 @@ import { createRequire } from 'node:module'; -import type { ReindexSummary } from './context/index-sync/types.js'; +import type { ReindexSummary } from '../src/context/index-sync/types.js'; import { describe, expect, it, vi } from 'vitest'; -import { renderReindexJson, renderReindexPlain, reindexHasErrors } from './admin-reindex.js'; -import { runKtxCli } from './index.js'; +import { renderReindexJson, renderReindexPlain, reindexHasErrors } from '../src/admin-reindex.js'; +import { runKtxCli } from '../src/index.js'; const cliVersion = (createRequire(import.meta.url)('@kaelio/ktx/package.json') as { version: string }) .version; diff --git a/packages/cli/src/admin.test.ts b/packages/cli/test/admin.test.ts similarity index 99% rename from packages/cli/src/admin.test.ts rename to packages/cli/test/admin.test.ts index 15f4179e..4f425e14 100644 --- a/packages/cli/src/admin.test.ts +++ b/packages/cli/test/admin.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { runKtxCli } from './index.js'; +import { runKtxCli } from '../src/index.js'; function makeIo() { let stdout = ''; diff --git a/packages/cli/src/cli-program-telemetry.test.ts b/packages/cli/test/cli-program-telemetry.test.ts similarity index 79% rename from packages/cli/src/cli-program-telemetry.test.ts rename to packages/cli/test/cli-program-telemetry.test.ts index db905442..30e2bd2b 100644 --- a/packages/cli/src/cli-program-telemetry.test.ts +++ b/packages/cli/test/cli-program-telemetry.test.ts @@ -3,8 +3,15 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { runCommanderKtxCli } from './cli-program.js'; -import type { KtxCliDeps, KtxCliIo, KtxCliPackageInfo } from './cli-runtime.js'; +import { runCommanderKtxCli } from '../src/cli-program.js'; +import type { KtxCliDeps, KtxCliIo, KtxCliPackageInfo } from '../src/cli-runtime.js'; +import { TELEMETRY_NOTICE } from '../src/telemetry/identity.js'; + +const reportExceptionMock = vi.hoisted(() => vi.fn(async () => {})); + +vi.mock('../src/telemetry/exception.js', () => ({ + reportException: reportExceptionMock, +})); function makeIo(stdoutIsTTY = true): { io: KtxCliIo; stdout: () => string; stderr: () => string } { let stdout = ''; @@ -42,6 +49,7 @@ describe('runCommanderKtxCli telemetry', () => { vi.stubEnv('CI', ''); vi.stubEnv('KTX_TELEMETRY_DISABLED', ''); vi.stubEnv('DO_NOT_TRACK', ''); + reportExceptionMock.mockClear(); }); afterEach(async () => { @@ -85,7 +93,7 @@ describe('runCommanderKtxCli telemetry', () => { expect(statusIo.stderr()).toContain('"connectionCount"'); expect(statusIo.stderr()).not.toContain(tempDir); - const noticeIndex = statusIo.stderr().indexOf('ktx collects anonymous usage data'); + const noticeIndex = statusIo.stderr().indexOf(TELEMETRY_NOTICE); const firstTelemetryIndex = statusIo.stderr().indexOf('[telemetry]'); expect(noticeIndex).toBeGreaterThanOrEqual(0); expect(firstTelemetryIndex).toBeGreaterThan(noticeIndex); @@ -130,4 +138,30 @@ describe('runCommanderKtxCli telemetry', () => { await expect(runCommanderKtxCli(['unknown'], unknownIo.io, {}, info, { runInit: async () => 0 })).resolves.toBe(1); expect(unknownIo.stderr()).not.toContain('[telemetry]'); }); + + it('reports genuine top-level command catches as handled exceptions', async () => { + const io = makeIo(true); + const deps: KtxCliDeps = { + doctor: async () => { + throw new Error('status failed'); + }, + }; + + await expect( + runCommanderKtxCli( + ['--project-dir', tempDir, 'status', '--json'], + io.io, + deps, + info, + { runInit: async () => 0 }, + ), + ).resolves.toBe(1); + + expect(reportExceptionMock).toHaveBeenCalledWith( + expect.objectContaining({ + context: expect.objectContaining({ source: 'ktx status', handled: true, fatal: false }), + projectDir: tempDir, + }), + ); + }); }); diff --git a/packages/cli/src/cli-program.test.ts b/packages/cli/test/cli-program.test.ts similarity index 93% rename from packages/cli/src/cli-program.test.ts rename to packages/cli/test/cli-program.test.ts index 009dfb8a..332645aa 100644 --- a/packages/cli/src/cli-program.test.ts +++ b/packages/cli/test/cli-program.test.ts @@ -1,7 +1,7 @@ import { Command, type CommandUnknownOpts } from '@commander-js/extra-typings'; import { describe, expect, it } from 'vitest'; -import { buildKtxProgram, collectCommandFlagsPresent } from './cli-program.js'; -import type { KtxCliIo, KtxCliPackageInfo } from './cli-runtime.js'; +import { buildKtxProgram, collectCommandFlagsPresent } from '../src/cli-program.js'; +import type { KtxCliIo, KtxCliPackageInfo } from '../src/cli-runtime.js'; function stubIo(): KtxCliIo { return { diff --git a/packages/cli/src/command-tree.test.ts b/packages/cli/test/command-tree.test.ts similarity index 98% rename from packages/cli/src/command-tree.test.ts rename to packages/cli/test/command-tree.test.ts index 181fac77..2a9d4f87 100644 --- a/packages/cli/src/command-tree.test.ts +++ b/packages/cli/test/command-tree.test.ts @@ -1,6 +1,6 @@ import { Command } from '@commander-js/extra-typings'; import { describe, expect, it } from 'vitest'; -import { formatCommandTree, walkCommandTree } from './command-tree.js'; +import { formatCommandTree, walkCommandTree } from '../src/command-tree.js'; describe('walkCommandTree', () => { it('captures name, description, aliases, and nested children', () => { diff --git a/packages/cli/src/commands/mcp-commands.test.ts b/packages/cli/test/commands/mcp-commands.test.ts similarity index 97% rename from packages/cli/src/commands/mcp-commands.test.ts rename to packages/cli/test/commands/mcp-commands.test.ts index dfcd1946..cf9c0cd5 100644 --- a/packages/cli/src/commands/mcp-commands.test.ts +++ b/packages/cli/test/commands/mcp-commands.test.ts @@ -1,7 +1,7 @@ import { Command } from '@commander-js/extra-typings'; import { describe, expect, it, vi } from 'vitest'; -import type { KtxCliCommandContext } from '../cli-program.js'; -import { registerMcpCommands } from './mcp-commands.js'; +import type { KtxCliCommandContext } from '../../src/cli-program.js'; +import { registerMcpCommands } from '../../src/commands/mcp-commands.js'; function makeContext(overrides: Partial = {}): KtxCliCommandContext { let exitCode = 0; diff --git a/packages/cli/src/commands/sql-commands.test.ts b/packages/cli/test/commands/sql-commands.test.ts similarity index 95% rename from packages/cli/src/commands/sql-commands.test.ts rename to packages/cli/test/commands/sql-commands.test.ts index 4f2c0277..9db7e8bf 100644 --- a/packages/cli/src/commands/sql-commands.test.ts +++ b/packages/cli/test/commands/sql-commands.test.ts @@ -1,7 +1,7 @@ import { Command } from '@commander-js/extra-typings'; import { describe, expect, it, vi } from 'vitest'; -import type { KtxCliCommandContext } from '../cli-program.js'; -import { registerSqlCommands } from './sql-commands.js'; +import type { KtxCliCommandContext } from '../../src/cli-program.js'; +import { registerSqlCommands } from '../../src/commands/sql-commands.js'; function makeContext(overrides: Partial = {}): KtxCliCommandContext { let exitCode = 0; diff --git a/packages/cli/test/commands/wiki-sl-read-commands.test.ts b/packages/cli/test/commands/wiki-sl-read-commands.test.ts new file mode 100644 index 00000000..69e3c51a --- /dev/null +++ b/packages/cli/test/commands/wiki-sl-read-commands.test.ts @@ -0,0 +1,157 @@ +import { Command } from '@commander-js/extra-typings'; +import { describe, expect, it, vi } from 'vitest'; +import type { KtxCliCommandContext } from '../../src/cli-program.js'; +import { registerWikiCommands } from '../../src/commands/knowledge-commands.js'; +import { registerSlCommands } from '../../src/commands/sl-commands.js'; + +function makeContext(overrides: Partial = {}): KtxCliCommandContext { + let exitCode = 0; + return { + io: { + stdout: { write: vi.fn() }, + stderr: { write: vi.fn() }, + }, + deps: {}, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + setExitCode: (code) => { + exitCode = code; + }, + runInit: vi.fn(), + writeDebug: vi.fn(), + ...overrides, + get exitCode() { + return exitCode; + }, + } as KtxCliCommandContext; +} + +describe('wiki and sl read command routing', () => { + it('routes wiki read through the knowledge runner', async () => { + const program = new Command().exitOverride().option('--project-dir '); + const knowledge = vi.fn(async () => 0); + const context = makeContext({ deps: { knowledge } }); + registerWikiCommands(program, context); + + await expect( + program.parseAsync(['--project-dir', '/tmp/ktx-project', 'wiki', 'read', 'metrics-revenue'], { + from: 'user', + }), + ).resolves.toBe(program); + + expect(knowledge).toHaveBeenCalledWith( + { + command: 'read', + projectDir: '/tmp/ktx-project', + key: 'metrics-revenue', + userId: 'local', + }, + context.io, + ); + }); + + it('routes wiki read with the parent --user-id option', async () => { + const program = new Command().exitOverride().option('--project-dir '); + const knowledge = vi.fn(async () => 0); + const context = makeContext({ deps: { knowledge } }); + registerWikiCommands(program, context); + + await expect( + program.parseAsync( + ['--project-dir', '/tmp/ktx-project', 'wiki', '--user-id', 'alex', 'read', 'handoff'], + { from: 'user' }, + ), + ).resolves.toBe(program); + + expect(knowledge).toHaveBeenCalledWith( + { + command: 'read', + projectDir: '/tmp/ktx-project', + key: 'handoff', + userId: 'alex', + }, + context.io, + ); + }); + + it('routes sl read through the semantic-layer runner', async () => { + const program = new Command().exitOverride().option('--project-dir '); + const sl = vi.fn(async () => 0); + const context = makeContext({ deps: { sl } }); + registerSlCommands(program, context); + + await expect( + program.parseAsync( + ['--project-dir', '/tmp/ktx-project', 'sl', '--connection-id', 'warehouse', 'read', 'orders'], + { from: 'user' }, + ), + ).resolves.toBe(program); + + expect(sl).toHaveBeenCalledWith( + { + command: 'read', + projectDir: '/tmp/ktx-project', + connectionId: 'warehouse', + sourceName: 'orders', + }, + context.io, + ); + }); + + it('routes sl read without --connection-id through the semantic-layer runner', async () => { + const program = new Command().exitOverride().option('--project-dir '); + const sl = vi.fn(async () => 0); + const context = makeContext({ deps: { sl } }); + registerSlCommands(program, context); + + await expect( + program.parseAsync(['--project-dir', '/tmp/ktx-project', 'sl', 'read', 'orders'], { from: 'user' }), + ).resolves.toBe(program); + + expect(sl).toHaveBeenCalledWith( + { + command: 'read', + projectDir: '/tmp/ktx-project', + connectionId: undefined, + sourceName: 'orders', + }, + context.io, + ); + }); + + it('routes sl validate without --connection-id through the semantic-layer runner', async () => { + const program = new Command().exitOverride().option('--project-dir '); + const sl = vi.fn(async () => 0); + const context = makeContext({ deps: { sl } }); + registerSlCommands(program, context); + + await expect( + program.parseAsync(['--project-dir', '/tmp/ktx-project', 'sl', 'validate', 'orders'], { from: 'user' }), + ).resolves.toBe(program); + + expect(sl).toHaveBeenCalledWith( + { + command: 'validate', + projectDir: '/tmp/ktx-project', + connectionId: undefined, + sourceName: 'orders', + }, + context.io, + ); + }); + + it('keeps sl query requiring --connection-id before invoking the runner', async () => { + const program = new Command().exitOverride().option('--project-dir '); + const sl = vi.fn(async () => 0); + const context = makeContext({ deps: { sl } }); + registerSlCommands(program, context); + + await expect( + program.parseAsync( + ['--project-dir', '/tmp/ktx-project', 'sl', 'query', '--measure', 'orders.count'], + { from: 'user' }, + ), + ).rejects.toThrow("error: required option '--connection-id ' not specified"); + + expect(sl).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli/test/completion/complete-engine.test.ts b/packages/cli/test/completion/complete-engine.test.ts new file mode 100644 index 00000000..f3893340 --- /dev/null +++ b/packages/cli/test/completion/complete-engine.test.ts @@ -0,0 +1,137 @@ +import type { Command } from '@commander-js/extra-typings'; +import { describe, expect, it } from 'vitest'; +import { buildKtxProgram } from '../../src/cli-program.js'; +import type { KtxCliIo, KtxCliPackageInfo } from '../../src/cli-runtime.js'; +import { type CompletionProviders, computeCompletions } from '../../src/completion/complete-engine.js'; + +function stubIo(): KtxCliIo { + return { stdout: { isTTY: false, columns: 80, write: () => {} }, stderr: { write: () => {} } }; +} + +function stubPackageInfo(): KtxCliPackageInfo { + return { name: '@kaelio/ktx', version: '0.0.0-test' }; +} + +function buildProgram(): Command { + return buildKtxProgram({ io: stubIo(), deps: {}, packageInfo: stubPackageInfo(), runInit: async () => 0 }); +} + +const SOURCES = ['orders', 'customers']; +const WIKI_KEYS = ['revenue', 'churn']; +const CONNECTIONS = ['warehouse']; + +function fakeProviders(overrides: Partial = {}): CompletionProviders { + return { + async positionalCandidates(commandPath) { + const key = commandPath.join(' '); + if (key === 'sl read' || key === 'sl validate') { + return SOURCES; + } + if (key === 'wiki read') { + return WIKI_KEYS; + } + return []; + }, + async optionValueCandidates(_commandPath, optionFlag) { + return optionFlag === '--connection-id' ? CONNECTIONS : []; + }, + ...overrides, + }; +} + +function complete(words: string[], providers: CompletionProviders = fakeProviders()): Promise { + return computeCompletions(buildProgram(), words, providers); +} + +describe('computeCompletions', () => { + it('lists top-level commands and hides internal ones', async () => { + const result = await complete(['']); + expect(result).toContain('sl'); + expect(result).toContain('wiki'); + expect(result).toContain('completion'); + expect(result).not.toContain('__complete'); + }); + + it('filters top-level commands by prefix', async () => { + expect(await complete(['co'])).toEqual(['completion', 'connection']); + }); + + it('hides Commander-hidden subcommands such as `mcp serve-internal`', async () => { + const result = await complete(['mcp', '']); + expect(result).not.toContain('serve-internal'); + expect(result).toEqual(['logs', 'start', 'status', 'stdio', 'stop']); + }); + + it('offers only sl subcommands at the bare sl positional', async () => { + expect(await complete(['sl', ''])).toEqual(['query', 'read', 'validate']); + }); + + it('offers source names for sl read and sl validate', async () => { + expect(await complete(['sl', 'read', ''])).toEqual(['customers', 'orders']); + expect(await complete(['sl', 'validate', ''])).toEqual(['customers', 'orders']); + }); + + it('offers only the wiki read subcommand at the bare wiki positional', async () => { + expect(await complete(['wiki', ''])).toEqual(['read']); + }); + + it('offers wiki page keys for wiki read', async () => { + expect(await complete(['wiki', 'read', ''])).toEqual(['churn', 'revenue']); + }); + + it('does not complete entity names for bare search positionals', async () => { + expect(await complete(['sl', 'o'])).toEqual([]); + expect(await complete(['wiki', 'r'])).toEqual(['read']); + }); + + it('completes flags (own + inherited globals) when the partial starts with a dash', async () => { + const result = await complete(['sl', '-']); + expect(result).toContain('--connection-id'); + expect(result).toContain('--output'); + expect(result).toContain('--json'); + expect(result).toContain('--debug'); + expect(result).toContain('--project-dir'); + }); + + it('completes option choices for the `--opt value` form', async () => { + expect(await complete(['sl', '--output', ''])).toEqual(['json', 'plain', 'pretty']); + }); + + it('completes option choices for the `--opt=value` form', async () => { + expect(await complete(['sl', '--output=pr'])).toEqual(['--output=pretty']); + }); + + it('completes option values from a provider for options without static choices', async () => { + expect(await complete(['sl', '--connection-id', ''])).toEqual(['warehouse']); + }); + + it('falls through to positional completion after a boolean flag', async () => { + const result = await complete(['sl', '--json', '']); + expect(result).toEqual(['query', 'read', 'validate']); + }); + + it('does not treat a value-taking option value as a subcommand', async () => { + // A connection id that happens to match a subcommand name (`query`, `read`) + // is the `--connection-id` value, not a subcommand: the next positional must + // still offer the `sl` subcommands rather than resolving into `sl query`/`sl read`. + expect(await complete(['sl', '--connection-id', 'query', ''])).toEqual(['query', 'read', 'validate']); + expect(await complete(['sl', '--connection-id', 'read', ''])).toEqual(['query', 'read', 'validate']); + }); + + it('still returns subcommands/flags when dynamic providers yield nothing (no project)', async () => { + const empty = fakeProviders({ + positionalCandidates: async () => [], + optionValueCandidates: async () => [], + }); + expect(await complete(['sl', ''], empty)).toEqual(['query', 'read', 'validate']); + expect(await complete(['-'], empty)).toContain('--debug'); + }); + + it('completes the completion command shell positional from its static choices', async () => { + expect(await complete(['completion', ''])).toEqual(['bash', 'zsh']); + }); + + it('filters positional argument choices by prefix', async () => { + expect(await complete(['completion', 'z'])).toEqual(['zsh']); + }); +}); diff --git a/packages/cli/test/completion/completion-scripts.test.ts b/packages/cli/test/completion/completion-scripts.test.ts new file mode 100644 index 00000000..24723a95 --- /dev/null +++ b/packages/cli/test/completion/completion-scripts.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from 'vitest'; +import { completionScript } from '../../src/completion/completion-scripts.js'; + +describe('completionScript', () => { + it('emits a zsh script that registers _ktx and delegates to ktx __complete', () => { + const script = completionScript('zsh'); + expect(script).toContain('#compdef ktx'); + expect(script).toContain('compdef _ktx ktx'); + expect(script).toContain('ktx __complete --'); + expect(script).toContain('compadd -- $candidates'); + }); + + it('emits a bash script that registers _ktx and preserves newline-split candidates', () => { + const script = completionScript('bash'); + expect(script).toContain('complete -F _ktx ktx'); + expect(script).toContain('ktx __complete --'); + expect(script).toContain("local IFS=$'\\n'"); + expect(script).toContain('COMPREPLY=($(compgen -W "${out}" -- "$cur"))'); + }); +}); diff --git a/packages/cli/test/completion/dynamic-candidates.test.ts b/packages/cli/test/completion/dynamic-candidates.test.ts new file mode 100644 index 00000000..560f38f0 --- /dev/null +++ b/packages/cli/test/completion/dynamic-candidates.test.ts @@ -0,0 +1,103 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { createProjectCompletionProviders } from '../../src/completion/dynamic-candidates.js'; + +const KTX_YAML = ['connections:', ' warehouse:', ' driver: sqlite', ' analytics:', ' driver: sqlite', ''].join( + '\n', +); + +describe('createProjectCompletionProviders', () => { + let projectDir: string; + + beforeEach(async () => { + projectDir = await mkdtemp(join(tmpdir(), 'ktx-completion-')); + await writeFile(join(projectDir, 'ktx.yaml'), KTX_YAML, 'utf-8'); + }); + + afterEach(async () => { + await rm(projectDir, { recursive: true, force: true }); + }); + + async function seedProjectEntities(): Promise { + await mkdir(join(projectDir, 'semantic-layer', 'warehouse'), { recursive: true }); + await writeFile( + join(projectDir, 'semantic-layer', 'warehouse', 'orders.yaml'), + ['name: orders', 'table: public.orders', 'grain: [order_id]', 'columns: []', ''].join('\n'), + 'utf-8', + ); + await mkdir(join(projectDir, 'semantic-layer', 'analytics'), { recursive: true }); + await writeFile( + join(projectDir, 'semantic-layer', 'analytics', 'orders.yaml'), + ['name: orders', 'table: public.analytics_orders', 'grain: [order_id]', 'columns: []', ''].join('\n'), + 'utf-8', + ); + await writeFile( + join(projectDir, 'semantic-layer', 'analytics', 'tickets.yaml'), + ['name: tickets', 'table: public.tickets', 'grain: [ticket_id]', 'columns: []', ''].join('\n'), + 'utf-8', + ); + await mkdir(join(projectDir, 'wiki', 'global'), { recursive: true }); + await writeFile( + join(projectDir, 'wiki', 'global', 'revenue.md'), + ['---', 'summary: Revenue', 'tags: []', 'refs: []', 'sl_refs: []', '---', '', 'Revenue rules.', ''].join('\n'), + 'utf-8', + ); + } + + it('completes connection ids for the `connection test` positional', async () => { + const providers = createProjectCompletionProviders(); + const result = await providers.positionalCandidates(['connection', 'test'], ['--project-dir', projectDir]); + expect(result).toEqual(['analytics', 'warehouse']); + }); + + it('completes connection ids for the `ingest` positional', async () => { + const providers = createProjectCompletionProviders(); + const result = await providers.positionalCandidates(['ingest'], ['--project-dir', projectDir]); + expect(result).toEqual(['analytics', 'warehouse']); + }); + + it('completes entity names only for read and validate subcommands', async () => { + await seedProjectEntities(); + const providers = createProjectCompletionProviders(); + + await expect(providers.positionalCandidates(['sl'], ['--project-dir', projectDir])).resolves.toEqual([]); + await expect(providers.positionalCandidates(['sl', 'read'], ['--project-dir', projectDir])).resolves.toEqual([ + 'orders', + 'tickets', + ]); + await expect(providers.positionalCandidates(['sl', 'validate'], ['--project-dir', projectDir])).resolves.toEqual([ + 'orders', + 'tickets', + ]); + await expect( + providers.positionalCandidates(['sl', 'read'], ['--project-dir', projectDir, '--connection-id', 'warehouse']), + ).resolves.toEqual(['orders']); + await expect( + providers.positionalCandidates(['sl', 'validate'], ['--project-dir', projectDir, '--connection-id', 'analytics']), + ).resolves.toEqual(['orders', 'tickets']); + await expect(providers.positionalCandidates(['wiki'], ['--project-dir', projectDir])).resolves.toEqual([]); + await expect(providers.positionalCandidates(['wiki', 'read'], ['--project-dir', projectDir])).resolves.toEqual([ + 'revenue', + ]); + }); + + it('returns no positional candidates outside a project', async () => { + const providers = createProjectCompletionProviders(); + const result = await providers.positionalCandidates(['connection', 'test'], ['--project-dir', join(projectDir, 'nope')]); + expect(result).toEqual([]); + }); + + it('completes connection ids for the sql --connection option', async () => { + const providers = createProjectCompletionProviders(); + const result = await providers.optionValueCandidates(['sql'], '--connection', ['--project-dir', projectDir]); + expect(result).toEqual(['analytics', 'warehouse']); + }); + + it('still completes connection ids for the --connection-id option', async () => { + const providers = createProjectCompletionProviders(); + const result = await providers.optionValueCandidates(['ingest'], '--connection-id', ['--project-dir', projectDir]); + expect(result).toEqual(['analytics', 'warehouse']); + }); +}); diff --git a/packages/cli/test/connection-recovery.test.ts b/packages/cli/test/connection-recovery.test.ts new file mode 100644 index 00000000..b164c7e2 --- /dev/null +++ b/packages/cli/test/connection-recovery.test.ts @@ -0,0 +1,171 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + runConnectionSetupWithRecovery, + type ConfigureResult, + type RecoveryAction, + type ValidateResult, +} from '../src/connection-recovery.js'; + +function input(overrides: { + interactive?: boolean; + allowSkip?: boolean; + configure?: () => Promise; + validate?: () => Promise; + selectValues?: string[]; + extraActions?: RecoveryAction[]; +}) { + const selectValues = [...(overrides.selectValues ?? [])]; + const rollback = vi.fn(async () => {}); + const select = vi.fn(async () => selectValues.shift() ?? 'back'); + const validate = overrides.validate ?? vi.fn(async () => ({ status: 'ok' as const })); + return { + rollback, + select, + validate, + run: () => + runConnectionSetupWithRecovery({ + label: 'warehouse', + interactive: overrides.interactive ?? true, + allowSkip: overrides.allowSkip ?? true, + io: { + stdout: { write: vi.fn() }, + stderr: { write: vi.fn() }, + }, + prompts: { select }, + snapshot: vi.fn(async () => rollback), + configure: overrides.configure ?? vi.fn(async () => 'configured' as const), + validate, + }), + }; +} + +describe('runConnectionSetupWithRecovery', () => { + it('returns ready without opening the menu when first validation passes', async () => { + const setup = input({}); + + await expect(setup.run()).resolves.toBe('ready'); + + expect(setup.select).not.toHaveBeenCalled(); + expect(setup.rollback).not.toHaveBeenCalled(); + }); + + it('fails fast without prompting or rollback when noninteractive validation fails', async () => { + const setup = input({ + interactive: false, + validate: vi.fn(async () => ({ status: 'failed' as const })), + }); + + await expect(setup.run()).resolves.toBe('failed'); + + expect(setup.select).not.toHaveBeenCalled(); + expect(setup.rollback).not.toHaveBeenCalled(); + }); + + it('retries the same config after Retry and returns ready', async () => { + let calls = 0; + const setup = input({ + selectValues: ['retry'], + validate: vi.fn(async () => { + calls += 1; + return calls === 1 ? { status: 'failed' as const } : { status: 'ok' as const }; + }), + }); + + await expect(setup.run()).resolves.toBe('ready'); + + expect(setup.validate).toHaveBeenCalledTimes(2); + expect(setup.rollback).not.toHaveBeenCalled(); + }); + + it('re-enters config and validates the new attempt', async () => { + let calls = 0; + const configure = vi.fn(async () => 'configured' as const); + const setup = input({ + configure, + selectValues: ['re-enter'], + validate: vi.fn(async () => { + calls += 1; + return calls === 1 ? { status: 'failed' as const } : { status: 'ok' as const }; + }), + }); + + await expect(setup.run()).resolves.toBe('ready'); + + expect(configure).toHaveBeenCalledTimes(2); + expect(setup.validate).toHaveBeenCalledTimes(2); + expect(setup.rollback).not.toHaveBeenCalled(); + }); + + it('rolls back once and returns skip when Skip is selected', async () => { + const setup = input({ + selectValues: ['skip'], + validate: vi.fn(async () => ({ status: 'failed' as const })), + }); + + await expect(setup.run()).resolves.toBe('skip'); + + expect(setup.rollback).toHaveBeenCalledTimes(1); + }); + + it('omits Skip when allowSkip is false and rolls back on Back', async () => { + const setup = input({ + allowSkip: false, + selectValues: ['back'], + validate: vi.fn(async () => ({ status: 'failed' as const })), + }); + + await expect(setup.run()).resolves.toBe('back'); + + expect(setup.select).toHaveBeenCalledWith({ + message: 'Connection setup failed for warehouse', + options: [ + { value: 'retry', label: 'Retry connection test' }, + { value: 're-enter', label: 'Re-enter connection details' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(setup.rollback).toHaveBeenCalledTimes(1); + }); + + it('runs an extra action and then revalidates', async () => { + const action = vi.fn(async () => {}); + let calls = 0; + const setup = input({ + selectValues: ['disable-query-history'], + validate: vi.fn(async () => { + calls += 1; + return calls === 1 + ? { + status: 'failed' as const, + extraActions: [ + { value: 'disable-query-history', label: 'Disable query history and retry', run: action }, + ], + } + : { status: 'ok' as const }; + }), + }); + + await expect(setup.run()).resolves.toBe('ready'); + + expect(action).toHaveBeenCalledTimes(1); + expect(setup.validate).toHaveBeenCalledTimes(2); + }); + + it('rolls back when re-enter returns back or cancelled', async () => { + const backSetup = input({ + selectValues: ['re-enter'], + configure: vi.fn(async () => 'back' as const), + validate: vi.fn(async () => ({ status: 'failed' as const })), + }); + await expect(backSetup.run()).resolves.toBe('back'); + expect(backSetup.rollback).toHaveBeenCalledTimes(1); + + const cancelledSetup = input({ + selectValues: ['re-enter'], + configure: vi.fn(async () => 'cancelled' as const), + validate: vi.fn(async () => ({ status: 'failed' as const })), + }); + await expect(cancelledSetup.run()).resolves.toBe('failed'); + expect(cancelledSetup.rollback).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/cli/src/connection.test.ts b/packages/cli/test/connection.test.ts similarity index 85% rename from packages/cli/src/connection.test.ts rename to packages/cli/test/connection.test.ts index 7cfc5b93..22c8bbe9 100644 --- a/packages/cli/src/connection.test.ts +++ b/packages/cli/test/connection.test.ts @@ -1,14 +1,20 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import type { LookerClient } from './context/ingest/adapters/looker/client.js'; -import type { MetabaseRuntimeClient } from './context/ingest/adapters/metabase/client-port.js'; -import type { NotionClient } from './context/ingest/adapters/notion/notion-client.js'; -import { initKtxProject } from './context/project/project.js'; -import { parseKtxProjectConfig, serializeKtxProjectConfig } from './context/project/config.js'; -import type { KtxConnectionDriver, KtxScanConnector } from './context/scan/types.js'; +import type { LookerClient } from '../src/context/ingest/adapters/looker/client.js'; +import type { MetabaseRuntimeClient } from '../src/context/ingest/adapters/metabase/client-port.js'; +import type { NotionClient } from '../src/context/ingest/adapters/notion/notion-client.js'; +import { initKtxProject } from '../src/context/project/project.js'; +import { parseKtxProjectConfig, serializeKtxProjectConfig } from '../src/context/project/config.js'; +import type { KtxConnectionDriver, KtxScanConnector } from '../src/context/scan/types.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { runKtxConnection } from './connection.js'; +import { runKtxConnection } from '../src/connection.js'; + +const reportExceptionMock = vi.hoisted(() => vi.fn(async () => {})); + +vi.mock('../src/telemetry/exception.js', () => ({ + reportException: reportExceptionMock, +})); function stripAnsi(s: string): string { return s.replace(/\[[0-9;]*m/g, ''); @@ -38,7 +44,7 @@ function makeIo() { function nativeConnector( driver: KtxConnectionDriver, - testResult: { success: true } | { success: false; error: string } = { success: true }, + testResult: { success: true } | { success: false; error: string; cause?: unknown } = { success: true }, ) { const testConnection = vi.fn(async () => testResult); const cleanup = vi.fn(async () => undefined); @@ -59,6 +65,8 @@ function nativeConnector( introspect: vi.fn(async () => { throw new Error('introspect should not be called from connection test'); }), + listSchemas: vi.fn(async () => []), + listTables: vi.fn(async () => []), testConnection, cleanup, }; @@ -70,6 +78,7 @@ describe('runKtxConnection', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-connection-')); + reportExceptionMock.mockClear(); }); afterEach(async () => { @@ -160,6 +169,66 @@ describe('runKtxConnection', () => { expect(io.stderr()).not.toContain(projectDir); }); + it('records the raw errorDetail in connection_test telemetry when a native test fails', async () => { + vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); + vi.stubEnv('CI', ''); + vi.stubEnv('DATABASE_URL', 'postgres://svc:db-url-password@db.example.test/analytics'); // pragma: allowlist secret + const projectDir = join(tempDir, 'project'); + await initKtxProject({ projectDir }); + await writeConnections(projectDir, { + warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' }, + }); + const { connector } = nativeConnector('postgres', { success: false, error: 'database file is unreadable' }); + const io = makeIo(); + + const code = await runKtxConnection({ command: 'test', projectDir, connectionId: 'warehouse' }, io.io, { + createScanConnector: vi.fn(async () => connector), + }); + + expect(code).toBe(1); + expect(io.stderr()).toContain('"event":"connection_test"'); + expect(io.stderr()).toContain('"outcome":"error"'); + expect(io.stderr()).toContain('"errorDetail":"database file is unreadable"'); + expect(reportExceptionMock).toHaveBeenCalledWith( + expect.objectContaining({ + context: expect.objectContaining({ source: 'connection test', handled: true, fatal: false }), + projectDir, + redactionSecrets: expect.arrayContaining([ + 'postgres://svc:db-url-password@db.example.test/analytics', // pragma: allowlist secret + 'db-url-password', + ]), + }), + ); + }); + + it('preserves the driver error class and code in connection_test telemetry', async () => { + vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); + vi.stubEnv('CI', ''); + const projectDir = join(tempDir, 'project'); + await initKtxProject({ projectDir }); + await writeConnections(projectDir, { + warehouse: { driver: 'sqlserver', host: 'db.example.test', database: 'analytics', username: 'svc_ro' }, + }); + class ConnectionError extends Error { + readonly code = 'ELOGIN'; + } + const driverError = new ConnectionError("Login failed for user 'svc_ro'."); + const { connector } = nativeConnector('sqlserver', { + success: false, + error: driverError.message, + cause: driverError, + }); + const io = makeIo(); + + const code = await runKtxConnection({ command: 'test', projectDir, connectionId: 'warehouse' }, io.io, { + createScanConnector: vi.fn(async () => connector), + }); + + expect(code).toBe(1); + expect(io.stderr()).toContain('"errorClass":"ConnectionError"'); + expect(io.stderr()).toContain('"errorDetail":"ELOGIN: Login failed for user \'svc_ro\'."'); + }); + it('reports the connector error and still cleans up when native testConnection fails', async () => { const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir }); diff --git a/packages/cli/src/connectors/bigquery/connector.test.ts b/packages/cli/test/connectors/bigquery/connector.test.ts similarity index 86% rename from packages/cli/src/connectors/bigquery/connector.test.ts rename to packages/cli/test/connectors/bigquery/connector.test.ts index be65af1e..11ad69d8 100644 --- a/packages/cli/src/connectors/bigquery/connector.test.ts +++ b/packages/cli/test/connectors/bigquery/connector.test.ts @@ -1,9 +1,9 @@ import { describe, expect, it, vi } from 'vitest'; -import { bigQueryConnectionConfigFromConfig, isKtxBigQueryConnectionConfig, type KtxBigQueryClient, KtxBigQueryScanConnector, type KtxBigQueryClientFactory, type KtxBigQueryDataset, type KtxBigQueryQueryJob, type KtxBigQueryTableRef } from '../../connectors/bigquery/connector.js'; -import { createBigQueryLiveDatabaseIntrospection } from '../../connectors/bigquery/live-database-introspection.js'; -import { tableRefSet } from '../../context/scan/table-ref.js'; +import { bigQueryConnectionConfigFromConfig, isKtxBigQueryConnectionConfig, type KtxBigQueryClient, KtxBigQueryScanConnector, type KtxBigQueryClientFactory, type KtxBigQueryDataset, type KtxBigQueryQueryJob, type KtxBigQueryTableRef, prepareBigQueryReadOnlyQuery } from '../../../src/connectors/bigquery/connector.js'; +import { createBigQueryLiveDatabaseIntrospection } from '../../../src/connectors/bigquery/live-database-introspection.js'; +import { tableRefSet } from '../../../src/context/scan/table-ref.js'; -function fakeClientFactory(): KtxBigQueryClientFactory { +function fakeClientFactory(options: { primaryKeyError?: Error } = {}): KtxBigQueryClientFactory { const queryResults = vi.fn(async (): ReturnType => [ [{ id: 1, status: 'paid' }], undefined, @@ -11,6 +11,9 @@ function fakeClientFactory(): KtxBigQueryClientFactory { ]); const createQueryJob = vi.fn(async (input: { query: string }): ReturnType => { if (input.query.includes('INFORMATION_SCHEMA.TABLE_CONSTRAINTS')) { + if (options.primaryKeyError) { + throw options.primaryKeyError; + } return [ { getQueryResults: async (): ReturnType => [ @@ -95,6 +98,17 @@ const connection = { } as const; describe('KtxBigQueryScanConnector', () => { + it('prepares read-only SQL parameters with BigQuery named placeholders', () => { + expect(prepareBigQueryReadOnlyQuery('SELECT * FROM orders WHERE id = :id AND id_2 = :id_2', { id: 1, id_2: 2 })).toEqual({ + sql: 'SELECT * FROM orders WHERE id = @id AND id_2 = @id_2', + params: { id: 1, id_2: 2 }, + }); + expect(prepareBigQueryReadOnlyQuery('SELECT * FROM orders')).toEqual({ + sql: 'SELECT * FROM orders', + params: undefined, + }); + }); + it('resolves configuration safely', () => { expect(isKtxBigQueryConnectionConfig(connection)).toBe(true); expect(isKtxBigQueryConnectionConfig({ driver: 'mysql' })).toBe(false); @@ -170,6 +184,34 @@ describe('KtxBigQueryScanConnector', () => { ]); }); + it.each([ + Object.assign(new Error('Access Denied'), { code: 403 }), + Object.assign(new Error('Not found'), { errors: [{ reason: 'notFound' }] }), + ])('soft-fails denied BigQuery primary-key discovery with a scan warning', async (primaryKeyError) => { + const connector = new KtxBigQueryScanConnector({ + connectionId: 'warehouse', + connection, + clientFactory: fakeClientFactory({ primaryKeyError }), + now: () => new Date('2026-04-29T17:00:00.000Z'), + }); + + const snapshot = await connector.introspect( + { connectionId: 'warehouse', driver: 'bigquery' }, + { runId: 'scan-run-bigquery-denied-pk' }, + ); + + expect(snapshot.warnings).toEqual([ + { + code: 'constraint_discovery_unauthorized', + message: 'Skipped primary-key discovery in analytics (insufficient grants on system catalogs)', + recoverable: true, + metadata: { schema: 'analytics', kind: 'primary_key' }, + }, + ]); + expect(snapshot.tables[0]?.foreignKeys).toEqual([]); + expect(snapshot.tables[0]?.columns.every((column) => column.primaryKey === false)).toBe(true); + }); + it('runs samples, read-only SQL, distinct values, dataset listing, row counts, and cleanup', async () => { const connector = new KtxBigQueryScanConnector({ connectionId: 'warehouse', @@ -225,7 +267,7 @@ describe('KtxBigQueryScanConnector', () => { ), ).resolves.toEqual({ values: ['open', 'paid'], cardinality: 2 }); await expect(connector.getTableRowCount('orders')).resolves.toBe(12); - await expect(connector.listDatasets()).resolves.toEqual(['analytics', 'staging']); + await expect(connector.listSchemas()).resolves.toEqual(['analytics', 'staging']); await expect( connector.columnStats( { connectionId: 'warehouse', table: { catalog: 'project-1', db: 'analytics', name: 'orders' }, column: 'status' }, @@ -335,9 +377,9 @@ describe('KtxBigQueryScanConnector', () => { }); await expect(connector.listTables(['analytics', 'mart'])).resolves.toEqual([ - { schema: 'analytics', name: 'orders', kind: 'table' }, - { schema: 'analytics', name: 'order_clone', kind: 'table' }, - { schema: 'mart', name: 'orders_mv', kind: 'view' }, + { catalog: 'project-1', schema: 'analytics', name: 'orders', kind: 'table' }, + { catalog: 'project-1', schema: 'analytics', name: 'order_clone', kind: 'table' }, + { catalog: 'project-1', schema: 'mart', name: 'orders_mv', kind: 'view' }, ]); expect(createQueryJob).toHaveBeenCalledTimes(1); diff --git a/packages/cli/src/connectors/bigquery/dialect.test.ts b/packages/cli/test/connectors/bigquery/dialect.test.ts similarity index 81% rename from packages/cli/src/connectors/bigquery/dialect.test.ts rename to packages/cli/test/connectors/bigquery/dialect.test.ts index d2033bd9..171617ce 100644 --- a/packages/cli/src/connectors/bigquery/dialect.test.ts +++ b/packages/cli/test/connectors/bigquery/dialect.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { KtxBigQueryDialect } from './dialect.js'; +import { KtxBigQueryDialect } from '../../../src/connectors/bigquery/dialect.js'; describe('KtxBigQueryDialect', () => { const dialect = new KtxBigQueryDialect(); @@ -38,14 +38,6 @@ describe('KtxBigQueryDialect', () => { ); }); - it('rewrites colon parameters to BigQuery named parameters', () => { - expect(dialect.prepareQuery('SELECT * FROM orders WHERE id = :id AND id_2 = :id_2', { id: 1, id_2: 2 })).toEqual({ - sql: 'SELECT * FROM orders WHERE id = @id AND id_2 = @id_2', - params: { id: 1, id_2: 2 }, - }); - expect(dialect.prepareQuery('SELECT * FROM orders')).toEqual({ sql: 'SELECT * FROM orders', params: undefined }); - }); - it('keeps unsupported statistics explicit', () => { expect(dialect.generateColumnStatisticsQuery('analytics', 'orders')).toBeNull(); }); diff --git a/packages/cli/src/connectors/clickhouse/connector.test.ts b/packages/cli/test/connectors/clickhouse/connector.test.ts similarity index 88% rename from packages/cli/src/connectors/clickhouse/connector.test.ts rename to packages/cli/test/connectors/clickhouse/connector.test.ts index abc7cad5..aba3143f 100644 --- a/packages/cli/src/connectors/clickhouse/connector.test.ts +++ b/packages/cli/test/connectors/clickhouse/connector.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import { clickHouseClientConfigFromConfig, isKtxClickHouseConnectionConfig, KtxClickHouseScanConnector, type KtxClickHouseClientFactory } from '../../connectors/clickhouse/connector.js'; -import { createClickHouseLiveDatabaseIntrospection } from '../../connectors/clickhouse/live-database-introspection.js'; -import { tableRefSet } from '../../context/scan/table-ref.js'; +import { clickHouseClientConfigFromConfig, isKtxClickHouseConnectionConfig, KtxClickHouseScanConnector, prepareClickHouseReadOnlyQuery, type KtxClickHouseClientFactory } from '../../../src/connectors/clickhouse/connector.js'; +import { createClickHouseLiveDatabaseIntrospection } from '../../../src/connectors/clickhouse/live-database-introspection.js'; +import { tableRefSet } from '../../../src/context/scan/table-ref.js'; function result(payload: T) { return { @@ -15,8 +15,8 @@ function fakeClientFactory(): KtxClickHouseClientFactory { const query = vi.fn(async (input: { query: string; format: string; query_params?: Record }) => { if (input.query.includes('FROM system.tables')) { return result([ - { name: 'events', engine: 'MergeTree', comment: 'Event stream' }, - { name: 'event_summary', engine: 'View', comment: '' }, + { database: 'analytics', name: 'event_summary', engine: 'View', comment: '' }, + { database: 'analytics', name: 'events', engine: 'MergeTree', comment: 'Event stream' }, ]); } if (input.query.includes('FROM system.columns')) { @@ -136,6 +136,33 @@ function multiDatabaseClickHouseClientFactory(): KtxClickHouseClientFactory { } describe('KtxClickHouseScanConnector', () => { + it('prepares read-only SQL parameters with ClickHouse typed placeholders', () => { + expect( + prepareClickHouseReadOnlyQuery('select * from events where id = :id and event_name = :name', { + id: 10, + name: 'signup', + }), + ).toEqual({ + sql: 'select * from events where id = {id:Int64} and event_name = {name:String}', + params: { id: 10, name: 'signup' }, + }); + expect( + prepareClickHouseReadOnlyQuery('select * from events where enabled = :enabled and ratio = :ratio and created_at = :created_at', { + enabled: true, + ratio: 1.5, + created_at: new Date('2026-05-25T00:00:00.000Z'), + }), + ).toEqual({ + sql: 'select * from events where enabled = {enabled:Bool} and ratio = {ratio:Float64} and created_at = {created_at:DateTime}', + params: { + enabled: true, + ratio: 1.5, + created_at: new Date('2026-05-25T00:00:00.000Z'), + }, + }); + expect(prepareClickHouseReadOnlyQuery('select 1')).toEqual({ sql: 'select 1', params: undefined }); + }); + it('resolves ClickHouse connection configuration safely', () => { expect(isKtxClickHouseConnectionConfig({ driver: 'clickhouse', host: 'localhost', database: 'analytics' })).toBe( true, @@ -196,8 +223,8 @@ describe('KtxClickHouseScanConnector', () => { }, }); expect(snapshot.tables.map((table) => [table.name, table.kind, table.estimatedRows, table.comment])).toEqual([ - ['events', 'table', 2, 'Event stream'], ['event_summary', 'view', null, null], + ['events', 'table', 2, 'Event stream'], ]); expect(snapshot.tables.find((table) => table.name === 'events')?.columns[0]).toMatchObject({ name: 'id', @@ -344,6 +371,10 @@ describe('KtxClickHouseScanConnector', () => { await expect(connector.getTableRowCount('events')).resolves.toBe(2); await expect(connector.listSchemas()).resolves.toEqual(['analytics', 'warehouse']); + await expect(connector.listTables(['analytics'])).resolves.toEqual([ + { catalog: null, schema: 'analytics', name: 'event_summary', kind: 'view' }, + { catalog: null, schema: 'analytics', name: 'events', kind: 'table' }, + ]); await expect( connector.columnStats( { connectionId: 'warehouse', table: { catalog: null, db: 'analytics', name: 'events' }, column: 'event_name' }, diff --git a/packages/cli/src/connectors/clickhouse/dialect.test.ts b/packages/cli/test/connectors/clickhouse/dialect.test.ts similarity index 75% rename from packages/cli/src/connectors/clickhouse/dialect.test.ts rename to packages/cli/test/connectors/clickhouse/dialect.test.ts index 14a1032c..809b1304 100644 --- a/packages/cli/src/connectors/clickhouse/dialect.test.ts +++ b/packages/cli/test/connectors/clickhouse/dialect.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { KtxClickHouseDialect } from './dialect.js'; +import { KtxClickHouseDialect } from '../../../src/connectors/clickhouse/dialect.js'; describe('KtxClickHouseDialect', () => { const dialect = new KtxClickHouseDialect(); @@ -23,7 +23,7 @@ describe('KtxClickHouseDialect', () => { expect(dialect.mapToDimensionType('')).toBe('string'); }); - it('builds sampling, distinct-value, pagination, and time SQL', () => { + it('builds sampling, distinct-value, and pagination SQL', () => { expect(dialect.generateSampleQuery('`analytics`.`events`', 25, ['id', 'event_name'])).toBe( 'SELECT `id`, `event_name` FROM `analytics`.`events` LIMIT 25', ); @@ -34,16 +34,6 @@ describe('KtxClickHouseDialect', () => { 'SELECT DISTINCT toString(`event_name`) AS val', ); expect(dialect.getLimitOffsetClause(10, 20)).toBe('LIMIT 10 OFFSET 20'); - expect(dialect.getTimeTruncExpression('created_at', 'week')).toBe('toStartOfWeek(created_at, 1)'); }); - it('prepares named parameters using ClickHouse typed placeholders', () => { - expect(dialect.prepareQuery('select * from events where id = :id and event_name = :name', { - id: 10, - name: 'signup', - })).toEqual({ - sql: 'select * from events where id = {id:Int64} and event_name = {name:String}', - params: { id: 10, name: 'signup' }, - }); - }); }); diff --git a/packages/cli/src/connectors/mysql/connector.test.ts b/packages/cli/test/connectors/mysql/connector.test.ts similarity index 74% rename from packages/cli/src/connectors/mysql/connector.test.ts rename to packages/cli/test/connectors/mysql/connector.test.ts index 5a21ada7..c8334164 100644 --- a/packages/cli/src/connectors/mysql/connector.test.ts +++ b/packages/cli/test/connectors/mysql/connector.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it, vi } from 'vitest'; import type { FieldPacket, RowDataPacket } from 'mysql2/promise'; -import { createMysqlLiveDatabaseIntrospection } from '../../connectors/mysql/live-database-introspection.js'; -import { isKtxMysqlConnectionConfig, KtxMysqlScanConnector, mysqlConnectionPoolConfigFromConfig, type KtxMysqlPoolFactory } from '../../connectors/mysql/connector.js'; -import { tableRefSet } from '../../context/scan/table-ref.js'; +import { createMysqlLiveDatabaseIntrospection } from '../../../src/connectors/mysql/live-database-introspection.js'; +import { isKtxMysqlConnectionConfig, KtxMysqlScanConnector, mysqlConnectionPoolConfigFromConfig, prepareMysqlReadOnlyQuery, type KtxMysqlConnectionConfig, type KtxMysqlPoolFactory } from '../../../src/connectors/mysql/connector.js'; +import { tableRefSet } from '../../../src/context/scan/table-ref.js'; function mysqlResult(rows: Record[], fields: Array<{ name: string; type?: number }>): [RowDataPacket[], FieldPacket[]] { return [rows as RowDataPacket[], fields as FieldPacket[]]; @@ -13,9 +13,9 @@ function fakePoolFactory(): KtxMysqlPoolFactory { if (sql.includes('INFORMATION_SCHEMA.TABLES')) { return mysqlResult( [ - { TABLE_NAME: 'customers', TABLE_TYPE: 'BASE TABLE', TABLE_COMMENT: 'Customer table', TABLE_ROWS: 2 }, - { TABLE_NAME: 'orders', TABLE_TYPE: 'BASE TABLE', TABLE_COMMENT: 'InnoDB free: 1 kB; Order table', TABLE_ROWS: 2 }, - { TABLE_NAME: 'order_summary', TABLE_TYPE: 'VIEW', TABLE_COMMENT: '', TABLE_ROWS: null }, + { TABLE_SCHEMA: 'analytics', TABLE_NAME: 'customers', TABLE_TYPE: 'BASE TABLE', TABLE_COMMENT: 'Customer table', TABLE_ROWS: 2 }, + { TABLE_SCHEMA: 'analytics', TABLE_NAME: 'orders', TABLE_TYPE: 'BASE TABLE', TABLE_COMMENT: 'InnoDB free: 1 kB; Order table', TABLE_ROWS: 2 }, + { TABLE_SCHEMA: 'analytics', TABLE_NAME: 'order_summary', TABLE_TYPE: 'VIEW', TABLE_COMMENT: '', TABLE_ROWS: null }, ], [{ name: 'TABLE_NAME' }, { name: 'TABLE_TYPE' }, { name: 'TABLE_COMMENT' }, { name: 'TABLE_ROWS' }], ); @@ -86,7 +86,9 @@ function fakePoolFactory(): KtxMysqlPoolFactory { }; } -function multiSchemaMysqlPoolFactory(): KtxMysqlPoolFactory { +function multiSchemaMysqlPoolFactory( + options: { primaryKeyError?: Error; foreignKeyError?: Error } = {}, +): KtxMysqlPoolFactory { const query = vi.fn(async (sql: string, params?: unknown): Promise<[RowDataPacket[], FieldPacket[]]> => { if (sql.includes('INFORMATION_SCHEMA.TABLES')) { expect(params).toEqual(['analytics', 'mart']); @@ -141,6 +143,9 @@ function multiSchemaMysqlPoolFactory(): KtxMysqlPoolFactory { ); } if (sql.includes('INFORMATION_SCHEMA.KEY_COLUMN_USAGE') && sql.includes("CONSTRAINT_NAME = 'PRIMARY'")) { + if (options.primaryKeyError) { + throw options.primaryKeyError; + } expect(params).toEqual(['analytics', 'mart']); return mysqlResult( [ @@ -151,6 +156,9 @@ function multiSchemaMysqlPoolFactory(): KtxMysqlPoolFactory { ); } if (sql.includes('INFORMATION_SCHEMA.KEY_COLUMN_USAGE') && sql.includes('REFERENCED_TABLE_NAME IS NOT NULL')) { + if (options.foreignKeyError) { + throw options.foreignKeyError; + } expect(params).toEqual(['analytics', 'mart']); return mysqlResult([], []); } @@ -165,6 +173,19 @@ function multiSchemaMysqlPoolFactory(): KtxMysqlPoolFactory { } describe('KtxMysqlScanConnector', () => { + it('prepares read-only SQL parameters with MySQL positional placeholders', () => { + expect( + prepareMysqlReadOnlyQuery('select * from orders where id = :id and status = :status', { + status: 'paid', + id: 10, + }), + ).toEqual({ + sql: 'select * from orders where id = ? and status = ?', + params: [10, 'paid'], + }); + expect(prepareMysqlReadOnlyQuery('select 1')).toEqual({ sql: 'select 1', params: undefined }); + }); + it('resolves MySQL connection configuration safely', () => { expect(isKtxMysqlConnectionConfig({ driver: 'mysql', host: 'localhost', database: 'analytics' })).toBe(true); expect(isKtxMysqlConnectionConfig({ driver: 'postgres', host: 'localhost', database: 'analytics' })).toBe(false); @@ -191,6 +212,46 @@ describe('KtxMysqlScanConnector', () => { }); }); + it('defaults and validates MySQL maxConnections', () => { + const baseConnection: KtxMysqlConnectionConfig = { + driver: 'mysql', + host: 'db.example.test', + database: 'analytics', + username: 'reader', + password: 'secret', // pragma: allowlist secret + }; + + expect( + mysqlConnectionPoolConfigFromConfig({ + connectionId: 'warehouse', + connection: baseConnection, + }), + ).toMatchObject({ connectionLimit: 10 }); + + expect( + mysqlConnectionPoolConfigFromConfig({ + connectionId: 'warehouse', + connection: { ...baseConnection, maxConnections: 25 }, + }), + ).toMatchObject({ connectionLimit: 25 }); + + expect( + mysqlConnectionPoolConfigFromConfig({ + connectionId: 'warehouse', + connection: { ...baseConnection, maxConnections: '12' as never }, + }), + ).toMatchObject({ connectionLimit: 12 }); + + for (const maxConnections of [0, -1, 1.5, Number.NaN, 'abc' as never]) { + expect(() => + mysqlConnectionPoolConfigFromConfig({ + connectionId: 'warehouse', + connection: { ...baseConnection, maxConnections }, + }), + ).toThrow('connections.warehouse.maxConnections must be a positive integer'); + } + }); + it('introspects schema, primary keys, comments, row counts, views, and foreign keys', async () => { const connector = new KtxMysqlScanConnector({ connectionId: 'warehouse', @@ -276,6 +337,65 @@ describe('KtxMysqlScanConnector', () => { ]); }); + it('soft-fails denied MySQL constraint discovery with one warning per schema and kind', async () => { + const connector = new KtxMysqlScanConnector({ + connectionId: 'warehouse', + connection: { + driver: 'mysql', + host: 'db.example.test', + database: 'analytics', + schemas: ['analytics', 'mart'], + username: 'reader', + password: 'secret', // pragma: allowlist secret + }, + poolFactory: multiSchemaMysqlPoolFactory({ + primaryKeyError: Object.assign(new Error('select command denied'), { + code: 'ER_TABLEACCESS_DENIED_ERROR', + errno: 1142, + }), + foreignKeyError: Object.assign(new Error('database access denied'), { + code: 'ER_DBACCESS_DENIED_ERROR', + errno: 1044, + }), + }), + now: () => new Date('2026-04-29T12:00:00.000Z'), + }); + + const snapshot = await connector.introspect( + { connectionId: 'warehouse', driver: 'mysql' }, + { runId: 'scan-run-mysql-denied-constraints' }, + ); + + expect(snapshot.warnings).toEqual([ + { + code: 'constraint_discovery_unauthorized', + message: 'Skipped primary-key discovery in analytics (insufficient grants on system catalogs)', + recoverable: true, + metadata: { schema: 'analytics', kind: 'primary_key' }, + }, + { + code: 'constraint_discovery_unauthorized', + message: 'Skipped primary-key discovery in mart (insufficient grants on system catalogs)', + recoverable: true, + metadata: { schema: 'mart', kind: 'primary_key' }, + }, + { + code: 'constraint_discovery_unauthorized', + message: 'Skipped foreign-key discovery in analytics (insufficient grants on system catalogs)', + recoverable: true, + metadata: { schema: 'analytics', kind: 'foreign_key' }, + }, + { + code: 'constraint_discovery_unauthorized', + message: 'Skipped foreign-key discovery in mart (insufficient grants on system catalogs)', + recoverable: true, + metadata: { schema: 'mart', kind: 'foreign_key' }, + }, + ]); + expect(snapshot.tables.every((table) => table.columns.every((column) => column.primaryKey === false))).toBe(true); + expect(snapshot.tables.every((table) => table.foreignKeys.length === 0)).toBe(true); + }); + it('limits introspection to tables in tableScope', async () => { const queries: Array<{ sql: string; params?: unknown }> = []; const poolFactory: KtxMysqlPoolFactory = { @@ -390,6 +510,11 @@ describe('KtxMysqlScanConnector', () => { await expect(connector.getTableRowCount('orders')).resolves.toBe(2); await expect(connector.listSchemas()).resolves.toEqual(['analytics', 'warehouse']); + await expect(connector.listTables(['analytics'])).resolves.toEqual([ + { catalog: null, schema: 'analytics', name: 'customers', kind: 'table' }, + { catalog: null, schema: 'analytics', name: 'orders', kind: 'table' }, + { catalog: null, schema: 'analytics', name: 'order_summary', kind: 'view' }, + ]); await expect(connector.columnStats( { connectionId: 'warehouse', table: { catalog: null, db: 'analytics', name: 'orders' }, column: 'status' }, { runId: 'scan-run-1' }, diff --git a/packages/cli/src/connectors/mysql/dialect.test.ts b/packages/cli/test/connectors/mysql/dialect.test.ts similarity index 75% rename from packages/cli/src/connectors/mysql/dialect.test.ts rename to packages/cli/test/connectors/mysql/dialect.test.ts index cf15527b..a00d6188 100644 --- a/packages/cli/src/connectors/mysql/dialect.test.ts +++ b/packages/cli/test/connectors/mysql/dialect.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { KtxMysqlDialect } from './dialect.js'; +import { KtxMysqlDialect } from '../../../src/connectors/mysql/dialect.js'; describe('KtxMysqlDialect', () => { const dialect = new KtxMysqlDialect(); @@ -23,7 +23,7 @@ describe('KtxMysqlDialect', () => { expect(dialect.mapToDimensionType('')).toBe('string'); }); - it('builds sampling, distinct-value, pagination, and time SQL', () => { + it('builds sampling, distinct-value, and pagination SQL', () => { expect(dialect.generateSampleQuery('`analytics`.`orders`', 25, ['id', 'status'])).toBe( 'SELECT `id`, `status` FROM `analytics`.`orders` LIMIT 25', ); @@ -34,16 +34,6 @@ describe('KtxMysqlDialect', () => { 'SELECT DISTINCT CAST(`status` AS CHAR) AS val', ); expect(dialect.getLimitOffsetClause(10, 20)).toBe('LIMIT 10 OFFSET 20'); - expect(dialect.getTimeTruncExpression('created_at', 'month')).toBe("DATE_FORMAT(created_at, '%Y-%m-01')"); }); - it('prepares named parameters in deterministic SQL placeholder order', () => { - expect(dialect.prepareQuery('select * from orders where id = :id and status = :status', { - status: 'paid', - id: 10, - })).toEqual({ - sql: 'select * from orders where id = ? and status = ?', - params: [10, 'paid'], - }); - }); }); diff --git a/packages/cli/src/connectors/postgres/connector.test.ts b/packages/cli/test/connectors/postgres/connector.test.ts similarity index 71% rename from packages/cli/src/connectors/postgres/connector.test.ts rename to packages/cli/test/connectors/postgres/connector.test.ts index 346c2ef2..e43e05a4 100644 --- a/packages/cli/src/connectors/postgres/connector.test.ts +++ b/packages/cli/test/connectors/postgres/connector.test.ts @@ -1,18 +1,23 @@ import { describe, expect, it, vi } from 'vitest'; -import { createPostgresLiveDatabaseIntrospection } from '../../connectors/postgres/live-database-introspection.js'; -import { isKtxPostgresConnectionConfig, KtxPostgresScanConnector, postgresPoolConfigFromConfig, type KtxPostgresPoolFactory } from '../../connectors/postgres/connector.js'; -import { tableRefSet } from '../../context/scan/table-ref.js'; +import { createPostgresLiveDatabaseIntrospection } from '../../../src/connectors/postgres/live-database-introspection.js'; +import { isKtxPostgresConnectionConfig, KtxPostgresScanConnector, postgresPoolConfigFromConfig, preparePostgresReadOnlyQuery, type KtxPostgresConnectionConfig, type KtxPostgresPoolFactory } from '../../../src/connectors/postgres/connector.js'; +import { tableRefSet } from '../../../src/context/scan/table-ref.js'; interface FakeQueryResult { rows: Record[]; fields?: Array<{ name: string; dataTypeID: number }>; } -function fakePoolFactory(results: Map): KtxPostgresPoolFactory { +type FakeQueryResponse = FakeQueryResult | Error; + +function fakePoolFactory(results: Map): KtxPostgresPoolFactory { const query = vi.fn(async (sql: string, params?: unknown[]) => { const normalized = sql.replace(/\s+/g, ' ').trim(); for (const [key, value] of results.entries()) { if (normalized.includes(key)) { + if (value instanceof Error) { + throw value; + } return value; } } @@ -33,15 +38,15 @@ function fakePoolFactory(results: Map): KtxPostgresPool }; } -function metadataResults(): Map { - return new Map([ +function metadataResults(): Map { + return new Map([ [ 'FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n', { rows: [ - { table_name: 'customers', table_kind: 'r', row_count: '2', table_comment: 'Customers' }, - { table_name: 'orders', table_kind: 'r', row_count: '3', table_comment: null }, - { table_name: 'recent_orders', table_kind: 'v', row_count: '0', table_comment: 'Recent orders' }, + { schema_name: 'public', table_name: 'customers', table_kind: 'r', row_count: '2', table_comment: 'Customers' }, + { schema_name: 'public', table_name: 'orders', table_kind: 'r', row_count: '3', table_comment: null }, + { schema_name: 'public', table_name: 'recent_orders', table_kind: 'v', row_count: '0', table_comment: 'Recent orders' }, ], }, ], @@ -97,9 +102,31 @@ function metadataResults(): Map { } describe('KtxPostgresScanConnector', () => { + it('prepares read-only SQL parameters with PostgreSQL positional placeholders', () => { + expect( + preparePostgresReadOnlyQuery('select * from orders where id = :id and status = :status', { + id: 1, + status: 'paid', + }), + ).toEqual({ + sql: 'select * from orders where id = $1 and status = $2', + params: [1, 'paid'], + }); + expect( + preparePostgresReadOnlyQuery('select :Client_Name_10, :Client_Name_1', { + Client_Name_1: 'short', + Client_Name_10: 'long', + }), + ).toEqual({ + sql: 'select $2, $1', + params: ['short', 'long'], + }); + expect(preparePostgresReadOnlyQuery('select 1')).toEqual({ sql: 'select 1', params: undefined }); + }); + it('resolves configuration safely', () => { expect(isKtxPostgresConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL' })).toBe(true); - expect(isKtxPostgresConnectionConfig({ driver: 'postgresql', host: 'db', database: 'analytics' })).toBe(true); + expect(isKtxPostgresConnectionConfig({ driver: 'postgresql', host: 'db', database: 'analytics' })).toBe(false); expect(isKtxPostgresConnectionConfig({ driver: 'mysql', host: 'db' })).toBe(false); expect( postgresPoolConfigFromConfig({ @@ -154,6 +181,46 @@ describe('KtxPostgresScanConnector', () => { }); }); + it('defaults and validates Postgres maxConnections', () => { + const baseConnection: KtxPostgresConnectionConfig = { + driver: 'postgres', + host: 'db.example.test', + database: 'analytics', + username: 'reader', + password: 'test-password', // pragma: allowlist secret + }; + + expect( + postgresPoolConfigFromConfig({ + connectionId: 'warehouse', + connection: baseConnection, + }), + ).toMatchObject({ max: 10 }); + + expect( + postgresPoolConfigFromConfig({ + connectionId: 'warehouse', + connection: { ...baseConnection, maxConnections: 50 }, + }), + ).toMatchObject({ max: 50 }); + + expect( + postgresPoolConfigFromConfig({ + connectionId: 'warehouse', + connection: { ...baseConnection, maxConnections: '12' as never }, + }), + ).toMatchObject({ max: 12 }); + + for (const maxConnections of [0, -1, 1.5, Number.NaN, 'abc' as never]) { + expect(() => + postgresPoolConfigFromConfig({ + connectionId: 'warehouse', + connection: { ...baseConnection, maxConnections }, + }), + ).toThrow('connections.warehouse.maxConnections must be a positive integer'); + } + }); + it('introspects schemas, tables, views, primary keys, comments, row counts, and foreign keys', async () => { const connector = new KtxPostgresScanConnector({ connectionId: 'warehouse', @@ -212,6 +279,75 @@ describe('KtxPostgresScanConnector', () => { ]); }); + it('soft-fails denied Postgres constraint discovery with scan warnings', async () => { + const results = metadataResults(); + results.set( + "tc.constraint_type = 'PRIMARY KEY'", + Object.assign(new Error('permission denied for information_schema'), { code: '42501' }), + ); + results.set( + "tc.constraint_type = 'FOREIGN KEY'", + Object.assign(new Error('relation information_schema.key_column_usage does not exist'), { code: '42P01' }), + ); + const connector = new KtxPostgresScanConnector({ + connectionId: 'warehouse', + connection: { + driver: 'postgres', + host: 'db.example.test', + database: 'analytics', + username: 'reader', + password: 'test-password', // pragma: allowlist secret + schema: 'public', + }, + poolFactory: fakePoolFactory(results), + now: () => new Date('2026-04-29T10:00:00.000Z'), + }); + + const snapshot = await connector.introspect( + { connectionId: 'warehouse', driver: 'postgres' }, + { runId: 'scan-run-denied-constraints' }, + ); + + expect(snapshot.warnings).toEqual([ + { + code: 'constraint_discovery_unauthorized', + message: 'Skipped primary-key discovery in public (insufficient grants on system catalogs)', + recoverable: true, + metadata: { schema: 'public', kind: 'primary_key' }, + }, + { + code: 'constraint_discovery_unauthorized', + message: 'Skipped foreign-key discovery in public (insufficient grants on system catalogs)', + recoverable: true, + metadata: { schema: 'public', kind: 'foreign_key' }, + }, + ]); + expect(snapshot.tables.every((table) => table.columns.every((column) => column.primaryKey === false))).toBe(true); + expect(snapshot.tables.every((table) => table.foreignKeys.length === 0)).toBe(true); + }); + + it('propagates non-denial Postgres constraint discovery errors', async () => { + const results = metadataResults(); + const resetError = Object.assign(new Error('connection reset'), { code: 'ECONNRESET' }); + results.set("tc.constraint_type = 'PRIMARY KEY'", resetError); + const connector = new KtxPostgresScanConnector({ + connectionId: 'warehouse', + connection: { + driver: 'postgres', + host: 'db.example.test', + database: 'analytics', + username: 'reader', + password: 'test-password', // pragma: allowlist secret + schema: 'public', + }, + poolFactory: fakePoolFactory(results), + }); + + await expect( + connector.introspect({ connectionId: 'warehouse', driver: 'postgres' }, { runId: 'scan-run-network-error' }), + ).rejects.toBe(resetError); + }); + it('runs samples, distinct values, statistics, read-only SQL, and schema listing', async () => { const connector = new KtxPostgresScanConnector({ connectionId: 'warehouse', @@ -253,6 +389,11 @@ describe('KtxPostgresScanConnector', () => { }); await expect(connector.getTableRowCount({ db: 'public', name: 'orders' })).resolves.toBe(3); await expect(connector.listSchemas()).resolves.toEqual(['public']); + await expect(connector.listTables(['public'])).resolves.toEqual([ + { catalog: null, schema: 'public', name: 'customers', kind: 'table' }, + { catalog: null, schema: 'public', name: 'orders', kind: 'table' }, + { catalog: null, schema: 'public', name: 'recent_orders', kind: 'view' }, + ]); await expect(connector.testConnection()).resolves.toEqual({ success: true }); await expect( diff --git a/packages/cli/src/connectors/postgres/dialect.test.ts b/packages/cli/test/connectors/postgres/dialect.test.ts similarity index 64% rename from packages/cli/src/connectors/postgres/dialect.test.ts rename to packages/cli/test/connectors/postgres/dialect.test.ts index ffe85497..1a1d4768 100644 --- a/packages/cli/src/connectors/postgres/dialect.test.ts +++ b/packages/cli/test/connectors/postgres/dialect.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { KtxPostgresDialect } from './dialect.js'; +import { KtxPostgresDialect } from '../../../src/connectors/postgres/dialect.js'; describe('KtxPostgresDialect', () => { const dialect = new KtxPostgresDialect(); @@ -18,7 +18,7 @@ describe('KtxPostgresDialect', () => { expect(dialect.mapToDimensionType('jsonb')).toBe('string'); }); - it('generates sample, distinct-value, statistics, and time SQL', () => { + it('generates sample, distinct-value, and statistics SQL', () => { expect(dialect.generateSampleQuery('"public"."orders"', 5, ['id', 'status'])).toBe( 'SELECT "id", "status" FROM "public"."orders" LIMIT 5', ); @@ -29,24 +29,6 @@ describe('KtxPostgresDialect', () => { 'SELECT DISTINCT "status"::text AS val', ); expect(dialect.generateColumnStatisticsQuery('public', 'orders')).toContain('FROM pg_stats s'); - expect(dialect.getTimeTruncExpression('"created_at"', 'month')).toBe('DATE_TRUNC(\'month\', "created_at")'); }); - it('prepares named parameters with PostgreSQL positional parameters', () => { - expect( - dialect.prepareQuery('select * from orders where id = :id and status = :status', { id: 1, status: 'paid' }), - ).toEqual({ - sql: 'select * from orders where id = $1 and status = $2', - params: [1, 'paid'], - }); - expect( - dialect.prepareQuery('select :Client_Name_10, :Client_Name_1', { - Client_Name_1: 'short', - Client_Name_10: 'long', - }), - ).toEqual({ - sql: 'select $2, $1', - params: ['short', 'long'], - }); - }); }); diff --git a/packages/cli/src/connectors/postgres/historic-sql-query-client.test.ts b/packages/cli/test/connectors/postgres/historic-sql-query-client.test.ts similarity index 90% rename from packages/cli/src/connectors/postgres/historic-sql-query-client.test.ts rename to packages/cli/test/connectors/postgres/historic-sql-query-client.test.ts index b9c9fd40..0ec03fe1 100644 --- a/packages/cli/src/connectors/postgres/historic-sql-query-client.test.ts +++ b/packages/cli/test/connectors/postgres/historic-sql-query-client.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { KtxPostgresHistoricSqlQueryClient } from './historic-sql-query-client.js'; -import type { KtxPostgresPoolConfig, KtxPostgresPoolFactory } from './connector.js'; +import { KtxPostgresHistoricSqlQueryClient } from '../../../src/connectors/postgres/historic-sql-query-client.js'; +import type { KtxPostgresPoolConfig, KtxPostgresPoolFactory } from '../../../src/connectors/postgres/connector.js'; describe('KtxPostgresHistoricSqlQueryClient', () => { it('executes parameterized read-only SQL through the native Postgres connector pool', async () => { diff --git a/packages/cli/src/connectors/snowflake/connector.test.ts b/packages/cli/test/connectors/snowflake/connector.test.ts similarity index 81% rename from packages/cli/src/connectors/snowflake/connector.test.ts rename to packages/cli/test/connectors/snowflake/connector.test.ts index a321e289..1b00061b 100644 --- a/packages/cli/src/connectors/snowflake/connector.test.ts +++ b/packages/cli/test/connectors/snowflake/connector.test.ts @@ -7,9 +7,9 @@ vi.mock('snowflake-sdk', () => ({ createPool, })); -import { createSnowflakeLiveDatabaseIntrospection } from '../../connectors/snowflake/live-database-introspection.js'; -import { isKtxSnowflakeConnectionConfig, KtxSnowflakeScanConnector, snowflakeConnectionConfigFromConfig, type KtxSnowflakeDriver, type KtxSnowflakeDriverFactory } from '../../connectors/snowflake/connector.js'; -import { tableRefSet } from '../../context/scan/table-ref.js'; +import { createSnowflakeLiveDatabaseIntrospection } from '../../../src/connectors/snowflake/live-database-introspection.js'; +import { isKtxSnowflakeConnectionConfig, KtxSnowflakeScanConnector, prepareSnowflakeReadOnlyQuery, snowflakeConnectionConfigFromConfig, type KtxSnowflakeConnectionConfig, type KtxSnowflakeDriver, type KtxSnowflakeDriverFactory } from '../../../src/connectors/snowflake/connector.js'; +import { tableRefSet } from '../../../src/context/scan/table-ref.js'; function fakeDriverFactory(): KtxSnowflakeDriverFactory { const driver: KtxSnowflakeDriver = { @@ -64,8 +64,8 @@ function fakeDriverFactory(): KtxSnowflakeDriverFactory { ]), listSchemas: vi.fn(async () => ['PUBLIC', 'MART']), listTables: vi.fn(async () => [ - { schema: 'PUBLIC', name: 'ORDERS', kind: 'table' as const }, - { schema: 'PUBLIC', name: 'ORDER_SUMMARY', kind: 'view' as const }, + { catalog: 'ANALYTICS', schema: 'PUBLIC', name: 'ORDERS', kind: 'table' as const }, + { catalog: 'ANALYTICS', schema: 'PUBLIC', name: 'ORDER_SUMMARY', kind: 'view' as const }, ]), cleanup: vi.fn(async () => undefined), }; @@ -105,6 +105,17 @@ function installSnowflakePoolMock() { } describe('KtxSnowflakeScanConnector', () => { + it('prepares read-only SQL parameters with Snowflake bind arrays', () => { + expect(prepareSnowflakeReadOnlyQuery('SELECT * FROM ORDERS WHERE ID = ? AND STATUS = ?', { id: 1, status: 'paid' })).toEqual({ + sql: 'SELECT * FROM ORDERS WHERE ID = ? AND STATUS = ?', + params: [1, 'paid'], + }); + expect(prepareSnowflakeReadOnlyQuery('SELECT * FROM ORDERS')).toEqual({ + sql: 'SELECT * FROM ORDERS', + params: undefined, + }); + }); + it('resolves Snowflake connection configuration safely', () => { expect( isKtxSnowflakeConnectionConfig({ @@ -140,8 +151,8 @@ describe('KtxSnowflakeScanConnector', () => { }); }); - it('defaults and validates Snowflake maxSessions', () => { - const baseConnection = { + it('defaults and validates Snowflake maxConnections', () => { + const baseConnection: KtxSnowflakeConnectionConfig = { driver: 'snowflake', authMethod: 'password', account: 'acct', @@ -150,32 +161,59 @@ describe('KtxSnowflakeScanConnector', () => { schema_name: 'PUBLIC', username: 'reader', password: 'fixture-pass', // pragma: allowlist secret - } as const; + }; expect( snowflakeConnectionConfigFromConfig({ connectionId: 'warehouse', connection: baseConnection, }), - ).toMatchObject({ maxSessions: 4 }); + ).toMatchObject({ maxConnections: 4 }); expect( snowflakeConnectionConfigFromConfig({ connectionId: 'warehouse', - connection: { ...baseConnection, maxSessions: 8 }, + connection: { ...baseConnection, maxConnections: 8 }, }), - ).toMatchObject({ maxSessions: 8 }); + ).toMatchObject({ maxConnections: 8 }); - for (const maxSessions of [0, -1, 1.5, Number.NaN]) { + expect( + snowflakeConnectionConfigFromConfig({ + connectionId: 'warehouse', + connection: { ...baseConnection, maxConnections: '12' as never }, + }), + ).toMatchObject({ maxConnections: 12 }); + + for (const maxConnections of [0, -1, 1.5, Number.NaN, 'abc' as never]) { expect(() => snowflakeConnectionConfigFromConfig({ connectionId: 'warehouse', - connection: { ...baseConnection, maxSessions }, + connection: { ...baseConnection, maxConnections }, }), - ).toThrow('connections.warehouse.maxSessions must be a positive integer'); + ).toThrow('connections.warehouse.maxConnections must be a positive integer'); } }); + it('rejects stale Snowflake pool config key', () => { + const baseConnection: KtxSnowflakeConnectionConfig = { + driver: 'snowflake', + authMethod: 'password', + account: 'acct', + warehouse: 'WH', + database: 'ANALYTICS', + schema_name: 'PUBLIC', + username: 'reader', + password: 'fixture-pass', // pragma: allowlist secret + }; + + expect(() => + snowflakeConnectionConfigFromConfig({ + connectionId: 'warehouse', + connection: { ...baseConnection, maxSessions: 8 }, + }), + ).toThrow(/renamed to maxConnections/); + }); + it('uses one lazy Snowflake pool and drains it during cleanup', async () => { const { pool, executedSql } = installSnowflakePoolMock(); const close = vi.fn(async () => undefined); @@ -191,7 +229,7 @@ describe('KtxSnowflakeScanConnector', () => { username: 'reader', password: 'fixture-pass', // pragma: allowlist secret role: 'ANALYST', - maxSessions: 3, + maxConnections: 3, }, sdkOptionsProvider: { resolve: vi.fn(async () => ({ sdkOptions: { application: 'ktx-test' }, close })), @@ -332,12 +370,56 @@ describe('KtxSnowflakeScanConnector', () => { expect(snapshot.tables.map((table) => table.name).sort()).toEqual(['ORDERS', 'ORDER_SUMMARY']); expect(snapshot.tables.every((table) => table.columns.every((column) => column.primaryKey === false))).toBe(true); + expect(snapshot.warnings).toEqual([ + { + code: 'constraint_discovery_unauthorized', + message: 'Skipped primary-key discovery in PUBLIC (insufficient grants on system catalogs)', + recoverable: true, + metadata: { schema: 'PUBLIC', kind: 'primary_key' }, + }, + ]); expect(warn).not.toHaveBeenCalled(); } finally { warn.mockRestore(); } }); + it('propagates non-denial Snowflake primary-key discovery errors', async () => { + const driverFactory = fakeDriverFactory(); + const driver = (driverFactory.createDriver as ReturnType).getMockImplementation() as + | (() => KtxSnowflakeDriver) + | undefined; + if (!driver) throw new Error('driver mock missing'); + const built = driver(); + const networkError = new Error('network unavailable'); + (built.query as ReturnType).mockImplementation(async (sql: string) => { + if (sql.includes('TABLE_CONSTRAINTS')) { + throw networkError; + } + throw new Error(`Unexpected SQL: ${sql}`); + }); + (driverFactory.createDriver as ReturnType).mockReturnValue(built); + + const connector = new KtxSnowflakeScanConnector({ + connectionId: 'warehouse', + connection: { + driver: 'snowflake', + authMethod: 'password', + account: 'acct', + warehouse: 'WH', + database: 'ANALYTICS', + schema_name: 'PUBLIC', + username: 'reader', + password: 'fixture-pass', // pragma: allowlist secret + }, + driverFactory, + }); + + await expect( + connector.introspect({ connectionId: 'warehouse', driver: 'snowflake' }, { runId: 'scan-run-snowflake-network' }), + ).rejects.toBe(networkError); + }); + it('limits introspection to tables in tableScope', async () => { const queries: Array<{ sql: string; params?: unknown }> = []; const getSchemaMetadata = vi.fn(async (_schemaName?: string, scopedNames?: readonly string[] | null) => @@ -490,8 +572,8 @@ describe('KtxSnowflakeScanConnector', () => { }); await expect(connector.listTables(['MART', 'PUBLIC'])).resolves.toEqual([ - { schema: 'MART', name: 'ORDERS', kind: 'table' }, - { schema: 'PUBLIC', name: 'ORDER_SUMMARY', kind: 'view' }, + { catalog: 'ANALYTICS', schema: 'MART', name: 'ORDERS', kind: 'table' }, + { catalog: 'ANALYTICS', schema: 'PUBLIC', name: 'ORDER_SUMMARY', kind: 'view' }, ]); expect(queries).toHaveLength(1); diff --git a/packages/cli/src/connectors/snowflake/dialect.test.ts b/packages/cli/test/connectors/snowflake/dialect.test.ts similarity index 80% rename from packages/cli/src/connectors/snowflake/dialect.test.ts rename to packages/cli/test/connectors/snowflake/dialect.test.ts index 991a30b5..4f966ffb 100644 --- a/packages/cli/src/connectors/snowflake/dialect.test.ts +++ b/packages/cli/test/connectors/snowflake/dialect.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { KtxSnowflakeDialect } from './dialect.js'; +import { KtxSnowflakeDialect } from '../../../src/connectors/snowflake/dialect.js'; describe('KtxSnowflakeDialect', () => { const dialect = new KtxSnowflakeDialect(); @@ -36,14 +36,6 @@ describe('KtxSnowflakeDialect', () => { ); }); - it('passes Snowflake positional parameters as bind arrays', () => { - expect(dialect.prepareQuery('SELECT * FROM ORDERS WHERE ID = ? AND STATUS = ?', { id: 1, status: 'paid' })).toEqual({ - sql: 'SELECT * FROM ORDERS WHERE ID = ? AND STATUS = ?', - params: [1, 'paid'], - }); - expect(dialect.prepareQuery('SELECT * FROM ORDERS')).toEqual({ sql: 'SELECT * FROM ORDERS', params: undefined }); - }); - it('keeps unsupported statistics explicit', () => { expect(dialect.generateColumnStatisticsQuery('PUBLIC', 'ORDERS')).toBeNull(); }); diff --git a/packages/cli/src/connectors/snowflake/identifiers.test.ts b/packages/cli/test/connectors/snowflake/identifiers.test.ts similarity index 93% rename from packages/cli/src/connectors/snowflake/identifiers.test.ts rename to packages/cli/test/connectors/snowflake/identifiers.test.ts index d2c3e448..0a3b4cb8 100644 --- a/packages/cli/src/connectors/snowflake/identifiers.test.ts +++ b/packages/cli/test/connectors/snowflake/identifiers.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { assertSafeSnowflakeIdentifier, quoteSnowflakeIdentifier } from './identifiers.js'; +import { assertSafeSnowflakeIdentifier, quoteSnowflakeIdentifier } from '../../../src/connectors/snowflake/identifiers.js'; describe('Snowflake identifier guards', () => { it('quotes simple Snowflake identifiers', () => { diff --git a/packages/cli/src/connectors/snowflake/sdk-logger.test.ts b/packages/cli/test/connectors/snowflake/sdk-logger.test.ts similarity index 96% rename from packages/cli/src/connectors/snowflake/sdk-logger.test.ts rename to packages/cli/test/connectors/snowflake/sdk-logger.test.ts index 73bf0c76..1217b142 100644 --- a/packages/cli/src/connectors/snowflake/sdk-logger.test.ts +++ b/packages/cli/test/connectors/snowflake/sdk-logger.test.ts @@ -11,7 +11,7 @@ vi.mock('snowflake-sdk', () => ({ import { configureSnowflakeSdkLogger, resetSnowflakeSdkLoggerConfigurationForTests, -} from './sdk-logger.js'; +} from '../../../src/connectors/snowflake/sdk-logger.js'; describe('configureSnowflakeSdkLogger', () => { let projectDir: string; diff --git a/packages/cli/src/connectors/sqlite/connector.test.ts b/packages/cli/test/connectors/sqlite/connector.test.ts similarity index 91% rename from packages/cli/src/connectors/sqlite/connector.test.ts rename to packages/cli/test/connectors/sqlite/connector.test.ts index ecd283b7..27b00c57 100644 --- a/packages/cli/src/connectors/sqlite/connector.test.ts +++ b/packages/cli/test/connectors/sqlite/connector.test.ts @@ -4,9 +4,9 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { createSqliteLiveDatabaseIntrospection } from '../../connectors/sqlite/live-database-introspection.js'; -import { isKtxSqliteConnectionConfig, KtxSqliteScanConnector, sqliteDatabasePathFromConfig } from '../../connectors/sqlite/connector.js'; -import { tableRefSet } from '../../context/scan/table-ref.js'; +import { createSqliteLiveDatabaseIntrospection } from '../../../src/connectors/sqlite/live-database-introspection.js'; +import { isKtxSqliteConnectionConfig, KtxSqliteScanConnector, sqliteDatabasePathFromConfig } from '../../../src/connectors/sqlite/connector.js'; +import { tableRefSet } from '../../../src/context/scan/table-ref.js'; describe('KtxSqliteScanConnector', () => { let tempDir: string; @@ -150,6 +150,20 @@ describe('KtxSqliteScanConnector', () => { ]); }); + it('lists schemaless tables and views for setup discovery', async () => { + const connector = new KtxSqliteScanConnector({ + connectionId: 'warehouse', + connection: { driver: 'sqlite', path: dbPath }, + }); + + await expect(connector.listSchemas()).resolves.toEqual([]); + await expect(connector.listTables(['ignored'])).resolves.toEqual([ + { catalog: null, schema: '', name: 'customers', kind: 'table' }, + { catalog: null, schema: '', name: 'orders', kind: 'table' }, + { catalog: null, schema: '', name: 'recent_orders', kind: 'view' }, + ]); + }); + it('runs samples, distinct values, statistics, and read-only SQL', async () => { const connector = new KtxSqliteScanConnector({ connectionId: 'warehouse', diff --git a/packages/cli/src/connectors/sqlite/dialect.test.ts b/packages/cli/test/connectors/sqlite/dialect.test.ts similarity index 95% rename from packages/cli/src/connectors/sqlite/dialect.test.ts rename to packages/cli/test/connectors/sqlite/dialect.test.ts index cefed21a..879d133c 100644 --- a/packages/cli/src/connectors/sqlite/dialect.test.ts +++ b/packages/cli/test/connectors/sqlite/dialect.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { KtxSqliteDialect } from './dialect.js'; +import { KtxSqliteDialect } from '../../../src/connectors/sqlite/dialect.js'; describe('KtxSqliteDialect', () => { const dialect = new KtxSqliteDialect(); diff --git a/packages/cli/src/connectors/sqlserver/connector.test.ts b/packages/cli/test/connectors/sqlserver/connector.test.ts similarity index 72% rename from packages/cli/src/connectors/sqlserver/connector.test.ts rename to packages/cli/test/connectors/sqlserver/connector.test.ts index ef00bd3a..b7318ab5 100644 --- a/packages/cli/src/connectors/sqlserver/connector.test.ts +++ b/packages/cli/test/connectors/sqlserver/connector.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import { createSqlServerLiveDatabaseIntrospection } from '../../connectors/sqlserver/live-database-introspection.js'; -import { isKtxSqlServerConnectionConfig, KtxSqlServerScanConnector, sqlServerConnectionPoolConfigFromConfig, type KtxSqlServerPoolFactory, type KtxSqlServerQueryResult } from '../../connectors/sqlserver/connector.js'; -import { tableRefSet } from '../../context/scan/table-ref.js'; +import { createSqlServerLiveDatabaseIntrospection } from '../../../src/connectors/sqlserver/live-database-introspection.js'; +import { isKtxSqlServerConnectionConfig, KtxSqlServerScanConnector, prepareSqlServerReadOnlyQuery, sqlServerConnectionPoolConfigFromConfig, type KtxSqlServerConnectionConfig, type KtxSqlServerPoolFactory, type KtxSqlServerQueryResult } from '../../../src/connectors/sqlserver/connector.js'; +import { tableRefSet } from '../../../src/context/scan/table-ref.js'; function recordset>( rows: T[], @@ -16,14 +16,14 @@ function result>(rows: T[], columnNames: strin return { recordset: recordset(rows, columnNames) }; } -function fakePoolFactory(): KtxSqlServerPoolFactory { +function fakePoolFactory(options: { primaryKeyError?: Error; foreignKeyError?: Error } = {}): KtxSqlServerPoolFactory { const query = vi.fn(async (sql: string): Promise => { if (sql.includes('INFORMATION_SCHEMA.TABLES')) { return result( [ - { table_name: 'customers', table_type: 'BASE TABLE' }, - { table_name: 'orders', table_type: 'BASE TABLE' }, - { table_name: 'order_summary', table_type: 'VIEW' }, + { schema_name: 'dbo', table_name: 'customers', table_type: 'BASE TABLE' }, + { schema_name: 'dbo', table_name: 'orders', table_type: 'BASE TABLE' }, + { schema_name: 'dbo', table_name: 'order_summary', table_type: 'VIEW' }, ], ['table_name', 'table_type'], ); @@ -55,6 +55,9 @@ function fakePoolFactory(): KtxSqlServerPoolFactory { ); } if (sql.includes("CONSTRAINT_TYPE = 'PRIMARY KEY'")) { + if (options.primaryKeyError) { + throw options.primaryKeyError; + } return result( [ { table_name: 'customers', column_name: 'id' }, @@ -64,6 +67,9 @@ function fakePoolFactory(): KtxSqlServerPoolFactory { ); } if (sql.includes('REFERENTIAL_CONSTRAINTS')) { + if (options.foreignKeyError) { + throw options.foreignKeyError; + } return result( [ { @@ -94,13 +100,13 @@ function fakePoolFactory(): KtxSqlServerPoolFactory { ['table_name', 'row_count'], ); } - if (sql.includes('SELECT TOP 1 [id], [status] FROM [dbo].[orders]')) { + if (sql.includes('SELECT TOP 1 [id], [status] FROM [analytics].[dbo].[orders]')) { return result([{ id: 10, status: 'paid' }], ['id', 'status']); } if (sql.includes('SELECT TOP 1 * FROM (select id, status from dbo.orders) AS ktx_query_result')) { return result([{ id: 10, status: 'paid' }], ['id', 'status']); } - if (sql.includes('SELECT TOP 5 [status] FROM [dbo].[orders]')) { + if (sql.includes('SELECT TOP 5 [status] FROM [analytics].[dbo].[orders]')) { return result([{ status: 'paid' }, { status: 'open' }], ['status']); } if (sql.includes('COUNT(DISTINCT val)')) { @@ -112,6 +118,16 @@ function fakePoolFactory(): KtxSqlServerPoolFactory { if (sql.includes('SUM(p.rows) AS row_count') && sql.includes('t.name = @tableName')) { return result([{ row_count: 2 }], ['row_count']); } + if (sql.includes('FROM sys.objects o')) { + return result( + [ + { schema_name: 'dbo', table_name: 'customers', table_type: 'USER_TABLE' }, + { schema_name: 'dbo', table_name: 'order_summary', table_type: 'VIEW' }, + { schema_name: 'dbo', table_name: 'orders', table_type: 'USER_TABLE' }, + ], + ['schema_name', 'table_name', 'table_type'], + ); + } if (sql.includes('SELECT s.name AS schema_name')) { return result([{ schema_name: 'dbo' }, { schema_name: 'sales' }], ['schema_name']); } @@ -134,6 +150,19 @@ function fakePoolFactory(): KtxSqlServerPoolFactory { } describe('KtxSqlServerScanConnector', () => { + it('prepares read-only SQL parameters with SQL Server named placeholders', () => { + expect( + prepareSqlServerReadOnlyQuery('select * from events where id = :id and name = :name', { + id: 10, + name: 'signup', + }), + ).toEqual({ + sql: 'select * from events where id = @id and name = @name', + params: { id: 10, name: 'signup' }, + }); + expect(prepareSqlServerReadOnlyQuery('select 1')).toEqual({ sql: 'select 1', params: undefined }); + }); + it('resolves SQL Server connection configuration safely', () => { expect( isKtxSqlServerConnectionConfig({ @@ -164,6 +193,45 @@ describe('KtxSqlServerScanConnector', () => { }); }); + it('defaults and validates SQL Server maxConnections', () => { + const baseConnection: KtxSqlServerConnectionConfig = { + driver: 'sqlserver', + host: 'db.example.test', + database: 'analytics', + username: 'reader', + }; + + expect( + sqlServerConnectionPoolConfigFromConfig({ + connectionId: 'warehouse', + connection: baseConnection, + }), + ).toMatchObject({ pool: { max: 10 } }); + + expect( + sqlServerConnectionPoolConfigFromConfig({ + connectionId: 'warehouse', + connection: { ...baseConnection, maxConnections: 15 }, + }), + ).toMatchObject({ pool: { max: 15 } }); + + expect( + sqlServerConnectionPoolConfigFromConfig({ + connectionId: 'warehouse', + connection: { ...baseConnection, maxConnections: '12' as never }, + }), + ).toMatchObject({ pool: { max: 12 } }); + + for (const maxConnections of [0, -1, 1.5, Number.NaN, 'abc' as never]) { + expect(() => + sqlServerConnectionPoolConfigFromConfig({ + connectionId: 'warehouse', + connection: { ...baseConnection, maxConnections }, + }), + ).toThrow('connections.warehouse.maxConnections must be a positive integer'); + } + }); + it('introspects schema, primary keys, comments, row counts, views, and foreign keys', async () => { const connector = new KtxSqlServerScanConnector({ connectionId: 'warehouse', @@ -222,6 +290,46 @@ describe('KtxSqlServerScanConnector', () => { ]); }); + it('soft-fails denied SQL Server constraint discovery with scan warnings', async () => { + const connector = new KtxSqlServerScanConnector({ + connectionId: 'warehouse', + connection: { + driver: 'sqlserver', + host: 'db.example.test', + database: 'analytics', + username: 'reader', + schema: 'dbo', + }, + poolFactory: fakePoolFactory({ + primaryKeyError: Object.assign(new Error('SELECT permission denied'), { number: 229 }), + foreignKeyError: Object.assign(new Error('EXECUTE permission denied'), { number: 230 }), + }), + now: () => new Date('2026-04-29T16:00:00.000Z'), + }); + + const snapshot = await connector.introspect( + { connectionId: 'warehouse', driver: 'sqlserver' }, + { runId: 'scan-run-sqlserver-denied-constraints' }, + ); + + expect(snapshot.warnings).toEqual([ + { + code: 'constraint_discovery_unauthorized', + message: 'Skipped primary-key discovery in dbo (insufficient grants on system catalogs)', + recoverable: true, + metadata: { schema: 'dbo', kind: 'primary_key' }, + }, + { + code: 'constraint_discovery_unauthorized', + message: 'Skipped foreign-key discovery in dbo (insufficient grants on system catalogs)', + recoverable: true, + metadata: { schema: 'dbo', kind: 'foreign_key' }, + }, + ]); + expect(snapshot.tables.every((table) => table.columns.every((column) => column.primaryKey === false))).toBe(true); + expect(snapshot.tables.every((table) => table.foreignKeys.length === 0)).toBe(true); + }); + it('runs samples, distinct values, read-only SQL, row count, schema list, and cleanup', async () => { const poolFactory = fakePoolFactory(); const connector = new KtxSqlServerScanConnector({ @@ -281,6 +389,11 @@ describe('KtxSqlServerScanConnector', () => { await expect(connector.getTableRowCount('orders')).resolves.toBe(2); await expect(connector.listSchemas()).resolves.toEqual(['dbo', 'sales']); + await expect(connector.listTables(['dbo'])).resolves.toEqual([ + { catalog: 'analytics', schema: 'dbo', name: 'customers', kind: 'table' }, + { catalog: 'analytics', schema: 'dbo', name: 'order_summary', kind: 'view' }, + { catalog: 'analytics', schema: 'dbo', name: 'orders', kind: 'table' }, + ]); await expect( connector.columnStats( { connectionId: 'warehouse', table: { catalog: 'analytics', db: 'dbo', name: 'orders' }, column: 'status' }, diff --git a/packages/cli/src/connectors/sqlserver/dialect.test.ts b/packages/cli/test/connectors/sqlserver/dialect.test.ts similarity index 64% rename from packages/cli/src/connectors/sqlserver/dialect.test.ts rename to packages/cli/test/connectors/sqlserver/dialect.test.ts index 5c855340..f019991c 100644 --- a/packages/cli/src/connectors/sqlserver/dialect.test.ts +++ b/packages/cli/test/connectors/sqlserver/dialect.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { KtxSqlServerDialect } from './dialect.js'; +import { KtxSqlServerDialect } from '../../../src/connectors/sqlserver/dialect.js'; describe('KtxSqlServerDialect', () => { const dialect = new KtxSqlServerDialect(); @@ -7,7 +7,9 @@ describe('KtxSqlServerDialect', () => { it('quotes identifiers and formats schema-qualified table names', () => { expect(dialect.quoteIdentifier('events')).toBe('[events]'); expect(dialect.quoteIdentifier('odd]name')).toBe('[odd]]name]'); - expect(dialect.formatTableName({ catalog: 'warehouse', db: 'dbo', name: 'events' })).toBe('[dbo].[events]'); + expect(dialect.formatTableName({ catalog: 'warehouse', db: 'dbo', name: 'events' })).toBe( + '[warehouse].[dbo].[events]', + ); expect(dialect.formatTableName({ catalog: null, db: null, name: 'events' })).toBe('[events]'); }); @@ -20,7 +22,7 @@ describe('KtxSqlServerDialect', () => { expect(dialect.mapToDimensionType('')).toBe('string'); }); - it('builds sampling, distinct-value, pagination, and time SQL', () => { + it('builds sampling, distinct-value, and pagination SQL', () => { expect(dialect.generateSampleQuery('[dbo].[events]', 25, ['id', 'event_name'])).toBe( 'SELECT TOP 25 [id], [event_name] FROM [dbo].[events]', ); @@ -28,22 +30,8 @@ describe('KtxSqlServerDialect', () => { "SELECT TOP 10 [event_name] FROM [dbo].[events] WHERE [event_name] IS NOT NULL AND LTRIM(RTRIM(CAST([event_name] AS NVARCHAR(MAX)))) != ''", ); expect(dialect.generateDistinctValuesQuery('[dbo].[events]', '[event_name]', 5)).toContain('SELECT TOP 5 val'); - expect(dialect.getTopClause(10)).toBe('TOP 10'); - expect(dialect.getLimitOffsetClause(10, 20)).toBe('OFFSET 20 ROWS FETCH NEXT 10 ROWS ONLY'); - expect(dialect.getTimeTruncExpression('created_at', 'month')).toBe( - 'DATEFROMPARTS(YEAR(created_at), MONTH(created_at), 1)', - ); + expect(dialect.getTopClause(10)).toBe('TOP (10)'); + expect(dialect.getLimitOffsetClause(10, 20)).toBe(''); }); - it('prepares named parameters using SQL Server @ parameters', () => { - expect( - dialect.prepareQuery('select * from events where id = :id and name = :name', { - id: 10, - name: 'signup', - }), - ).toEqual({ - sql: 'select * from events where id = @id and name = @name', - params: { id: 10, name: 'signup' }, - }); - }); }); diff --git a/packages/cli/src/context-build-view.test.ts b/packages/cli/test/context-build-view.test.ts similarity index 97% rename from packages/cli/src/context-build-view.test.ts rename to packages/cli/test/context-build-view.test.ts index 089124f5..d8692eb5 100644 --- a/packages/cli/src/context-build-view.test.ts +++ b/packages/cli/test/context-build-view.test.ts @@ -1,6 +1,6 @@ -import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from '../src/context/project/config.js'; import { describe, expect, it, vi } from 'vitest'; -import type { KtxPublicIngestProject, KtxPublicIngestTargetResult } from './public-ingest.js'; +import type { KtxPublicIngestProject, KtxPublicIngestTargetResult } from '../src/public-ingest.js'; import { type ContextBuildTargetState, extractProgressMessage, @@ -11,7 +11,7 @@ import { renderContextBuildView, runContextBuild, viewStateFromSourceProgress, -} from './context-build-view.js'; +} from '../src/context-build-view.js'; function makeIo(options: { isTTY?: boolean; columns?: number } = {}) { let stdout = ''; @@ -228,11 +228,11 @@ describe('renderContextBuildView', () => { const rendered = renderContextBuildView(state, { styled: false, - warnings: ['--deep affects database ingest only; ignoring it for docs.'], + warnings: ['--query-history affects database ingest only; ignoring it for docs.'], }); expect(rendered).toContain('Warnings:'); - expect(rendered).toContain('--deep affects database ingest only; ignoring it for docs.'); + expect(rendered).toContain('--query-history affects database ingest only; ignoring it for docs.'); }); it('renders public notices in the foreground view before warnings', () => { @@ -243,7 +243,6 @@ describe('renderContextBuildView', () => { operation: 'database-ingest', debugCommand: 'ktx ingest warehouse --debug', steps: ['database-schema', 'query-history'], - databaseDepth: 'deep', detectRelationships: true, queryHistory: { enabled: true, dialect: 'postgres' }, }, @@ -252,12 +251,12 @@ describe('renderContextBuildView', () => { const rendered = renderContextBuildView(state, { styled: false, notices: ['Schema ingest runs before query history for warehouse.'], - warnings: ['--query-history requires deep ingest; running warehouse with --deep.'], + warnings: ['--query-history is not supported for sqlite; running schema ingest for local.'], }); expect(rendered.indexOf('Notices:')).toBeLessThan(rendered.indexOf('Warnings:')); expect(rendered).toContain('Schema ingest runs before query history for warehouse.'); - expect(rendered).toContain('--query-history requires deep ingest; running warehouse with --deep.'); + expect(rendered).toContain('--query-history is not supported for sqlite; running schema ingest for local.'); }); it('renders dynamic separator matching header width', () => { @@ -653,7 +652,6 @@ describe('runContextBuild', () => { inputMode: 'disabled', targetConnectionId: 'warehouse', all: false, - depth: 'fast', queryHistory: 'default', }, io.io, @@ -665,7 +663,6 @@ describe('runContextBuild', () => { expect(executeTarget.mock.calls[0]?.[0]).toMatchObject({ connectionId: 'warehouse', operation: 'database-ingest', - databaseDepth: 'fast', }); expect(io.stdout()).toContain('Databases:'); expect(io.stdout()).toContain('warehouse'); @@ -716,7 +713,7 @@ describe('runContextBuild', () => { it('renders localhost SQL analysis refusal as a runtime failure during query history', async () => { const io = makeIo(); const project = projectWithConnections({ - warehouse: { driver: 'postgres', context: { depth: 'deep', queryHistory: { enabled: true } } }, + warehouse: { driver: 'postgres', context: { queryHistory: { enabled: true } } }, }); const executeTarget = vi.fn(async (target, _args, targetIo) => { targetIo.stderr.write('connect ECONNREFUSED 127.0.0.1:8765\n'); @@ -751,7 +748,7 @@ describe('runContextBuild', () => { it('uses captured query-history stderr instead of generic failed-at detail', async () => { const io = makeIo(); const project = projectWithConnections({ - warehouse: { driver: 'postgres', context: { depth: 'deep', queryHistory: { enabled: true } } }, + warehouse: { driver: 'postgres', context: { queryHistory: { enabled: true } } }, }); const executeTarget = vi.fn(async (target, _args, targetIo) => { targetIo.stdout.write('KTX scan completed\n'); @@ -768,7 +765,7 @@ describe('runContextBuild', () => { operation: 'query-history', status: 'failed', detail: - 'warehouse failed at query-history. Retry: ktx ingest warehouse --project-dir /tmp/project --deep --query-history', + 'warehouse failed at query-history. Retry: ktx ingest warehouse --project-dir /tmp/project --query-history', }, { operation: 'source-ingest', status: 'skipped' }, { operation: 'memory-update', status: 'skipped' }, @@ -785,7 +782,7 @@ describe('runContextBuild', () => { expect(result).toEqual({ exitCode: 1 }); expect(io.stdout()).toContain('Missing bundled Python runtime manifest: /tmp/assets/python/manifest.json.'); - expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --deep --query-history'); + expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --query-history'); expect(io.stdout()).not.toContain('Then retry the runtime-backed KTX command'); expect(io.stdout()).not.toContain('warehouse failed at query-history'); expect(io.stdout().match(/Retry: /g)).toHaveLength(1); @@ -899,12 +896,12 @@ describe('runContextBuild', () => { const io = makeIo(); const project: KtxPublicIngestProject = { ...projectWithConnections({ - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, + warehouse: { driver: 'postgres' }, }), config: { ...projectWithConnections({ warehouse: { driver: 'postgres' } }).config, connections: { - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, + warehouse: { driver: 'postgres' }, }, llm: { provider: { backend: 'gateway', gateway: { api_key: 'env:KTX_GATEWAY_API_KEY' } }, // pragma: allowlist secret @@ -987,6 +984,7 @@ describe('runContextBuild', () => { scanProgress: expect.anything(), ingestProgress: expect.any(Function), }), + project, ); }); @@ -1018,6 +1016,7 @@ describe('runContextBuild', () => { expect.objectContaining({ runtimeIo: io.io, }), + project, ); }); diff --git a/packages/cli/src/context/connections/bigquery-identifiers.test.ts b/packages/cli/test/context/connections/bigquery-identifiers.test.ts similarity index 92% rename from packages/cli/src/context/connections/bigquery-identifiers.test.ts rename to packages/cli/test/context/connections/bigquery-identifiers.test.ts index a1fd2e09..abc59164 100644 --- a/packages/cli/src/context/connections/bigquery-identifiers.test.ts +++ b/packages/cli/test/context/connections/bigquery-identifiers.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { normalizeBigQueryProjectId, normalizeBigQueryRegion } from './bigquery-identifiers.js'; +import { normalizeBigQueryProjectId, normalizeBigQueryRegion } from '../../../src/context/connections/bigquery-identifiers.js'; describe('BigQuery identifier normalization', () => { it('normalizes project ids and regions for information schema paths', () => { diff --git a/packages/cli/test/context/connections/dialects.test.ts b/packages/cli/test/context/connections/dialects.test.ts new file mode 100644 index 00000000..0b72566e --- /dev/null +++ b/packages/cli/test/context/connections/dialects.test.ts @@ -0,0 +1,316 @@ +import { describe, expect, it } from 'vitest'; +import { getDialectForDriver } from '../../../src/context/connections/dialects.js'; +import type { KtxConnectionDriver, KtxTableRef } from '../../../src/context/scan/types.js'; + +interface DialectFixture { + driver: KtxConnectionDriver; + table: KtxTableRef; + quoteInput: string; + quotedIdentifier: string; + formattedTable: string; + display: string; + invalidDisplay: string; + columnDisplayTablePartCount: 1 | 2 | 3; + limitClause: string; + topClause: string; + randomFilter: string; + tableSampleClause: string; + sampleQuery: string; + columnSampleContains: string; + nullCountExpression: string; + distinctCountExpression: string; + textLengthExpression: string; + castToText: string; + sampleValueAggregation: string; + cardinalityContains: string; + randomizedCardinalityContains: string; + distinctValuesContains: string; + statisticsContains: string | null; + dimensionInput: string; + dimensionType: 'time' | 'string' | 'number' | 'boolean'; + nativeTypeInput: string; + normalizedType: string; +} + +const innerSampleSql = 'SELECT status AS value FROM orders'; + +const fixtures: DialectFixture[] = [ + { + driver: 'postgres', + table: { catalog: null, db: 'public', name: 'orders' }, + quoteInput: 'order"items', + quotedIdentifier: '"order""items"', + formattedTable: '"public"."orders"', + display: 'public.orders', + invalidDisplay: 'orders', + columnDisplayTablePartCount: 2, + limitClause: 'LIMIT 25 OFFSET 5', + topClause: '', + randomFilter: 'RANDOM() < 0.25', + tableSampleClause: 'TABLESAMPLE SYSTEM (25)', + sampleQuery: 'SELECT "id", "status" FROM "public"."orders" LIMIT 5', + columnSampleContains: 'TRIM(CAST("status" AS TEXT)) != \'\'', + nullCountExpression: 'COUNT(*) FILTER (WHERE "status" IS NULL)', + distinctCountExpression: 'COUNT(DISTINCT "status")', + textLengthExpression: 'LENGTH(CAST("status" AS TEXT))', + castToText: 'CAST("status" AS TEXT)', + sampleValueAggregation: + '(SELECT STRING_AGG(CAST(value AS TEXT), CHR(31)) FROM (SELECT status AS value FROM orders) AS relationship_profile_values)', + cardinalityContains: 'SELECT COUNT(DISTINCT val) AS cardinality', + randomizedCardinalityContains: 'ORDER BY RANDOM()', + distinctValuesContains: 'SELECT DISTINCT "status"::text AS val', + statisticsContains: 'FROM pg_stats s', + dimensionInput: 'timestamp with time zone', + dimensionType: 'time', + nativeTypeInput: 'numeric(12,2)', + normalizedType: 'numeric(12,2)', + }, + { + driver: 'mysql', + table: { catalog: null, db: 'analytics', name: 'orders' }, + quoteInput: 'order`items', + quotedIdentifier: '`order``items`', + formattedTable: '`analytics`.`orders`', + display: 'analytics.orders', + invalidDisplay: 'orders', + columnDisplayTablePartCount: 2, + limitClause: 'LIMIT 25 OFFSET 5', + topClause: '', + randomFilter: 'RAND() < 0.25', + tableSampleClause: '', + sampleQuery: 'SELECT `id`, `status` FROM `analytics`.`orders` LIMIT 5', + columnSampleContains: 'TRIM(CAST(`status` AS CHAR)) != \'\'', + nullCountExpression: 'SUM(CASE WHEN `status` IS NULL THEN 1 ELSE 0 END)', + distinctCountExpression: 'COUNT(DISTINCT `status`)', + textLengthExpression: 'CHAR_LENGTH(CAST(`status` AS CHAR))', + castToText: 'CAST(`status` AS CHAR)', + sampleValueAggregation: + '(SELECT GROUP_CONCAT(CAST(value AS CHAR) SEPARATOR CHAR(31)) FROM (SELECT status AS value FROM orders) AS relationship_profile_values)', + cardinalityContains: 'SELECT COUNT(DISTINCT val) AS cardinality', + randomizedCardinalityContains: 'ORDER BY RAND()', + distinctValuesContains: 'SELECT DISTINCT CAST(`status` AS CHAR) AS val', + statisticsContains: null, + dimensionInput: 'tinyint(1)', + dimensionType: 'boolean', + nativeTypeInput: 'varchar(255)', + normalizedType: 'varchar(255)', + }, + { + driver: 'clickhouse', + table: { catalog: null, db: 'analytics', name: 'events' }, + quoteInput: 'order`items', + quotedIdentifier: '`order``items`', + formattedTable: '`analytics`.`events`', + display: 'analytics.events', + invalidDisplay: 'events', + columnDisplayTablePartCount: 2, + limitClause: 'LIMIT 25 OFFSET 5', + topClause: '', + randomFilter: 'rand() / 4294967295.0 < 0.25', + tableSampleClause: '', + sampleQuery: 'SELECT `id`, `status` FROM `analytics`.`events` LIMIT 5', + columnSampleContains: 'trim(toString(`status`)) != \'\'', + nullCountExpression: 'countIf(`status` IS NULL)', + distinctCountExpression: 'COUNT(DISTINCT `status`)', + textLengthExpression: 'length(toString(`status`))', + castToText: 'toString(`status`)', + sampleValueAggregation: + '(SELECT arrayStringConcat(groupArray(toString(value)), \'\\x1F\') FROM (SELECT status AS value FROM orders) AS relationship_profile_values)', + cardinalityContains: 'SELECT COUNT(DISTINCT val) AS cardinality', + randomizedCardinalityContains: 'ORDER BY rand()', + distinctValuesContains: 'SELECT DISTINCT toString(`status`) AS val', + statisticsContains: null, + dimensionInput: 'Nullable(DateTime64(3))', + dimensionType: 'time', + nativeTypeInput: 'LowCardinality(String)', + normalizedType: 'LowCardinality(String)', + }, + { + driver: 'sqlite', + table: { catalog: null, db: null, name: 'orders' }, + quoteInput: 'order"items', + quotedIdentifier: '"order""items"', + formattedTable: '"orders"', + display: 'orders', + invalidDisplay: 'public.orders', + columnDisplayTablePartCount: 1, + limitClause: 'LIMIT 25 OFFSET 5', + topClause: '', + randomFilter: '(RANDOM() % 100) < 25', + tableSampleClause: '', + sampleQuery: 'SELECT "id", "status" FROM "orders" LIMIT 5', + columnSampleContains: 'TRIM(CAST("status" AS TEXT)) != \'\'', + nullCountExpression: 'SUM(CASE WHEN "status" IS NULL THEN 1 ELSE 0 END)', + distinctCountExpression: 'COUNT(DISTINCT "status")', + textLengthExpression: 'LENGTH(CAST("status" AS TEXT))', + castToText: 'CAST("status" AS TEXT)', + sampleValueAggregation: + '(SELECT GROUP_CONCAT(CAST(value AS TEXT), char(31)) FROM (SELECT status AS value FROM orders) AS relationship_profile_values)', + cardinalityContains: 'SELECT COUNT(DISTINCT val) AS cardinality', + randomizedCardinalityContains: 'ORDER BY RANDOM()', + distinctValuesContains: 'SELECT DISTINCT CAST("status" AS TEXT) AS val', + statisticsContains: null, + dimensionInput: 'INTEGER', + dimensionType: 'number', + nativeTypeInput: 'VARCHAR(255)', + normalizedType: 'VARCHAR(255)', + }, + { + driver: 'snowflake', + table: { catalog: 'ANALYTICS', db: 'PUBLIC', name: 'ORDERS' }, + quoteInput: 'order"items', + quotedIdentifier: '"order""items"', + formattedTable: '"ANALYTICS"."PUBLIC"."ORDERS"', + display: 'ANALYTICS.PUBLIC.ORDERS', + invalidDisplay: 'PUBLIC.ORDERS', + columnDisplayTablePartCount: 3, + limitClause: 'LIMIT 25 OFFSET 5', + topClause: '', + randomFilter: 'UNIFORM(0::FLOAT, 1::FLOAT, RANDOM()) < 0.25', + tableSampleClause: 'SAMPLE (25)', + sampleQuery: 'SELECT "id", "status" FROM "ANALYTICS"."PUBLIC"."ORDERS" SAMPLE ROW (5 ROWS)', + columnSampleContains: 'TRIM(CAST("status" AS STRING)) != \'\'', + nullCountExpression: 'COUNT_IF("status" IS NULL)', + distinctCountExpression: 'APPROX_COUNT_DISTINCT("status")', + textLengthExpression: 'LENGTH(CAST("status" AS TEXT))', + castToText: 'CAST("status" AS VARCHAR)', + sampleValueAggregation: + '(SELECT LISTAGG(CAST(value AS VARCHAR), \'\\x1f\') FROM (SELECT status AS value FROM orders) AS relationship_profile_values)', + cardinalityContains: 'SELECT COUNT(DISTINCT val) AS cardinality', + randomizedCardinalityContains: 'SAMPLE ROW (100 ROWS)', + distinctValuesContains: 'SELECT DISTINCT "status"::VARCHAR AS val', + statisticsContains: null, + dimensionInput: 'TIMESTAMP_NTZ', + dimensionType: 'time', + nativeTypeInput: 'NUMBER(38,0)', + normalizedType: 'NUMBER(38,0)', + }, + { + driver: 'bigquery', + table: { catalog: 'analytics-project', db: 'warehouse', name: 'orders' }, + quoteInput: 'order`items', + quotedIdentifier: '`order\\`items`', + formattedTable: '`analytics-project`.`warehouse`.`orders`', + display: 'analytics-project.warehouse.orders', + invalidDisplay: 'warehouse.orders', + columnDisplayTablePartCount: 3, + limitClause: 'LIMIT 25 OFFSET 5', + topClause: '', + randomFilter: 'RAND() < 0.25', + tableSampleClause: 'TABLESAMPLE SYSTEM (25 PERCENT)', + sampleQuery: 'SELECT `id`, `status` FROM `analytics-project`.`warehouse`.`orders` ORDER BY RAND() LIMIT 5', + columnSampleContains: 'TRIM(CAST(`status` AS STRING)) != \'\'', + nullCountExpression: 'COUNTIF(`status` IS NULL)', + distinctCountExpression: 'APPROX_COUNT_DISTINCT(`status`)', + textLengthExpression: 'LENGTH(CAST(`status` AS STRING))', + castToText: 'CAST(`status` AS STRING)', + sampleValueAggregation: + '(SELECT STRING_AGG(CAST(value AS STRING), \'\\u001F\') FROM (SELECT status AS value FROM orders) AS relationship_profile_values)', + cardinalityContains: 'SELECT APPROX_COUNT_DISTINCT(val) AS cardinality', + randomizedCardinalityContains: 'ORDER BY RAND()', + distinctValuesContains: 'SELECT DISTINCT CAST(`status` AS STRING) AS val', + statisticsContains: null, + dimensionInput: 'INT64', + dimensionType: 'number', + nativeTypeInput: 'INT64', + normalizedType: 'BIGINT', + }, + { + driver: 'sqlserver', + table: { catalog: 'warehouse', db: 'dbo', name: 'events' }, + quoteInput: 'odd]name', + quotedIdentifier: '[odd]]name]', + formattedTable: '[warehouse].[dbo].[events]', + display: 'warehouse.dbo.events', + invalidDisplay: 'dbo.events', + columnDisplayTablePartCount: 3, + limitClause: '', + topClause: 'TOP (25)', + randomFilter: 'ABS(CHECKSUM(NEWID())) % 100 < 25', + tableSampleClause: 'TABLESAMPLE (25 PERCENT)', + sampleQuery: 'SELECT TOP 5 [id], [status] FROM [warehouse].[dbo].[events]', + columnSampleContains: 'LTRIM(RTRIM(CAST([status] AS NVARCHAR(MAX)))) != \'\'', + nullCountExpression: 'SUM(CASE WHEN [status] IS NULL THEN 1 ELSE 0 END)', + distinctCountExpression: 'COUNT(DISTINCT [status])', + textLengthExpression: 'LEN(CAST([status] AS NVARCHAR(MAX)))', + castToText: 'CAST([status] AS NVARCHAR(MAX))', + sampleValueAggregation: + '(SELECT STRING_AGG(CAST(value AS NVARCHAR(MAX)), CHAR(31)) FROM (SELECT status AS value FROM orders) AS relationship_profile_values)', + cardinalityContains: 'SELECT COUNT(DISTINCT val) AS cardinality', + randomizedCardinalityContains: 'ORDER BY NEWID()', + distinctValuesContains: 'SELECT TOP 20 val', + statisticsContains: null, + dimensionInput: 'datetime2', + dimensionType: 'time', + nativeTypeInput: 'uniqueidentifier', + normalizedType: 'uniqueidentifier', + }, +]; + +describe('getDialectForDriver', () => { + it.each(fixtures)('returns a full KtxDialect for $driver', (fixture) => { + const dialect = getDialectForDriver(fixture.driver); + const column = dialect.quoteIdentifier('status'); + + expect(dialect.type).toBe(fixture.driver); + expect(dialect.quoteIdentifier(fixture.quoteInput)).toBe(fixture.quotedIdentifier); + expect(dialect.formatTableName(fixture.table)).toBe(fixture.formattedTable); + expect(dialect.formatDisplayRef(fixture.table)).toBe(fixture.display); + expect(dialect.parseDisplayRef(fixture.display)).toEqual(fixture.table); + expect(dialect.parseDisplayRef(fixture.invalidDisplay)).toBeNull(); + expect(dialect.columnDisplayTablePartCount()).toBe(fixture.columnDisplayTablePartCount); + expect(dialect.getLimitOffsetClause(25, 5)).toBe(fixture.limitClause); + expect(dialect.getTopClause(25)).toBe(fixture.topClause); + expect(dialect.getRandomSampleFilter(0.25)).toBe(fixture.randomFilter); + expect(dialect.getTableSampleClause(0.25)).toBe(fixture.tableSampleClause); + expect(dialect.generateSampleQuery(fixture.formattedTable, 5, ['id', 'status'])).toBe(fixture.sampleQuery); + expect(dialect.generateColumnSampleQuery(fixture.formattedTable, 'status', 10)).toContain( + fixture.columnSampleContains, + ); + expect(dialect.getNullCountExpression(column)).toBe(fixture.nullCountExpression); + expect(dialect.getDistinctCountExpression(column)).toBe(fixture.distinctCountExpression); + expect(dialect.textLengthExpression(column)).toBe(fixture.textLengthExpression); + expect(dialect.castToText(column)).toBe(fixture.castToText); + expect(dialect.getSampleValueAggregation(innerSampleSql)).toBe(fixture.sampleValueAggregation); + expect(dialect.generateCardinalitySampleQuery(fixture.formattedTable, column, 100)).toContain( + fixture.cardinalityContains, + ); + expect(dialect.generateRandomizedCardinalitySampleQuery(fixture.formattedTable, column, 100)).toContain( + fixture.randomizedCardinalityContains, + ); + expect(dialect.generateDistinctValuesQuery(fixture.formattedTable, column, 20)).toContain( + fixture.distinctValuesContains, + ); + const statistics = dialect.generateColumnStatisticsQuery(fixture.table.db ?? '', fixture.table.name); + if (fixture.statisticsContains) { + expect(statistics).toContain(fixture.statisticsContains); + } else { + expect(statistics).toBeNull(); + } + expect(dialect.mapToDimensionType(fixture.dimensionInput)).toBe(fixture.dimensionType); + expect(dialect.mapDataType(fixture.nativeTypeInput)).toBe(fixture.normalizedType); + }); + + it('accepts three-part ANSI display refs while keeping one-part names caller-owned', () => { + for (const driver of ['postgres', 'mysql', 'clickhouse'] as const) { + const dialect = getDialectForDriver(driver); + expect(dialect.parseDisplayRef('warehouse.public.orders')).toEqual({ + catalog: 'warehouse', + db: 'public', + name: 'orders', + }); + expect(dialect.parseDisplayRef('orders')).toBeNull(); + } + }); + + it('throws with a supported-driver list for unknown drivers', () => { + expect(() => getDialectForDriver('oracle')).toThrow( + 'Unsupported warehouse driver "oracle". Supported drivers: bigquery, clickhouse, mysql, postgres, sqlite, snowflake, sqlserver', + ); + }); + + it('rejects legacy driver aliases', () => { + expect(() => getDialectForDriver('postgresql')).toThrow('Unsupported warehouse driver "postgresql"'); + expect(() => getDialectForDriver('sqlite3')).toThrow('Unsupported warehouse driver "sqlite3"'); + }); +}); diff --git a/packages/cli/test/context/connections/drivers.test.ts b/packages/cli/test/context/connections/drivers.test.ts new file mode 100644 index 00000000..380b2265 --- /dev/null +++ b/packages/cli/test/context/connections/drivers.test.ts @@ -0,0 +1,145 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + driverRegistrations, + getDriverRegistration, + listSupportedDrivers, +} from '../../../src/context/connections/drivers.js'; +import type { + KtxDriverConnectorModule, + KtxScopeConfigKey, +} from '../../../src/context/connections/drivers.js'; +import type { KtxConnectionDriver } from '../../../src/context/scan/types.js'; + +type FixtureFactory = (projectDir: string) => Record; + +const connectionFixtures: Record = { + postgres: () => ({ + driver: 'postgres', + url: 'postgresql://reader:secret@localhost:5432/analytics', // pragma: allowlist secret + schemas: ['public'], + }), + sqlite: () => ({ driver: 'sqlite', path: 'warehouse.db' }), + mysql: () => ({ + driver: 'mysql', + host: 'localhost', + database: 'analytics', + username: 'reader', + password: 'secret', // pragma: allowlist secret + schemas: ['analytics'], + }), + clickhouse: () => ({ + driver: 'clickhouse', + url: 'http://localhost:8123', + database: 'analytics', + username: 'reader', + password: 'secret', // pragma: allowlist secret + }), + sqlserver: () => ({ + driver: 'sqlserver', + host: 'localhost', + database: 'analytics', + username: 'reader', + password: 'secret', // pragma: allowlist secret + schemas: ['dbo'], + }), + bigquery: () => ({ + driver: 'bigquery', + dataset_id: 'analytics', + credentials_json: JSON.stringify({ + project_id: 'project-1', + client_email: 'reader@example.test', + private_key: '-----BEGIN PRIVATE KEY-----\nsecret\n-----END PRIVATE KEY-----\n', // pragma: allowlist secret + }), + location: 'US', + }), + snowflake: () => ({ + driver: 'snowflake', + account: 'example-account', + username: 'reader', + password: 'secret', // pragma: allowlist secret + warehouse: 'COMPUTE_WH', + database: 'ANALYTICS', + schema: 'PUBLIC', + }), +}; + +const allowedScopeKeys = new Set(['dataset_ids', 'databases', 'schemas', 'schema_names']); +const historicSqlReaderDrivers = new Set(['postgres', 'bigquery', 'snowflake']); +const localExecutorDrivers = new Set(['postgres', 'sqlite']); + +function assertExportedRegistryBoundaryTypes(input: { + scopeConfigKey: KtxScopeConfigKey; + connectorModule: KtxDriverConnectorModule; +}): { + scopeConfigKey: KtxScopeConfigKey; + connectorModule: KtxDriverConnectorModule; +} { + return input; +} + +describe('driverRegistrations', () => { + let projectDir: string; + + beforeEach(async () => { + projectDir = await mkdtemp(join(tmpdir(), 'ktx-driver-registry-')); + }); + + afterEach(async () => { + await rm(projectDir, { recursive: true, force: true }); + }); + + it('lists every supported warehouse driver', () => { + const registryDrivers = Object.keys(driverRegistrations).sort(); + expect(listSupportedDrivers()).toEqual(registryDrivers); + expect(listSupportedDrivers()).toEqual([ + 'bigquery', + 'clickhouse', + 'mysql', + 'postgres', + 'snowflake', + 'sqlite', + 'sqlserver', + ]); + }); + + it('resolves registered drivers case-insensitively', () => { + expect(getDriverRegistration(' Postgres ')?.driver).toBe('postgres'); + expect(getDriverRegistration('unknown')).toBeUndefined(); + }); + + it.each(Object.values(driverRegistrations))('adapts $driver connector exports', async (registration) => { + const connectorModule = await registration.load(); + const connection = connectionFixtures[registration.driver](projectDir); + const exportedBoundary = assertExportedRegistryBoundaryTypes({ + scopeConfigKey: registration.scopeConfigKey ?? 'schemas', + connectorModule, + }); + expect(exportedBoundary.connectorModule.createScanConnector).toEqual(expect.any(Function)); + + expect(connectorModule.isConnectionConfig(connection)).toBe(true); + expect(connectorModule.isConnectionConfig({})).toBe(false); + + const connector = connectorModule.createScanConnector({ + connectionId: 'warehouse', + connection, + projectDir, + }); + + expect(connector.driver).toBe(registration.driver); + expect(connector.listSchemas).toEqual(expect.any(Function)); + expect(connector.listTables).toEqual(expect.any(Function)); + await connector.cleanup?.(); + + if (registration.driver === 'sqlite') { + expect(registration.scopeConfigKey).toBeNull(); + } else { + expect(registration.scopeConfigKey).not.toBeNull(); + expect(allowedScopeKeys.has(registration.scopeConfigKey ?? '')).toBe(true); + } + expect(registration.hasHistoricSqlReader).toBe(historicSqlReaderDrivers.has(registration.driver)); + expect(registration.hasLocalQueryExecutor).toBe(localExecutorDrivers.has(registration.driver)); + }); +}); diff --git a/packages/cli/src/context/connections/local-query-executor.test.ts b/packages/cli/test/context/connections/local-query-executor.test.ts similarity index 93% rename from packages/cli/src/context/connections/local-query-executor.test.ts rename to packages/cli/test/context/connections/local-query-executor.test.ts index d2f77975..ca700b04 100644 --- a/packages/cli/src/context/connections/local-query-executor.test.ts +++ b/packages/cli/test/context/connections/local-query-executor.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { createDefaultLocalQueryExecutor } from './local-query-executor.js'; +import { createDefaultLocalQueryExecutor } from '../../../src/context/connections/local-query-executor.js'; describe('createDefaultLocalQueryExecutor', () => { it('dispatches postgres and sqlite drivers to their executors', async () => { diff --git a/packages/cli/src/context/connections/local-warehouse-descriptor.test.ts b/packages/cli/test/context/connections/local-warehouse-descriptor.test.ts similarity index 86% rename from packages/cli/src/context/connections/local-warehouse-descriptor.test.ts rename to packages/cli/test/context/connections/local-warehouse-descriptor.test.ts index 0eee9f34..e0a285a9 100644 --- a/packages/cli/src/context/connections/local-warehouse-descriptor.test.ts +++ b/packages/cli/test/context/connections/local-warehouse-descriptor.test.ts @@ -3,7 +3,7 @@ import { localConnectionInfoFromConfig, localConnectionToWarehouseDescriptor, localConnectionTypeForConfig, -} from './local-warehouse-descriptor.js'; +} from '../../../src/context/connections/local-warehouse-descriptor.js'; describe('localConnectionToWarehouseDescriptor', () => { it('maps local Postgres URLs to canonical warehouse descriptors', () => { @@ -53,6 +53,11 @@ describe('local connection info helpers', () => { expect(localConnectionTypeForConfig('snowflake', { driver: 'snowflake' })).toBe('SNOWFLAKE'); }); + it('keeps removed driver aliases as display-only labels', () => { + expect(localConnectionTypeForConfig('warehouse', { driver: 'postgresql' } as never)).toBe('postgresql'); + expect(localConnectionTypeForConfig('warehouse', { driver: 'mssql' } as never)).toBe('mssql'); + }); + it('keeps non-warehouse adapter labels for display-only local connection surfaces', () => { expect(localConnectionTypeForConfig('prod-metabase', { driver: 'metabase', api_url: 'https://metabase.example.com' })).toBe( 'metabase', diff --git a/packages/cli/src/context/connections/notion-config.test.ts b/packages/cli/test/context/connections/notion-config.test.ts similarity index 98% rename from packages/cli/src/context/connections/notion-config.test.ts rename to packages/cli/test/context/connections/notion-config.test.ts index 6416bf99..4b1cde96 100644 --- a/packages/cli/src/context/connections/notion-config.test.ts +++ b/packages/cli/test/context/connections/notion-config.test.ts @@ -7,7 +7,7 @@ import { parseNotionConnectionConfig, redactNotionConnectionConfig, resolveNotionAuthToken, -} from './notion-config.js'; +} from '../../../src/context/connections/notion-config.js'; describe('standalone Notion connection config', () => { let tempDir: string; diff --git a/packages/cli/src/context/connections/postgres-query-executor.test.ts b/packages/cli/test/context/connections/postgres-query-executor.test.ts similarity index 96% rename from packages/cli/src/context/connections/postgres-query-executor.test.ts rename to packages/cli/test/context/connections/postgres-query-executor.test.ts index 6bb522cf..fe4ab15c 100644 --- a/packages/cli/src/context/connections/postgres-query-executor.test.ts +++ b/packages/cli/test/context/connections/postgres-query-executor.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { createPostgresQueryExecutor } from './postgres-query-executor.js'; +import { createPostgresQueryExecutor } from '../../../src/context/connections/postgres-query-executor.js'; function makeClient() { const calls: unknown[] = []; diff --git a/packages/cli/src/context/connections/read-only-sql.test.ts b/packages/cli/test/context/connections/read-only-sql.test.ts similarity index 91% rename from packages/cli/src/context/connections/read-only-sql.test.ts rename to packages/cli/test/context/connections/read-only-sql.test.ts index 217bf23d..90affa04 100644 --- a/packages/cli/src/context/connections/read-only-sql.test.ts +++ b/packages/cli/test/context/connections/read-only-sql.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { assertReadOnlySql, limitSqlForExecution } from './read-only-sql.js'; +import { assertReadOnlySql, limitSqlForExecution } from '../../../src/context/connections/read-only-sql.js'; describe('assertReadOnlySql', () => { it('allows select and with queries', () => { diff --git a/packages/cli/src/context/connections/sqlite-query-executor.test.ts b/packages/cli/test/context/connections/sqlite-query-executor.test.ts similarity index 98% rename from packages/cli/src/context/connections/sqlite-query-executor.test.ts rename to packages/cli/test/context/connections/sqlite-query-executor.test.ts index facb5139..a9e61ba5 100644 --- a/packages/cli/src/context/connections/sqlite-query-executor.test.ts +++ b/packages/cli/test/context/connections/sqlite-query-executor.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import Database from 'better-sqlite3'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { createSqliteQueryExecutor, sqliteDatabasePathFromConnection } from './sqlite-query-executor.js'; +import { createSqliteQueryExecutor, sqliteDatabasePathFromConnection } from '../../../src/context/connections/sqlite-query-executor.js'; describe('createSqliteQueryExecutor', () => { let tempDir: string; diff --git a/packages/cli/test/context/core/abort.test.ts b/packages/cli/test/context/core/abort.test.ts new file mode 100644 index 00000000..aed46c1e --- /dev/null +++ b/packages/cli/test/context/core/abort.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it, vi } from 'vitest'; +import { createAbortError, isAbortError, linkAbortSignal, throwIfAborted } from '../../../src/context/core/abort.js'; + +describe('abort helpers', () => { + it('recognizes DOMException abort errors and common abort-shaped errors', () => { + expect(isAbortError(createAbortError())).toBe(true); + expect(isAbortError(Object.assign(new Error('cancelled'), { name: 'AbortError' }))).toBe(true); + expect(isAbortError(Object.assign(new Error('operation aborted'), { code: 'ABORT_ERR' }))).toBe(true); + expect(isAbortError(new Error('ordinary failure'))).toBe(false); + }); + + it('throws when the provided signal is already aborted', () => { + const controller = new AbortController(); + controller.abort(); + + expect(() => throwIfAborted(controller.signal)).toThrow(/Aborted/); + }); + + it('links a child controller to a parent signal and removes the listener on dispose', () => { + const parent = new AbortController(); + const child = linkAbortSignal(parent.signal); + + expect(child.controller.signal.aborted).toBe(false); + parent.abort(); + expect(child.controller.signal.aborted).toBe(true); + + const removeSpy = vi.spyOn(parent.signal, 'removeEventListener'); + child.dispose(); + expect(removeSpy).toHaveBeenCalledWith('abort', expect.any(Function)); + }); +}); diff --git a/packages/cli/src/context/core/config-reference.test.ts b/packages/cli/test/context/core/config-reference.test.ts similarity index 96% rename from packages/cli/src/context/core/config-reference.test.ts rename to packages/cli/test/context/core/config-reference.test.ts index f12d0bd9..c4b7c848 100644 --- a/packages/cli/src/context/core/config-reference.test.ts +++ b/packages/cli/test/context/core/config-reference.test.ts @@ -2,7 +2,7 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { resolveKtxConfigReference, resolveKtxHomePath } from './config-reference.js'; +import { resolveKtxConfigReference, resolveKtxHomePath } from '../../../src/context/core/config-reference.js'; describe('KTX config references', () => { it('resolves env references without returning empty values', () => { diff --git a/packages/cli/src/context/core/git.service.assert-worktree-clean.test.ts b/packages/cli/test/context/core/git.service.assert-worktree-clean.test.ts similarity index 92% rename from packages/cli/src/context/core/git.service.assert-worktree-clean.test.ts rename to packages/cli/test/context/core/git.service.assert-worktree-clean.test.ts index db7d7bd3..18ee0b74 100644 --- a/packages/cli/src/context/core/git.service.assert-worktree-clean.test.ts +++ b/packages/cli/test/context/core/git.service.assert-worktree-clean.test.ts @@ -3,9 +3,9 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { SimpleGit } from 'simple-git'; -import type { KtxCoreConfig } from './config.js'; -import { createSimpleGit } from './git-env.js'; -import { GitService } from './git.service.js'; +import type { KtxCoreConfig } from '../../../src/context/core/config.js'; +import { createSimpleGit } from '../../../src/context/core/git-env.js'; +import { GitService } from '../../../src/context/core/git.service.js'; describe('GitService.assertWorktreeClean', () => { let workdir: string; diff --git a/packages/cli/src/context/core/git.service.delete-directories.test.ts b/packages/cli/test/context/core/git.service.delete-directories.test.ts similarity index 92% rename from packages/cli/src/context/core/git.service.delete-directories.test.ts rename to packages/cli/test/context/core/git.service.delete-directories.test.ts index b6156349..1ccd6b95 100644 --- a/packages/cli/src/context/core/git.service.delete-directories.test.ts +++ b/packages/cli/test/context/core/git.service.delete-directories.test.ts @@ -3,9 +3,9 @@ import { mkdir, mkdtemp, readdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { SimpleGit } from 'simple-git'; -import type { KtxCoreConfig } from './config.js'; -import { createSimpleGit } from './git-env.js'; -import { GitService } from './git.service.js'; +import type { KtxCoreConfig } from '../../../src/context/core/config.js'; +import { createSimpleGit } from '../../../src/context/core/git-env.js'; +import { GitService } from '../../../src/context/core/git.service.js'; describe('GitService.deleteDirectories', () => { let workdir: string; diff --git a/packages/cli/src/context/core/git.service.patch.test.ts b/packages/cli/test/context/core/git.service.patch.test.ts similarity index 96% rename from packages/cli/src/context/core/git.service.patch.test.ts rename to packages/cli/test/context/core/git.service.patch.test.ts index de1ccb9f..160c64aa 100644 --- a/packages/cli/src/context/core/git.service.patch.test.ts +++ b/packages/cli/test/context/core/git.service.patch.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { GitService } from './git.service.js'; +import { GitService } from '../../../src/context/core/git.service.js'; async function makeGit() { const homeDir = await mkdtemp(join(tmpdir(), 'ktx-git-patch-')); diff --git a/packages/cli/src/context/core/git.service.reset-hard.test.ts b/packages/cli/test/context/core/git.service.reset-hard.test.ts similarity index 90% rename from packages/cli/src/context/core/git.service.reset-hard.test.ts rename to packages/cli/test/context/core/git.service.reset-hard.test.ts index e688b8b3..e68a3dbe 100644 --- a/packages/cli/src/context/core/git.service.reset-hard.test.ts +++ b/packages/cli/test/context/core/git.service.reset-hard.test.ts @@ -3,9 +3,9 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { SimpleGit } from 'simple-git'; -import type { KtxCoreConfig } from './config.js'; -import { createSimpleGit } from './git-env.js'; -import { GitService } from './git.service.js'; +import type { KtxCoreConfig } from '../../../src/context/core/config.js'; +import { createSimpleGit } from '../../../src/context/core/git-env.js'; +import { GitService } from '../../../src/context/core/git.service.js'; describe('GitService.resetHardTo', () => { let workdir: string; diff --git a/packages/cli/src/context/core/git.service.test.ts b/packages/cli/test/context/core/git.service.test.ts similarity index 99% rename from packages/cli/src/context/core/git.service.test.ts rename to packages/cli/test/context/core/git.service.test.ts index e8a5aa73..8b19ed09 100644 --- a/packages/cli/src/context/core/git.service.test.ts +++ b/packages/cli/test/context/core/git.service.test.ts @@ -2,8 +2,8 @@ import { mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import type { KtxCoreConfig } from './config.js'; -import { GitService } from './git.service.js'; +import type { KtxCoreConfig } from '../../../src/context/core/config.js'; +import { GitService } from '../../../src/context/core/git.service.js'; // These tests drive a real git repo inside a temp directory — simple-git shells out to the // system `git` binary. They are fast enough to run as unit tests and catch real issues that diff --git a/packages/cli/src/context/core/session-worktree.service.test.ts b/packages/cli/test/context/core/session-worktree.service.test.ts similarity index 95% rename from packages/cli/src/context/core/session-worktree.service.test.ts rename to packages/cli/test/context/core/session-worktree.service.test.ts index 3cb66742..19f899e4 100644 --- a/packages/cli/src/context/core/session-worktree.service.test.ts +++ b/packages/cli/test/context/core/session-worktree.service.test.ts @@ -2,9 +2,9 @@ import { mkdtemp, realpath, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { KtxCoreConfig } from './config.js'; -import { GitService } from './git.service.js'; -import { SessionWorktreeService, type WorktreeConfigPort } from './session-worktree.service.js'; +import type { KtxCoreConfig } from '../../../src/context/core/config.js'; +import { GitService } from '../../../src/context/core/git.service.js'; +import { SessionWorktreeService, type WorktreeConfigPort } from '../../../src/context/core/session-worktree.service.js'; interface TestWorktreeConfig extends WorktreeConfigPort { workdir?: string; diff --git a/packages/cli/src/context/daemon/semantic-layer-compute.test.ts b/packages/cli/test/context/daemon/semantic-layer-compute.test.ts similarity index 99% rename from packages/cli/src/context/daemon/semantic-layer-compute.test.ts rename to packages/cli/test/context/daemon/semantic-layer-compute.test.ts index dac37ef4..e6bbddbc 100644 --- a/packages/cli/src/context/daemon/semantic-layer-compute.test.ts +++ b/packages/cli/test/context/daemon/semantic-layer-compute.test.ts @@ -1,7 +1,7 @@ import { once } from 'node:events'; import { createServer } from 'node:http'; import { describe, expect, it, vi } from 'vitest'; -import { createHttpSemanticLayerComputePort, createPythonSemanticLayerComputePort } from './semantic-layer-compute.js'; +import { createHttpSemanticLayerComputePort, createPythonSemanticLayerComputePort } from '../../../src/context/daemon/semantic-layer-compute.js'; const source = { name: 'orders', diff --git a/packages/cli/src/context/index-sync/reindex.test.ts b/packages/cli/test/context/index-sync/reindex.test.ts similarity index 95% rename from packages/cli/src/context/index-sync/reindex.test.ts rename to packages/cli/test/context/index-sync/reindex.test.ts index 90c6a178..fe7698ba 100644 --- a/packages/cli/src/context/index-sync/reindex.test.ts +++ b/packages/cli/test/context/index-sync/reindex.test.ts @@ -2,10 +2,10 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; -import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../context/project/project.js'; -import { SqliteKnowledgeIndex } from '../wiki/sqlite-knowledge-index.js'; -import { reindexLocalIndexes } from './reindex.js'; +import type { KtxEmbeddingPort } from '../../../src/context/core/embedding.js'; +import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../../src/context/project/project.js'; +import { SqliteKnowledgeIndex } from '../../../src/context/wiki/sqlite-knowledge-index.js'; +import { reindexLocalIndexes } from '../../../src/context/index-sync/reindex.js'; class FakeEmbeddingPort implements KtxEmbeddingPort { readonly maxBatchSize = 8; diff --git a/packages/cli/src/context/ingest/action-identity.test.ts b/packages/cli/test/context/ingest/action-identity.test.ts similarity index 96% rename from packages/cli/src/context/ingest/action-identity.test.ts rename to packages/cli/test/context/ingest/action-identity.test.ts index 725a1d99..e4baaa70 100644 --- a/packages/cli/src/context/ingest/action-identity.test.ts +++ b/packages/cli/test/context/ingest/action-identity.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { actionTargetConnectionId, memoryActionIdentity } from './action-identity.js'; +import { actionTargetConnectionId, memoryActionIdentity } from '../../../src/context/ingest/action-identity.js'; describe('memory action target identity', () => { it('keys SL actions by target connection and wiki actions by run connection', () => { diff --git a/packages/cli/src/context/ingest/adapters/dbt-descriptions/parse-schema.test.ts b/packages/cli/test/context/ingest/adapters/dbt-descriptions/parse-schema.test.ts similarity index 97% rename from packages/cli/src/context/ingest/adapters/dbt-descriptions/parse-schema.test.ts rename to packages/cli/test/context/ingest/adapters/dbt-descriptions/parse-schema.test.ts index f29cab06..8de52354 100644 --- a/packages/cli/src/context/ingest/adapters/dbt-descriptions/parse-schema.test.ts +++ b/packages/cli/test/context/ingest/adapters/dbt-descriptions/parse-schema.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { parseDbtSchemaFile, parseDbtSchemaFiles } from './parse-schema.js'; +import { parseDbtSchemaFile, parseDbtSchemaFiles } from '../../../../../src/context/ingest/adapters/dbt-descriptions/parse-schema.js'; describe('dbt descriptions schema parser', () => { it('resolves shared dbt vars and defaults before parsing schema YAML', () => { diff --git a/packages/cli/src/context/ingest/adapters/dbt/chunk.test.ts b/packages/cli/test/context/ingest/adapters/dbt/chunk.test.ts similarity index 94% rename from packages/cli/src/context/ingest/adapters/dbt/chunk.test.ts rename to packages/cli/test/context/ingest/adapters/dbt/chunk.test.ts index 6eece2ac..ffda2887 100644 --- a/packages/cli/src/context/ingest/adapters/dbt/chunk.test.ts +++ b/packages/cli/test/context/ingest/adapters/dbt/chunk.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { chunkDbtProject } from './chunk.js'; +import { chunkDbtProject } from '../../../../../src/context/ingest/adapters/dbt/chunk.js'; describe('chunkDbtProject', () => { const diffSet = (modified: string[]) => ({ added: [], modified, deleted: [], unchanged: [] }); diff --git a/packages/cli/src/context/ingest/adapters/dbt/dbt.adapter.test.ts b/packages/cli/test/context/ingest/adapters/dbt/dbt.adapter.test.ts similarity index 92% rename from packages/cli/src/context/ingest/adapters/dbt/dbt.adapter.test.ts rename to packages/cli/test/context/ingest/adapters/dbt/dbt.adapter.test.ts index 2851318e..692e1e7f 100644 --- a/packages/cli/src/context/ingest/adapters/dbt/dbt.adapter.test.ts +++ b/packages/cli/test/context/ingest/adapters/dbt/dbt.adapter.test.ts @@ -2,8 +2,8 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import type { SourceAdapter } from '../../types.js'; -import { DbtSourceAdapter } from './dbt.adapter.js'; +import type { SourceAdapter } from '../../../../../src/context/ingest/types.js'; +import { DbtSourceAdapter } from '../../../../../src/context/ingest/adapters/dbt/dbt.adapter.js'; describe('DbtSourceAdapter', () => { let stagedDir: string; diff --git a/packages/cli/src/context/ingest/adapters/dbt/fetch.test.ts b/packages/cli/test/context/ingest/adapters/dbt/fetch.test.ts similarity index 94% rename from packages/cli/src/context/ingest/adapters/dbt/fetch.test.ts rename to packages/cli/test/context/ingest/adapters/dbt/fetch.test.ts index eebff7c6..98ed1804 100644 --- a/packages/cli/src/context/ingest/adapters/dbt/fetch.test.ts +++ b/packages/cli/test/context/ingest/adapters/dbt/fetch.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { fetchDbtRepo } from './fetch.js'; +import { fetchDbtRepo } from '../../../../../src/context/ingest/adapters/dbt/fetch.js'; describe('fetchDbtRepo', () => { let tempDir: string; diff --git a/packages/cli/src/context/ingest/adapters/dbt/parse.test.ts b/packages/cli/test/context/ingest/adapters/dbt/parse.test.ts similarity index 73% rename from packages/cli/src/context/ingest/adapters/dbt/parse.test.ts rename to packages/cli/test/context/ingest/adapters/dbt/parse.test.ts index f373fd5b..33ff8b0e 100644 --- a/packages/cli/src/context/ingest/adapters/dbt/parse.test.ts +++ b/packages/cli/test/context/ingest/adapters/dbt/parse.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { normalizeDbtPath } from './parse.js'; +import { normalizeDbtPath } from '../../../../../src/context/ingest/adapters/dbt/parse.js'; describe('normalizeDbtPath', () => { it('normalizes Windows separators to POSIX separators', () => { diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts similarity index 77% rename from packages/cli/src/context/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts rename to packages/cli/test/context/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts index b9ee73b3..e3222ab5 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts +++ b/packages/cli/test/context/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { BigQueryHistoricSqlQueryHistoryReader } from './bigquery-query-history-reader.js'; -import { HistoricSqlGrantsMissingError } from './errors.js'; +import { BigQueryHistoricSqlQueryHistoryReader } from '../../../../../src/context/ingest/adapters/historic-sql/bigquery-query-history-reader.js'; +import { HistoricSqlGrantsMissingError } from '../../../../../src/context/ingest/adapters/historic-sql/errors.js'; interface FakeQueryResult { headers: string[]; @@ -91,7 +91,10 @@ describe('BigQueryHistoricSqlQueryHistoryReader', () => { 40, 0.05, null, - JSON.stringify([{ user: 'analyst@example.test', executions: 1 }]), + JSON.stringify([ + { user: 'svc-loader@example.test', executions: 40 }, + { user: 'analyst@example.test', executions: 2 }, + ]), ], ], totalRows: 1, @@ -103,15 +106,25 @@ describe('BigQueryHistoricSqlQueryHistoryReader', () => { for await (const row of reader.fetchAggregated( client, { start: new Date('2026-02-10T00:00:00.000Z'), end: new Date('2026-05-11T00:00:00.000Z') }, - { dialect: 'bigquery', minExecutions: 5, windowDays: 90, enabledTables: [], filters: { dropTrivialProbes: true }, redactionPatterns: [], staleArchiveAfterDays: 90 }, + { dialect: 'bigquery', minExecutions: 5, windowDays: 90, enabledTables: [], enabledSchemas: [], modeledTableCatalog: [], scopeFloorWarnings: [], filters: { dropTrivialProbes: true }, redactionPatterns: [], staleArchiveAfterDays: 90 }, )) { rows.push(row); } const sql = firstQuery(client); + expect(sql).toContain('WITH filtered_jobs AS'); + expect(sql).toContain('query_info.query_hashes.normalized_literals'); + expect(sql).toContain('TO_HEX(SHA256(query))'); + expect(sql).toContain('AS template_id'); + expect(sql).toContain('template_stats AS'); + expect(sql).toContain('template_users AS'); expect(sql).toContain('COUNT(*) AS executions'); expect(sql).toContain('COUNT(DISTINCT user_email) AS distinct_users'); - expect(sql).toContain('GROUP BY query_hash'); + expect(sql).toContain('GROUP BY template_id'); + expect(sql).toContain('GROUP BY template_id, user_email'); + expect(sql).toContain('ORDER BY users.executions DESC'); + expect(sql).not.toMatch(/\bquery_hash\b/); + expect(sql).not.toContain('LIMIT 5'); expect(sql).toContain('HAVING COUNT(*) >= 5'); expect(rows).toMatchObject([ { @@ -120,7 +133,10 @@ describe('BigQueryHistoricSqlQueryHistoryReader', () => { executions: 42, errorRate: 0.05, }, - topUsers: [{ user: 'analyst@example.test', executions: 1 }], + topUsers: [ + { user: 'svc-loader@example.test', executions: 40 }, + { user: 'analyst@example.test', executions: 2 }, + ], }, ]); }); @@ -137,6 +153,9 @@ describe('BigQueryHistoricSqlQueryHistoryReader', () => { minExecutions: 5, windowDays: 90, enabledTables: [], + enabledSchemas: [], + modeledTableCatalog: [], + scopeFloorWarnings: [], filters: { dropTrivialProbes: true }, redactionPatterns: [], staleArchiveAfterDays: 90, diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/buckets.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/buckets.test.ts similarity index 95% rename from packages/cli/src/context/ingest/adapters/historic-sql/buckets.test.ts rename to packages/cli/test/context/ingest/adapters/historic-sql/buckets.test.ts index 78dc2859..253add1e 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/buckets.test.ts +++ b/packages/cli/test/context/ingest/adapters/historic-sql/buckets.test.ts @@ -6,7 +6,7 @@ import { bucketFrequency, bucketP95Runtime, bucketRecency, -} from './buckets.js'; +} from '../../../../../src/context/ingest/adapters/historic-sql/buckets.js'; describe('historic-sql bucket helpers', () => { it('uses stable execution buckets', () => { diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/chunk-unified.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/chunk-unified.test.ts similarity index 90% rename from packages/cli/src/context/ingest/adapters/historic-sql/chunk-unified.test.ts rename to packages/cli/test/context/ingest/adapters/historic-sql/chunk-unified.test.ts index d8c0187f..f3d7d293 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/chunk-unified.test.ts +++ b/packages/cli/test/context/ingest/adapters/historic-sql/chunk-unified.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { chunkHistoricSqlUnifiedStagedDir, describeHistoricSqlUnifiedScope } from './chunk-unified.js'; +import { chunkHistoricSqlUnifiedStagedDir, describeHistoricSqlUnifiedScope } from '../../../../../src/context/ingest/adapters/historic-sql/chunk-unified.js'; async function tempDir(): Promise { return mkdtemp(join(tmpdir(), 'historic-sql-unified-chunk-')); @@ -30,6 +30,7 @@ async function writeUnifiedStagedDir(root: string): Promise { }); await writeJson(root, 'tables/public.orders.json', { table: 'public.orders', + tableRef: { catalog: null, db: 'public', name: 'orders' }, stats: { executionsBucket: '10-100', distinctUsersBucket: '2-5', @@ -46,7 +47,10 @@ async function writeUnifiedStagedDir(root: string): Promise { { id: 'orders', canonicalSql: 'select * from public.orders join public.customers on true', - tablesTouched: ['public.orders', 'public.customers'], + tablesTouched: [ + { catalog: null, db: 'public', name: 'orders' }, + { catalog: null, db: 'public', name: 'customers' }, + ], executionsBucket: '10-100', distinctUsersBucket: '2-5', dialect: 'postgres', @@ -58,7 +62,10 @@ async function writeUnifiedStagedDir(root: string): Promise { { id: 'orders', canonicalSql: 'select * from public.orders join public.customers on true', - tablesTouched: ['public.orders', 'public.customers'], + tablesTouched: [ + { catalog: null, db: 'public', name: 'orders' }, + { catalog: null, db: 'public', name: 'customers' }, + ], executionsBucket: '10-100', distinctUsersBucket: '2-5', dialect: 'postgres', @@ -155,7 +162,10 @@ describe('chunkHistoricSqlUnifiedStagedDir', () => { { id: 'line-items', canonicalSql: 'select * from public.orders join public.line_items on true', - tablesTouched: ['public.orders', 'public.line_items'], + tablesTouched: [ + { catalog: null, db: 'public', name: 'orders' }, + { catalog: null, db: 'public', name: 'line_items' }, + ], executionsBucket: '10-100', distinctUsersBucket: '2-5', dialect: 'postgres', diff --git a/packages/cli/test/context/ingest/adapters/historic-sql/connection-dialect.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/connection-dialect.test.ts new file mode 100644 index 00000000..935bab8e --- /dev/null +++ b/packages/cli/test/context/ingest/adapters/historic-sql/connection-dialect.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest'; +import { + historicSqlDialectForConnectionDriver, + queryHistoryDialectForConnection, +} from '../../../../../src/context/ingest/adapters/historic-sql/connection-dialect.js'; + +describe('queryHistoryDialectForConnection', () => { + it.each([ + ['postgres', 'postgres'], + ['bigquery', 'bigquery'], + ['snowflake', 'snowflake'], + ] as const)('returns %s when query history is enabled', (driver, dialect) => { + expect(queryHistoryDialectForConnection({ driver, context: { queryHistory: { enabled: true } } })).toBe(dialect); + }); + + it.each(['sqlite', 'mysql', 'clickhouse', 'sqlserver'] as const)( + 'returns null for %s because no historic-SQL reader is registered', + (driver) => { + expect(queryHistoryDialectForConnection({ driver, context: { queryHistory: { enabled: true } } })).toBeNull(); + }, + ); + + it('returns null when query history is disabled', () => { + expect(queryHistoryDialectForConnection({ driver: 'postgres', context: { queryHistory: { enabled: false } } })).toBeNull(); + }); +}); + +describe('historicSqlDialectForConnectionDriver', () => { + it('resolves the dialect from driver capability even when query history is disabled', () => { + expect( + historicSqlDialectForConnectionDriver({ driver: 'postgres', context: { queryHistory: { enabled: false } } }), + ).toBe('postgres'); + }); + + it('resolves the dialect when no query-history context is present', () => { + expect(historicSqlDialectForConnectionDriver({ driver: 'bigquery' })).toBe('bigquery'); + }); + + it('returns null for drivers without a historic-SQL reader', () => { + expect(historicSqlDialectForConnectionDriver({ driver: 'mysql', context: { queryHistory: { enabled: true } } })).toBeNull(); + }); +}); diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/detect.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/detect.test.ts similarity index 91% rename from packages/cli/src/context/ingest/adapters/historic-sql/detect.test.ts rename to packages/cli/test/context/ingest/adapters/historic-sql/detect.test.ts index 9ad3cf39..4baef647 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/detect.test.ts +++ b/packages/cli/test/context/ingest/adapters/historic-sql/detect.test.ts @@ -2,8 +2,8 @@ import { mkdir, mkdtemp, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { detectHistoricSqlStagedDir } from './detect.js'; -import { HISTORIC_SQL_SOURCE_KEY, stagedManifestSchema } from './types.js'; +import { detectHistoricSqlStagedDir } from '../../../../../src/context/ingest/adapters/historic-sql/detect.js'; +import { HISTORIC_SQL_SOURCE_KEY, stagedManifestSchema } from '../../../../../src/context/ingest/adapters/historic-sql/types.js'; async function tempDir(): Promise { return mkdtemp(join(tmpdir(), 'historic-sql-detect-')); diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/evidence-tool.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/evidence-tool.test.ts similarity index 59% rename from packages/cli/src/context/ingest/adapters/historic-sql/evidence-tool.test.ts rename to packages/cli/test/context/ingest/adapters/historic-sql/evidence-tool.test.ts index ae16d105..3638bb1b 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/evidence-tool.test.ts +++ b/packages/cli/test/context/ingest/adapters/historic-sql/evidence-tool.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; import { asSchema } from 'ai'; -import { createEmitHistoricSqlEvidenceTool } from './evidence-tool.js'; +import { createEmitHistoricSqlEvidenceTool } from '../../../../../src/context/ingest/adapters/historic-sql/evidence-tool.js'; describe('emit_historic_sql_evidence tool', () => { it('exposes an AI SDK v6 tool input schema with top-level object type', async () => { @@ -11,15 +11,14 @@ describe('emit_historic_sql_evidence tool', () => { }); }); - it('writes table usage evidence to the ignored run evidence directory', async () => { - const writeFile = vi.fn(async () => ({ success: true, commitHash: null })); + it('writes table usage evidence using the work unit allowed raw paths', async () => { + const writeFile = vi.fn(async (_path: string, _body: string) => ({ success: true, commitHash: null })); const tool = createEmitHistoricSqlEvidenceTool(); const result = await tool.execute!( { kind: 'table_usage', table: 'public.orders', - rawPath: 'tables/public.orders.json', usage: { narrative: 'Orders are repeatedly queried by paid status.', frequencyTier: 'high', @@ -36,6 +35,7 @@ describe('emit_historic_sql_evidence tool', () => { connectionId: 'warehouse', session: { ingest: { runId: 'run-1', jobId: 'job-1', syncId: 'sync-1', sourceKey: 'historic-sql' }, + allowedRawPaths: new Set(['tables/public.orders.json']), configService: { writeFile }, }, }, @@ -45,12 +45,53 @@ describe('emit_historic_sql_evidence tool', () => { expect(result).toBe('Recorded historic-SQL table_usage evidence for public.orders.'); expect(writeFile).toHaveBeenCalledWith( '.ktx/ingest-evidence/historic-sql/run-1/historic-sql-table-public-orders.json', - expect.stringContaining('"kind": "table_usage"'), + expect.stringContaining('"rawPaths"'), 'System User', 'system@example.com', 'Record historic-SQL evidence: historic-sql-table-public-orders', { skipLock: true }, ); + expect(writeFile).toHaveBeenCalledWith( + expect.any(String), + expect.stringContaining('tables/public.orders.json'), + expect.any(String), + expect.any(String), + expect.any(String), + expect.any(Object), + ); + }); + + it('rejects calls without a WorkUnit raw file context', async () => { + const tool = createEmitHistoricSqlEvidenceTool(); + + await expect( + tool.execute!( + { + kind: 'pattern', + pattern: { + slug: 'orders', + title: 'Orders', + narrative: 'Orders pattern.', + definitionSql: 'select * from public.orders', + tablesInvolved: ['public.orders'], + slRefs: ['orders'], + constituentTemplateIds: ['pg:1'], + }, + }, + { + toolCallId: 'call-1', + messages: [], + abortSignal: new AbortController().signal, + experimental_context: { + connectionId: 'warehouse', + session: { + ingest: { runId: 'run-1', jobId: 'job-1', syncId: 'sync-1', sourceKey: 'historic-sql' }, + configService: { writeFile: vi.fn() }, + }, + }, + } as never, + ), + ).resolves.toContain('emit_historic_sql_evidence requires a WorkUnit context'); }); it('rejects non-historic ingest sessions', async () => { @@ -60,7 +101,6 @@ describe('emit_historic_sql_evidence tool', () => { tool.execute!( { kind: 'pattern', - rawPath: 'patterns-input.json', pattern: { slug: 'orders', title: 'Orders', @@ -79,6 +119,7 @@ describe('emit_historic_sql_evidence tool', () => { connectionId: 'warehouse', session: { ingest: { runId: 'run-1', jobId: 'job-1', syncId: 'sync-1', sourceKey: 'notion' }, + allowedRawPaths: new Set(['patterns-input/part-0001.json']), configService: { writeFile: vi.fn() }, }, }, diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/evidence.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/evidence.test.ts similarity index 91% rename from packages/cli/src/context/ingest/adapters/historic-sql/evidence.test.ts rename to packages/cli/test/context/ingest/adapters/historic-sql/evidence.test.ts index 8858ed37..1d1d7f6c 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/evidence.test.ts +++ b/packages/cli/test/context/ingest/adapters/historic-sql/evidence.test.ts @@ -4,7 +4,7 @@ import { historicSqlEvidencePath, historicSqlPatternEvidenceSchema, historicSqlTableUsageEvidenceSchema, -} from './evidence.js'; +} from '../../../../../src/context/ingest/adapters/historic-sql/evidence.js'; describe('historic-sql evidence contracts', () => { it('validates table usage evidence emitted by table digest WorkUnits', () => { @@ -12,7 +12,7 @@ describe('historic-sql evidence contracts', () => { kind: 'table_usage', connectionId: 'warehouse', table: 'public.orders', - rawPath: 'tables/public.orders.json', + rawPaths: ['tables/public.orders.json'], usage: { narrative: 'Orders are repeatedly queried for paid/refunded lifecycle analysis.', frequencyTier: 'high', @@ -32,7 +32,7 @@ describe('historic-sql evidence contracts', () => { historicSqlEvidenceEnvelopeSchema.parse({ kind: 'pattern', connectionId: 'warehouse', - rawPath: 'patterns-input.json', + rawPaths: ['patterns-input/part-0001.json'], pattern: { slug: 'order-lifecycle-analysis', title: 'Order Lifecycle Analysis', diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/historic-sql.adapter.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/historic-sql.adapter.test.ts similarity index 84% rename from packages/cli/src/context/ingest/adapters/historic-sql/historic-sql.adapter.test.ts rename to packages/cli/test/context/ingest/adapters/historic-sql/historic-sql.adapter.test.ts index 80df5c26..dcd00d32 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/historic-sql.adapter.test.ts +++ b/packages/cli/test/context/ingest/adapters/historic-sql/historic-sql.adapter.test.ts @@ -2,10 +2,10 @@ import { mkdtemp } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js'; -import type { SourceAdapter } from '../../types.js'; -import { HistoricSqlSourceAdapter } from './historic-sql.adapter.js'; -import type { HistoricSqlReader } from './types.js'; +import type { SqlAnalysisPort } from '../../../../../src/context/sql-analysis/ports.js'; +import type { SourceAdapter } from '../../../../../src/context/ingest/types.js'; +import { HistoricSqlSourceAdapter } from '../../../../../src/context/ingest/adapters/historic-sql/historic-sql.adapter.js'; +import type { HistoricSqlReader } from '../../../../../src/context/ingest/adapters/historic-sql/types.js'; async function tempDir(): Promise { return mkdtemp(join(tmpdir(), 'historic-sql-adapter-')); @@ -76,7 +76,10 @@ describe('HistoricSqlSourceAdapter', () => { [ 'pg:1', { - tablesTouched: ['public.orders', 'public.customers'], + tablesTouched: [ + { catalog: null, db: 'public', name: 'orders' }, + { catalog: null, db: 'public', name: 'customers' }, + ], columnsByClause: { select: ['status'], join: ['customer_id', 'id'], groupBy: ['status'] }, }, ], diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts similarity index 83% rename from packages/cli/src/context/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts rename to packages/cli/test/context/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts index a443f995..e818f1c9 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts +++ b/packages/cli/test/context/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts @@ -2,15 +2,15 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import YAML from 'yaml'; -import type { AgentRunnerPort, RunLoopParams } from '../../../../context/llm/runtime-port.js'; -import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../../../context/project/project.js'; -import type { SqlAnalysisBatchItem, SqlAnalysisBatchResult, SqlAnalysisDialect, SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js'; -import { searchLocalSlSources } from '../../../sl/local-sl.js'; -import { searchLocalKnowledgePages } from '../../../wiki/local-knowledge.js'; -import { runLocalIngest } from '../../local-ingest.js'; +import type { AgentRunnerPort, RunLoopParams } from '../../../../../src/context/llm/runtime-port.js'; +import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../../../../src/context/project/project.js'; +import type { SqlAnalysisBatchItem, SqlAnalysisBatchResult, SqlAnalysisDialect, SqlAnalysisPort } from '../../../../../src/context/sql-analysis/ports.js'; +import { searchLocalSlSources } from '../../../../../src/context/sl/local-sl.js'; +import { searchLocalKnowledgePages } from '../../../../../src/context/wiki/local-knowledge.js'; +import { runLocalIngest } from '../../../../../src/context/ingest/local-ingest.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { HistoricSqlSourceAdapter } from './historic-sql.adapter.js'; -import type { AggregatedTemplate, HistoricSqlReader, HistoricSqlUnifiedPullConfig } from './types.js'; +import { HistoricSqlSourceAdapter } from '../../../../../src/context/ingest/adapters/historic-sql/historic-sql.adapter.js'; +import type { AggregatedTemplate, HistoricSqlReader, HistoricSqlUnifiedPullConfig } from '../../../../../src/context/ingest/adapters/historic-sql/types.js'; class AcceptanceHistoricSqlReader implements HistoricSqlReader { async probe() { @@ -57,7 +57,6 @@ class HistoricSqlAcceptanceAgentRunner implements AgentRunnerPort { const result = await emitEvidence.execute({ kind: 'table_usage', table: 'public.orders', - rawPath: 'tables/public.orders.json', usage: { narrative: 'Analysts repeatedly inspect paid order lifecycle by customer segment.', frequencyTier: 'high', @@ -76,7 +75,6 @@ class HistoricSqlAcceptanceAgentRunner implements AgentRunnerPort { const result = await emitEvidence.execute({ kind: 'table_usage', table: 'public.customers', - rawPath: 'tables/public.customers.json', usage: { narrative: 'Customers provide segment context for paid order lifecycle analysis.', frequencyTier: 'mid', @@ -94,7 +92,6 @@ class HistoricSqlAcceptanceAgentRunner implements AgentRunnerPort { if (params.telemetryTags.unitKey === 'historic-sql-patterns-part-0001') { const result = await emitEvidence.execute({ kind: 'pattern', - rawPath: 'patterns-input/part-0001.json', pattern: { slug: 'paid-order-lifecycle', title: 'Paid Order Lifecycle', @@ -129,7 +126,10 @@ function acceptanceSqlAnalysis(): SqlAnalysisPort { items.map((item) => [ item.id, { - tablesTouched: ['public.orders', 'public.customers'], + tablesTouched: [ + { catalog: null, db: 'public', name: 'orders' }, + { catalog: null, db: 'public', name: 'customers' }, + ], columnsByClause: { select: ['status', 'segment'], where: ['status'], @@ -257,6 +257,33 @@ describe('historic-SQL local ingest retrieval acceptance', () => { ]), ); + // Regression for KLO-698: the bundle report's provenance rows must + // attribute the table-usage merges and pattern-page writes back to + // their raw files instead of falling through as `actionType: 'skipped'` + // with null artifact metadata. + const provenanceRows = result.report.body.provenanceRows; + const nonSkipped = provenanceRows.filter((row) => row.actionType !== 'skipped'); + expect(nonSkipped).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + rawPath: 'tables/public.orders.json', + artifactKind: 'sl', + artifactKey: 'orders', + }), + expect.objectContaining({ + rawPath: 'tables/public.customers.json', + artifactKind: 'sl', + artifactKey: 'customers', + }), + expect.objectContaining({ + rawPath: 'patterns-input/part-0001.json', + artifactKind: 'wiki', + artifactKey: 'historic-sql-paid-order-lifecycle', + actionType: 'wiki_written', + }), + ]), + ); + await expect(readFile(join(project.projectDir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8')).resolves .toContain('Analysts repeatedly inspect paid order lifecycle by customer segment.'); await expect(readFile(join(project.projectDir, 'wiki/global/historic-sql-paid-order-lifecycle.md'), 'utf-8')) diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/pattern-inputs.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/pattern-inputs.test.ts similarity index 83% rename from packages/cli/src/context/ingest/adapters/historic-sql/pattern-inputs.test.ts rename to packages/cli/test/context/ingest/adapters/historic-sql/pattern-inputs.test.ts index d37ed193..780fcf3d 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/pattern-inputs.test.ts +++ b/packages/cli/test/context/ingest/adapters/historic-sql/pattern-inputs.test.ts @@ -4,16 +4,23 @@ import { isHistoricSqlPatternInputShardPath, serializedStagedPatternsInputByteLength, splitHistoricSqlPatternInputs, -} from './pattern-inputs.js'; -import type { StagedPatternsInput } from './types.js'; +} from '../../../../../src/context/ingest/adapters/historic-sql/pattern-inputs.js'; +import type { StagedPatternsInput } from '../../../../../src/context/ingest/adapters/historic-sql/types.js'; type PatternTemplate = StagedPatternsInput['templates'][number]; +function tableRef(value: string): { catalog: string | null; db: string | null; name: string } { + const parts = value.split('.'); + if (parts.length === 3) return { catalog: parts[0]!, db: parts[1]!, name: parts[2]! }; + if (parts.length === 2) return { catalog: null, db: parts[0]!, name: parts[1]! }; + return { catalog: null, db: null, name: value }; +} + function template(id: string, tablesTouched: string[], canonicalSql = 'select 1'): PatternTemplate { return { id, canonicalSql, - tablesTouched, + tablesTouched: tablesTouched.map(tableRef), executionsBucket: '10-100', distinctUsersBucket: '2-5', dialect: 'postgres', @@ -32,7 +39,7 @@ describe('historic-SQL pattern input sharding', () => { ], }; - const result = splitHistoricSqlPatternInputs(input, { maxBytes: 760 }); + const result = splitHistoricSqlPatternInputs(input, { maxBytes: 1200 }); expect(result.auditInput.templates.map((entry) => entry.id)).toEqual([ 'orders-customers-1', @@ -51,7 +58,7 @@ describe('historic-SQL pattern input sharding', () => { 'orders-customers-1', 'orders-customers-2', ]); - expect(result.shards.every((shard) => shard.byteLength <= 760)).toBe(true); + expect(result.shards.every((shard) => shard.byteLength <= 1200)).toBe(true); expect(result.shards.flatMap((shard) => shard.input.templates).some((entry) => entry.id === 'single-table-orders')).toBe(false); expect(result.warnings).toEqual([]); }); diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts similarity index 95% rename from packages/cli/src/context/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts rename to packages/cli/test/context/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts index a91171cd..4c9fc7bb 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts +++ b/packages/cli/test/context/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts @@ -3,8 +3,8 @@ import { HistoricSqlExtensionMissingError, HistoricSqlGrantsMissingError, HistoricSqlVersionUnsupportedError, -} from './errors.js'; -import { PostgresPgssReader } from './postgres-pgss-reader.js'; +} from '../../../../../src/context/ingest/adapters/historic-sql/errors.js'; +import { PostgresPgssReader } from '../../../../../src/context/ingest/adapters/historic-sql/postgres-pgss-reader.js'; interface FakeQueryResult { headers: string[]; @@ -215,7 +215,7 @@ describe('PostgresPgssReader aggregate path', () => { for await (const row of reader.fetchAggregated( { executeQuery }, { start: new Date('2026-02-10T00:00:00.000Z'), end: new Date('2026-05-11T00:00:00.000Z') }, - { dialect: 'postgres', minExecutions: 5, enabledTables: [], filters: { dropTrivialProbes: true }, redactionPatterns: [], staleArchiveAfterDays: 90 }, + { dialect: 'postgres', minExecutions: 5, enabledTables: [], enabledSchemas: [], modeledTableCatalog: [], scopeFloorWarnings: [], filters: { dropTrivialProbes: true }, redactionPatterns: [], staleArchiveAfterDays: 90 }, )) { rows.push(row); } diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/projection.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/projection.test.ts similarity index 97% rename from packages/cli/src/context/ingest/adapters/historic-sql/projection.test.ts rename to packages/cli/test/context/ingest/adapters/historic-sql/projection.test.ts index 28ddf5f8..8487cdfc 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/projection.test.ts +++ b/packages/cli/test/context/ingest/adapters/historic-sql/projection.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import YAML from 'yaml'; import { describe, expect, it } from 'vitest'; -import { projectHistoricSqlEvidence } from './projection.js'; +import { projectHistoricSqlEvidence } from '../../../../../src/context/ingest/adapters/historic-sql/projection.js'; async function tempWorkdir(): Promise { return mkdtemp(join(tmpdir(), 'historic-sql-projection-')); @@ -60,7 +60,7 @@ describe('projectHistoricSqlEvidence', () => { kind: 'table_usage', connectionId: 'warehouse', table: 'public.orders', - rawPath: 'tables/public.orders.json', + rawPaths: ['tables/public.orders.json'], usage: { narrative: 'Orders are repeatedly queried for lifecycle analysis.', frequencyTier: 'high', @@ -158,7 +158,7 @@ describe('projectHistoricSqlEvidence', () => { await writeJson(workdir, '.ktx/ingest-evidence/historic-sql/run-1/pattern.json', { kind: 'pattern', connectionId: 'warehouse', - rawPath: 'patterns-input.json', + rawPaths: ['patterns-input/part-0001.json'], pattern: { slug: 'order-lifecycle-analysis', title: 'Order Lifecycle Analysis', @@ -179,7 +179,7 @@ describe('projectHistoricSqlEvidence', () => { expect.objectContaining({ target: 'wiki', key: 'historic-sql-old-order-lifecycle', - rawPaths: ['patterns-input.json'], + rawPaths: ['patterns-input/part-0001.json'], }), ]), ); @@ -234,7 +234,7 @@ describe('projectHistoricSqlEvidence', () => { await writeJson(workdir, '.ktx/ingest-evidence/historic-sql/run-1/pattern.json', { kind: 'pattern', connectionId: 'warehouse', - rawPath: 'patterns-input.json', + rawPaths: ['patterns-input/part-0001.json'], pattern: { slug: 'order-lifecycle-analysis', title: 'Order Lifecycle Analysis', @@ -343,7 +343,7 @@ describe('projectHistoricSqlEvidence', () => { kind: 'table_usage', connectionId: 'warehouse', table: 'public.customers', - rawPath: 'tables/public.customers.json', + rawPaths: ['tables/public.customers.json'], usage: { narrative: 'Customers were queried.', frequencyTier: 'low', @@ -380,7 +380,7 @@ describe('projectHistoricSqlEvidence', () => { expect(result.touchedSources).toEqual([{ connectionId: 'warehouse', sourceName: 'orders' }]); const staleAction = result.actions.find((action) => action.target === 'sl' && action.key === 'orders'); expect(staleAction).toEqual(expect.objectContaining({ target: 'sl', key: 'orders' })); - expect(staleAction?.rawPaths).toBeUndefined(); + expect(staleAction?.rawPaths).toEqual(['manifest.json']); const shard = YAML.parse(await readFile(join(workdir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8')); expect(shard.tables.orders.usage).toEqual({ ownerNote: 'keep analyst annotation', diff --git a/packages/cli/test/context/ingest/adapters/historic-sql/query-history-filter-picker.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/query-history-filter-picker.test.ts new file mode 100644 index 00000000..5c9e2e60 --- /dev/null +++ b/packages/cli/test/context/ingest/adapters/historic-sql/query-history-filter-picker.test.ts @@ -0,0 +1,322 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { KtxLlmRuntimePort } from '../../../../../src/context/llm/runtime-port.js'; +import type { + SqlAnalysisBatchItem, + SqlAnalysisBatchResult, + SqlAnalysisPort, +} from '../../../../../src/context/sql-analysis/ports.js'; +import { + proposeQueryHistoryServiceAccountFilters, + regexEscapeForExactRolePattern, +} from '../../../../../src/context/ingest/adapters/historic-sql/query-history-filter-picker.js'; +import type { + AggregatedTemplate, + HistoricSqlReader, +} from '../../../../../src/context/ingest/adapters/historic-sql/types.js'; + +function aggregate(overrides: Partial & { templateId: string; canonicalSql: string }): AggregatedTemplate { + return { + templateId: overrides.templateId, + canonicalSql: overrides.canonicalSql, + dialect: overrides.dialect ?? 'postgres', + stats: overrides.stats ?? { + executions: 25, + distinctUsers: 1, + firstSeen: '2026-05-01T00:00:00.000Z', + lastSeen: '2026-06-01T00:00:00.000Z', + p50RuntimeMs: 50, + p95RuntimeMs: 100, + errorRate: 0, + rowsProduced: 10, + }, + topUsers: overrides.topUsers ?? [{ user: 'analyst', executions: 25 }], + }; +} + +function reader(...templates: AggregatedTemplate[]): HistoricSqlReader { + return { + async probe() { + return { warnings: [], info: [] }; + }, + async *fetchAggregated() { + for (const template of templates) { + yield template; + } + }, + }; +} + +function sqlAnalysis(tablesById: Record>): SqlAnalysisPort { + return { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi.fn(async (items: SqlAnalysisBatchItem[]): Promise> => + new Map( + items.map((item) => [ + item.id, + { + tablesTouched: tablesById[item.id] ?? [], + columnsByClause: {}, + }, + ]), + ), + ), + validateReadOnly: vi.fn(async () => ({ ok: true })), + }; +} + +function sqlAnalysisWithErrors( + tablesById: Record>, + errorIds: string[], +): SqlAnalysisPort { + const errors = new Set(errorIds); + return { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi.fn(async (items: SqlAnalysisBatchItem[]): Promise> => + new Map( + items.map((item) => [ + item.id, + errors.has(item.id) + ? { tablesTouched: [], columnsByClause: {}, error: 'parse boom' } + : { tablesTouched: tablesById[item.id] ?? [], columnsByClause: {} }, + ]), + ), + ), + validateReadOnly: vi.fn(async () => ({ ok: true })), + }; +} + +function llm(decisions: Array<{ role: string; exclude: boolean; reason: string }>): KtxLlmRuntimePort { + const generateObject = vi.fn(async () => ({ roles: decisions })) as KtxLlmRuntimePort['generateObject']; + return { + generateText: vi.fn(), + generateObject, + runAgentLoop: vi.fn(), + }; +} + +describe('query-history filter picker', () => { + it('emits anchored escaped patterns for excluded roles from one batched LLM call', async () => { + const runtime = llm([ + { role: 'svc.loader+prod', exclude: true, reason: 'Runs recurring loader traffic only.' }, + { role: 'analyst', exclude: false, reason: 'Interactive analytic usage.' }, + ]); + const analysis = sqlAnalysis({ + loader: [{ catalog: null, db: 'analytics', name: 'orders' }], + analyst: [{ catalog: null, db: 'analytics', name: 'orders' }], + }); + + const proposal = await proposeQueryHistoryServiceAccountFilters({ + connectionId: 'warehouse', + dialect: 'postgres', + queryClient: {}, + reader: reader( + aggregate({ + templateId: 'loader', + canonicalSql: 'merge into analytics.orders using staging.orders_delta on orders.id = orders_delta.id', + topUsers: [{ user: 'svc.loader+prod', executions: 40 }], + }), + aggregate({ + templateId: 'analyst', + canonicalSql: 'select status, count(*) from analytics.orders group by status', + topUsers: [{ user: 'analyst', executions: 25 }], + }), + ), + sqlAnalysis: analysis, + llmRuntime: runtime, + pullConfig: { + dialect: 'postgres', + enabledSchemas: ['analytics'], + enabledTables: [], + modeledTableCatalog: [{ catalog: null, db: 'analytics', name: 'orders' }], + filters: { dropTrivialProbes: true }, + }, + now: new Date('2026-06-03T00:00:00.000Z'), + }); + + expect(runtime.generateObject).toHaveBeenCalledTimes(1); + expect(proposal).toMatchObject({ + excludedRoles: [ + { + role: 'svc.loader+prod', + pattern: '^svc\\.loader\\+prod$', + reason: 'Runs recurring loader traffic only.', + }, + ], + consideredRoleCount: 2, + skipped: null, + warnings: [], + }); + }); + + it('redacts representative SQL before sending role records to the LLM', async () => { + const originalSql = + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret + const runtime = llm([ + { role: 'svc_loader', exclude: false, reason: 'Keep by default.' }, + { role: 'analyst', exclude: false, reason: 'Interactive analytic usage.' }, + ]); + const analysis = sqlAnalysis({ + secret: [{ catalog: null, db: 'public', name: 'api_events' }], + analyst: [{ catalog: null, db: 'public', name: 'orders' }], + }); + + await proposeQueryHistoryServiceAccountFilters({ + connectionId: 'warehouse', + dialect: 'postgres', + queryClient: {}, + reader: reader( + aggregate({ + templateId: 'secret', + canonicalSql: originalSql, + topUsers: [{ user: 'svc_loader', executions: 30 }], + }), + aggregate({ + templateId: 'analyst', + canonicalSql: 'select status, count(*) from public.orders group by status', + topUsers: [{ user: 'analyst', executions: 25 }], + }), + ), + sqlAnalysis: analysis, + llmRuntime: runtime, + pullConfig: { + dialect: 'postgres', + enabledSchemas: ['public'], + enabledTables: [], + modeledTableCatalog: [], + redactionPatterns: ['sk_live_[A-Za-z0-9]+', '(?i)secret_token_[a-z0-9]+'], + filters: { dropTrivialProbes: true }, + }, + now: new Date('2026-06-03T00:00:00.000Z'), + }); + + expect(analysis.analyzeBatch).toHaveBeenCalledWith( + [ + { id: 'secret', sql: originalSql }, + { id: 'analyst', sql: 'select status, count(*) from public.orders group by status' }, + ], + 'postgres', + undefined, + ); + const call = vi.mocked(runtime.generateObject).mock.calls[0]?.[0]; + expect(call?.prompt).toContain('[REDACTED]'); + expect(call?.prompt).not.toContain('sk_live_abc123'); + expect(call?.prompt).not.toContain('Secret_Token_9f'); + }); + + it('fails open with no LLM runtime', async () => { + const proposal = await proposeQueryHistoryServiceAccountFilters({ + connectionId: 'warehouse', + dialect: 'postgres', + queryClient: {}, + reader: reader(), + sqlAnalysis: sqlAnalysis({}), + llmRuntime: null, + pullConfig: { dialect: 'postgres', filters: { dropTrivialProbes: true } }, + }); + + expect(proposal).toEqual({ + excludedRoles: [], + consideredRoleCount: 0, + skipped: { reason: 'no-llm' }, + warnings: [], + parseFailedTemplateIds: [], + }); + }); + + it('proposes nothing for a single-role stack', async () => { + const runtime = llm([{ role: 'warehouse_user', exclude: true, reason: 'Only observed role.' }]); + + const proposal = await proposeQueryHistoryServiceAccountFilters({ + connectionId: 'warehouse', + dialect: 'postgres', + queryClient: {}, + reader: reader( + aggregate({ + templateId: 'single-role', + canonicalSql: 'select * from analytics.orders', + topUsers: [{ user: 'warehouse_user', executions: 40 }], + }), + ), + sqlAnalysis: sqlAnalysis({ + 'single-role': [{ catalog: null, db: 'analytics', name: 'orders' }], + }), + llmRuntime: runtime, + pullConfig: { dialect: 'postgres', enabledSchemas: ['analytics'], filters: { dropTrivialProbes: true } }, + }); + + expect(runtime.generateObject).not.toHaveBeenCalled(); + expect(proposal.excludedRoles).toEqual([]); + expect(proposal.skipped).toEqual({ reason: 'no-in-scope-history' }); + }); + + it('records parse failures as template ids, not warnings', async () => { + const proposal = await proposeQueryHistoryServiceAccountFilters({ + connectionId: 'warehouse', + dialect: 'postgres', + queryClient: {}, + reader: reader( + aggregate({ + templateId: 'good', + canonicalSql: 'select * from analytics.orders', + topUsers: [{ user: 'analyst', executions: 30 }], + }), + aggregate({ + templateId: 'broken', + canonicalSql: 'select * from where', + topUsers: [{ user: 'analyst', executions: 5 }], + }), + ), + sqlAnalysis: sqlAnalysisWithErrors({ good: [{ catalog: null, db: 'analytics', name: 'orders' }] }, ['broken']), + llmRuntime: llm([]), + pullConfig: { dialect: 'postgres', enabledSchemas: ['analytics'], filters: { dropTrivialProbes: true } }, + }); + + expect(proposal.parseFailedTemplateIds).toEqual(['broken']); + expect(proposal.warnings).toEqual([]); + }); + + it('keeps clean in-scope history when the model excludes nothing', async () => { + const proposal = await proposeQueryHistoryServiceAccountFilters({ + connectionId: 'warehouse', + dialect: 'bigquery', + queryClient: {}, + reader: reader( + aggregate({ + templateId: 'dashboard', + canonicalSql: 'select status, count(*) from `demo.analytics.orders` group by status', + dialect: 'bigquery', + topUsers: [{ user: 'bi_runner', executions: 1 }], + }), + aggregate({ + templateId: 'analyst', + canonicalSql: 'select * from `demo.analytics.orders` where id = @id', + dialect: 'bigquery', + topUsers: [{ user: 'analyst', executions: 1 }], + }), + ), + sqlAnalysis: sqlAnalysis({ + dashboard: [{ catalog: 'demo', db: 'analytics', name: 'orders' }], + analyst: [{ catalog: 'demo', db: 'analytics', name: 'orders' }], + }), + llmRuntime: llm([ + { role: 'bi_runner', exclude: false, reason: 'Dashboard usage is analytic.' }, + { role: 'analyst', exclude: false, reason: 'Interactive analyst usage.' }, + ]), + pullConfig: { + dialect: 'bigquery', + windowDays: 90, + enabledSchemas: ['analytics'], + filters: { dropTrivialProbes: true }, + }, + }); + + expect(proposal.excludedRoles).toEqual([]); + expect(proposal.consideredRoleCount).toBe(2); + expect(proposal.skipped).toBeNull(); + }); + + it('escapes regex metacharacters for exact role matches', () => { + expect(regexEscapeForExactRolePattern('svc.loader+prod')).toBe('^svc\\.loader\\+prod$'); + expect(regexEscapeForExactRolePattern('team[etl](west)')).toBe('^team\\[etl\\]\\(west\\)$'); + }); +}); diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/redaction.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/redaction.test.ts similarity index 94% rename from packages/cli/src/context/ingest/adapters/historic-sql/redaction.test.ts rename to packages/cli/test/context/ingest/adapters/historic-sql/redaction.test.ts index d27015a6..262bc4ae 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/redaction.test.ts +++ b/packages/cli/test/context/ingest/adapters/historic-sql/redaction.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { compileHistoricSqlRedactionPatterns, redactHistoricSqlText } from './redaction.js'; +import { compileHistoricSqlRedactionPatterns, redactHistoricSqlText } from '../../../../../src/context/ingest/adapters/historic-sql/redaction.js'; describe('historic-SQL redaction', () => { it('redacts regex matches and supports the (?i) case-insensitive prefix', () => { diff --git a/packages/cli/test/context/ingest/adapters/historic-sql/scope-floor.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/scope-floor.test.ts new file mode 100644 index 00000000..597cae46 --- /dev/null +++ b/packages/cli/test/context/ingest/adapters/historic-sql/scope-floor.test.ts @@ -0,0 +1,194 @@ +import { mkdir, mkdtemp, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { resolveQueryHistoryScopeFloor } from '../../../../../src/context/ingest/adapters/historic-sql/scope-floor.js'; + +async function tempProject(): Promise { + return mkdtemp(join(tmpdir(), 'ktx-qh-scope-')); +} + +async function seedLiveScanTable( + projectDir: string, + connectionId: string, + syncId: string, + table: { catalog: string | null; db: string | null; name: string }, +): Promise { + const root = join(projectDir, 'raw-sources', connectionId, 'live-database', syncId); + await mkdir(join(root, 'tables'), { recursive: true }); + await writeFile( + join(root, 'connection.json'), + `${JSON.stringify({ connectionId, driver: 'postgres' }, null, 2)}\n`, + 'utf-8', + ); + await writeFile( + join(root, 'tables', `${table.db ?? 'default'}-${table.name}.json`), + `${JSON.stringify( + { + ...table, + kind: 'table', + comment: null, + estimatedRows: null, + columns: [], + foreignKeys: [], + }, + null, + 2, + )}\n`, + 'utf-8', + ); + await writeFile( + join(root, 'scan-report.json'), + `${JSON.stringify( + { + connectionId, + driver: 'postgres', + syncId, + runId: `scan-${syncId}`, + trigger: 'cli', + mode: 'enriched', + dryRun: false, + artifactPaths: { + rawSourcesDir: `raw-sources/${connectionId}/live-database/${syncId}`, + reportPath: `raw-sources/${connectionId}/live-database/${syncId}/scan-report.json`, + manifestShards: [], + enrichmentArtifacts: [], + }, + counts: {}, + warnings: [], + enrichment: {}, + enrichmentState: {}, + }, + null, + 2, + )}\n`, + 'utf-8', + ); +} + +describe('resolveQueryHistoryScopeFloor', () => { + it('computes modeled schemas from connection schemas plus semantic source tables', async () => { + const projectDir = await tempProject(); + await mkdir(join(projectDir, 'semantic-layer/warehouse'), { recursive: true }); + await writeFile( + join(projectDir, 'semantic-layer/warehouse/revenue.yaml'), + [ + 'name: revenue', + 'table: orbit_analytics.mart_revenue', + 'grain: [id]', + 'columns:', + ' - name: id', + ' type: string', + '', + ].join('\n'), + 'utf-8', + ); + await seedLiveScanTable(projectDir, 'warehouse', 'sync-1', { + catalog: null, + db: 'orbit_raw', + name: 'accounts', + }); + + const scope = await resolveQueryHistoryScopeFloor({ + projectDir, + connectionId: 'warehouse', + driver: 'postgres', + connection: { driver: 'postgres', schemas: ['orbit_raw'] }, + storedQueryHistory: {}, + }); + + expect(scope.enabledSchemas).toEqual(['orbit_analytics', 'orbit_raw']); + expect(scope.modeledTableCatalog).toEqual([ + { catalog: null, db: 'orbit_analytics', name: 'mart_revenue' }, + { catalog: null, db: 'orbit_raw', name: 'accounts' }, + ]); + expect(scope.enabledTables).toEqual([]); + expect(scope.floorDisabled).toBe(false); + }); + + it('uses explicit enabledTables before explicit enabledSchemas and computed scope', async () => { + const scope = await resolveQueryHistoryScopeFloor({ + projectDir: await tempProject(), + connectionId: 'warehouse', + driver: 'postgres', + connection: { driver: 'postgres', schemas: ['orbit_raw'] }, + storedQueryHistory: { + enabledTables: ['orbit_analytics.mart_revenue'], + enabledSchemas: ['orbit_raw'], + }, + }); + + expect(scope.enabledTables).toEqual([{ catalog: null, db: 'orbit_analytics', name: 'mart_revenue' }]); + expect(scope.enabledSchemas).toEqual([]); + expect(scope.floorDisabled).toBe(false); + }); + + it('disables the floor for enabledSchemas star', async () => { + const scope = await resolveQueryHistoryScopeFloor({ + projectDir: await tempProject(), + connectionId: 'warehouse', + driver: 'postgres', + connection: { driver: 'postgres', schemas: ['orbit_raw'] }, + storedQueryHistory: { enabledSchemas: ['*'] }, + }); + + expect(scope.enabledTables).toEqual([]); + expect(scope.enabledSchemas).toEqual(['*']); + expect(scope.floorDisabled).toBe(true); + }); + + it('adds latest live-database scan tables to the modeled table catalog', async () => { + const projectDir = await tempProject(); + await mkdir(join(projectDir, 'semantic-layer/warehouse'), { recursive: true }); + await writeFile( + join(projectDir, 'semantic-layer/warehouse/revenue.yaml'), + [ + 'name: revenue', + 'table: orbit_analytics.mart_revenue', + 'grain: [id]', + 'columns:', + ' - name: id', + ' type: string', + '', + ].join('\n'), + 'utf-8', + ); + await seedLiveScanTable(projectDir, 'warehouse', 'sync-1', { + catalog: null, + db: 'orbit_raw', + name: 'accounts', + }); + + const scope = await resolveQueryHistoryScopeFloor({ + projectDir, + connectionId: 'warehouse', + driver: 'postgres', + connection: { driver: 'postgres', schemas: ['orbit_raw'] }, + storedQueryHistory: {}, + }); + + expect(scope.enabledSchemas).toEqual(['orbit_analytics', 'orbit_raw']); + expect(scope.modeledTableCatalog).toEqual([ + { catalog: null, db: 'orbit_analytics', name: 'mart_revenue' }, + { catalog: null, db: 'orbit_raw', name: 'accounts' }, + ]); + expect(scope.warnings).toEqual([]); + expect(scope.floorDisabled).toBe(false); + }); + + it('fails open when schema scope exists but the scan catalog is unavailable', async () => { + const scope = await resolveQueryHistoryScopeFloor({ + projectDir: await tempProject(), + connectionId: 'warehouse', + driver: 'postgres', + connection: { driver: 'postgres', schemas: ['orbit_raw'] }, + storedQueryHistory: {}, + }); + + expect(scope.enabledTables).toEqual([]); + expect(scope.enabledSchemas).toEqual(['*']); + expect(scope.modeledTableCatalog).toEqual([]); + expect(scope.floorDisabled).toBe(true); + expect(scope.warnings).toContain('query_history_scope_floor_disabled:catalog_unavailable'); + }); +}); diff --git a/packages/cli/test/context/ingest/adapters/historic-sql/scope-membership.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/scope-membership.test.ts new file mode 100644 index 00000000..bfcefc22 --- /dev/null +++ b/packages/cli/test/context/ingest/adapters/historic-sql/scope-membership.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest'; +import { + includedQueryHistoryTableRefs, + isQueryHistoryScopeFloorDisabled, + shouldFailOpenQueryHistoryScope, +} from '../../../../../src/context/ingest/adapters/historic-sql/scope-membership.js'; +import type { KtxTableRef } from '../../../../../src/context/scan/types.js'; + +function ref(db: string | null, name: string, catalog: string | null = null): KtxTableRef { + return { catalog, db, name }; +} + +describe('query-history scope membership', () => { + it('prefers explicit enabled tables over schema scope', () => { + const orders = ref('analytics', 'orders'); + const noise = ref('metabase', 'application_table'); + + expect( + includedQueryHistoryTableRefs([orders, noise], { + enabledTables: [orders], + enabledSchemas: ['metabase'], + }), + ).toEqual([orders]); + }); + + it('matches schema scope by the db component across catalogs', () => { + const modeled = ref('orbit_analytics', 'orders', 'demo-project'); + const noise = ref('metabase', 'application_table', 'demo-project'); + + expect( + includedQueryHistoryTableRefs([modeled, noise], { + enabledTables: [], + enabledSchemas: ['orbit_analytics'], + }), + ).toEqual([modeled]); + }); + + it('keeps every touched ref when wildcard scope disables the floor', () => { + const tables = [ref('analytics', 'orders'), ref('metabase', 'application_table')]; + + expect(isQueryHistoryScopeFloorDisabled({ enabledTables: [], enabledSchemas: ['*'] })).toBe(true); + expect(includedQueryHistoryTableRefs(tables, { enabledTables: [], enabledSchemas: ['*'] })).toEqual(tables); + }); + + it('fails open when no tables, schemas, or wildcard are configured', () => { + const tables = [ref('metabase', 'application_table')]; + + expect(shouldFailOpenQueryHistoryScope({ enabledTables: [], enabledSchemas: [] })).toBe(true); + expect(includedQueryHistoryTableRefs(tables, { enabledTables: [], enabledSchemas: [] })).toEqual(tables); + }); +}); diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/skill-schemas.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/skill-schemas.test.ts similarity index 96% rename from packages/cli/src/context/ingest/adapters/historic-sql/skill-schemas.test.ts rename to packages/cli/test/context/ingest/adapters/historic-sql/skill-schemas.test.ts index b384c0c0..2a6ac805 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/skill-schemas.test.ts +++ b/packages/cli/test/context/ingest/adapters/historic-sql/skill-schemas.test.ts @@ -4,7 +4,7 @@ import { patternOutputSchema, patternsArraySchema, tableUsageOutputSchema, -} from './skill-schemas.js'; +} from '../../../../../src/context/ingest/adapters/historic-sql/skill-schemas.js'; describe('historic-sql skill schemas', () => { it('accepts table usage output and preserves future keys', () => { diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts similarity index 78% rename from packages/cli/src/context/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts rename to packages/cli/test/context/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts index b33183d7..ab76c533 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts +++ b/packages/cli/test/context/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { HistoricSqlGrantsMissingError } from './errors.js'; -import { SnowflakeHistoricSqlQueryHistoryReader } from './snowflake-query-history-reader.js'; +import { HistoricSqlGrantsMissingError } from '../../../../../src/context/ingest/adapters/historic-sql/errors.js'; +import { SnowflakeHistoricSqlQueryHistoryReader } from '../../../../../src/context/ingest/adapters/historic-sql/snowflake-query-history-reader.js'; interface FakeQueryResult { headers: string[]; @@ -90,7 +90,10 @@ describe('SnowflakeHistoricSqlQueryHistoryReader', () => { 40, 0.05, 100, - JSON.stringify([{ user: 'ANALYST', executions: 1 }]), + JSON.stringify([ + { user: 'SVC_LOADER', executions: 40 }, + { user: 'ANALYST', executions: 2 }, + ]), ], ], totalRows: 1, @@ -102,15 +105,20 @@ describe('SnowflakeHistoricSqlQueryHistoryReader', () => { for await (const row of reader.fetchAggregated( client, { start: new Date('2026-02-10T00:00:00.000Z'), end: new Date('2026-05-11T00:00:00.000Z') }, - { dialect: 'snowflake', minExecutions: 5, windowDays: 90, enabledTables: [], filters: { dropTrivialProbes: true }, redactionPatterns: [], staleArchiveAfterDays: 90 }, + { dialect: 'snowflake', minExecutions: 5, windowDays: 90, enabledTables: [], enabledSchemas: [], modeledTableCatalog: [], scopeFloorWarnings: [], filters: { dropTrivialProbes: true }, redactionPatterns: [], staleArchiveAfterDays: 90 }, )) { rows.push(row); } const sql = firstQuery(client); + expect(sql).toContain('WITH filtered_queries AS'); + expect(sql).toContain('template_stats AS'); + expect(sql).toContain('template_users AS'); expect(sql).toContain('SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY'); expect(sql).toContain('COUNT(*) AS executions'); - expect(sql).toContain('GROUP BY query_hash'); + expect(sql).toContain('COUNT(DISTINCT user_name) AS distinct_users'); + expect(sql).toContain('GROUP BY query_hash, user_name'); + expect(sql).toContain('ORDER BY users.executions DESC'); expect(sql).toContain('HAVING COUNT(*) >= 5'); expect(rows).toMatchObject([ { @@ -119,7 +127,10 @@ describe('SnowflakeHistoricSqlQueryHistoryReader', () => { executions: 42, errorRate: 0.05, }, - topUsers: [{ user: 'ANALYST', executions: 1 }], + topUsers: [ + { user: 'SVC_LOADER', executions: 40 }, + { user: 'ANALYST', executions: 2 }, + ], }, ]); }); @@ -136,6 +147,9 @@ describe('SnowflakeHistoricSqlQueryHistoryReader', () => { minExecutions: 5, windowDays: 90, enabledTables: [], + enabledSchemas: [], + modeledTableCatalog: [], + scopeFloorWarnings: [], filters: { dropTrivialProbes: true }, redactionPatterns: [], staleArchiveAfterDays: 90, diff --git a/packages/cli/test/context/ingest/adapters/historic-sql/stage-unified.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/stage-unified.test.ts new file mode 100644 index 00000000..a104508d --- /dev/null +++ b/packages/cli/test/context/ingest/adapters/historic-sql/stage-unified.test.ts @@ -0,0 +1,859 @@ +import { mkdtemp, readFile, readdir } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it, vi } from 'vitest'; +import type { SqlAnalysisPort } from '../../../../../src/context/sql-analysis/ports.js'; +import { stageHistoricSqlAggregatedSnapshot } from '../../../../../src/context/ingest/adapters/historic-sql/stage-unified.js'; +import type { AggregatedTemplate, HistoricSqlReader } from '../../../../../src/context/ingest/adapters/historic-sql/types.js'; + +async function tempDir(): Promise { + return mkdtemp(join(tmpdir(), 'historic-sql-unified-stage-')); +} + +async function readJson(root: string, relPath: string): Promise { + return JSON.parse(await readFile(join(root, relPath), 'utf-8')) as T; +} + +function tableRef(value: string): { catalog: string | null; db: string | null; name: string } { + const parts = value.split('.'); + if (parts.length === 3) return { catalog: parts[0]!, db: parts[1]!, name: parts[2]! }; + if (parts.length === 2) return { catalog: null, db: parts[0]!, name: parts[1]! }; + return { catalog: null, db: null, name: value }; +} + +function aggregate(overrides: Partial & { templateId: string; canonicalSql: string }): AggregatedTemplate { + return { + templateId: overrides.templateId, + canonicalSql: overrides.canonicalSql, + dialect: overrides.dialect ?? 'postgres', + stats: overrides.stats ?? { + executions: 42, + distinctUsers: 3, + firstSeen: '2026-05-01T00:00:00.000Z', + lastSeen: '2026-05-11T00:00:00.000Z', + p50RuntimeMs: 20, + p95RuntimeMs: 80, + errorRate: 0, + rowsProduced: 100, + }, + topUsers: overrides.topUsers ?? [{ user: 'analyst', executions: 40 }], + }; +} + +describe('stageHistoricSqlAggregatedSnapshot', () => { + it('batch parses templates and writes stable table and patterns artifacts', async () => { + const stagedDir = await tempDir(); + const reader: HistoricSqlReader = { + async probe() { + return { warnings: ['pg_stat_statements.track is none; aggregation still proceeds'], info: [] }; + }, + async *fetchAggregated() { + yield aggregate({ + templateId: 'orders-by-status', + canonicalSql: 'select o.status, count(*) from public.orders o join public.customers c on c.id = o.customer_id where o.created_at >= $1 group by o.status', + }); + yield aggregate({ + templateId: 'service-account-only', + canonicalSql: 'select * from public.orders where id = $1', + stats: { + executions: 20, + distinctUsers: 1, + firstSeen: '2026-05-01T00:00:00.000Z', + lastSeen: '2026-05-11T00:00:00.000Z', + p50RuntimeMs: 5, + p95RuntimeMs: 10, + errorRate: 0, + rowsProduced: 1, + }, + topUsers: [{ user: 'svc_loader', executions: 20 }], + }); + yield aggregate({ + templateId: 'bad-parse', + canonicalSql: 'select broken from', + }); + }, + }; + const sqlAnalysis: SqlAnalysisPort = { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi.fn(async () => new Map([ + [ + 'orders-by-status', + { + tablesTouched: [tableRef('public.orders'), tableRef('public.customers')], + columnsByClause: { + select: ['status'], + where: ['created_at'], + join: ['customer_id'], + groupBy: ['status'], + }, + }, + ], + ['bad-parse', { tablesTouched: [], columnsByClause: {}, error: 'parse failed' }], + ])), + validateReadOnly: vi.fn(async () => ({ ok: true })), + }; + + await stageHistoricSqlAggregatedSnapshot({ + stagedDir, + connectionId: 'warehouse', + queryClient: {}, + reader, + sqlAnalysis, + pullConfig: { + dialect: 'postgres', + enabledSchemas: ['public'], + filters: { + serviceAccounts: { patterns: ['^svc_'], mode: 'exclude' }, + }, + }, + now: new Date('2026-05-11T12:00:00.000Z'), + }); + + expect(sqlAnalysis.analyzeBatch).toHaveBeenCalledTimes(1); + expect(sqlAnalysis.analyzeBatch).toHaveBeenCalledWith( + [ + { + id: 'orders-by-status', + sql: 'select o.status, count(*) from public.orders o join public.customers c on c.id = o.customer_id where o.created_at >= $1 group by o.status', + }, + { id: 'bad-parse', sql: 'select broken from' }, + ], + 'postgres', + undefined, + ); + + expect(await readdir(join(stagedDir, 'tables'))).toEqual(['public.customers.json', 'public.orders.json']); + + const manifest = await readJson>(stagedDir, 'manifest.json'); + expect(manifest).toMatchObject({ + source: 'historic-sql', + connectionId: 'warehouse', + dialect: 'postgres', + snapshotRowCount: 3, + touchedTableCount: 2, + parseFailures: 1, + warnings: ['parse_failed:bad-parse'], + probeWarnings: ['pg_stat_statements.track is none; aggregation still proceeds'], + staleArchiveAfterDays: 90, + }); + + const orders = await readJson>(stagedDir, 'tables/public.orders.json'); + expect(orders).toMatchObject({ + table: 'public.orders', + tableRef: tableRef('public.orders'), + stats: { + executionsBucket: '10-100', + distinctUsersBucket: '2-5', + errorRateBucket: 'none', + p95RuntimeBucket: '<100ms', + recencyBucket: 'current', + }, + columnsByClause: { + select: [['status', 'high']], + where: [['created_at', 'high']], + join: [['customer_id', 'high']], + groupBy: [['status', 'high']], + }, + observedJoins: [{ withTable: 'public.customers', on: ['customer_id'], freq: 'high' }], + topTemplates: [ + { + id: 'orders-by-status', + topUsers: [{ user: 'analyst' }], + }, + ], + }); + expect(orders.topTemplates[0].canonicalSql).toContain('group by o.status'); + + const patterns = await readJson>(stagedDir, 'patterns-input.json'); + expect(patterns.templates).toEqual([ + { + id: 'orders-by-status', + canonicalSql: expect.stringContaining('public.orders'), + tablesTouched: [tableRef('public.customers'), tableRef('public.orders')], + executionsBucket: '10-100', + distinctUsersBucket: '2-5', + dialect: 'postgres', + }, + ]); + }); + + it('keeps templates when service-account topUsers are only a partial execution sample', async () => { + const stagedDir = await tempDir(); + const reader: HistoricSqlReader = { + async probe() { + return { warnings: [], info: [] }; + }, + async *fetchAggregated() { + yield aggregate({ + templateId: 'shared-bigquery-template', + canonicalSql: 'select status, count(*) from `demo.analytics.orders` group by status', + dialect: 'bigquery', + stats: { + executions: 42, + distinctUsers: 2, + firstSeen: '2026-05-01T00:00:00.000Z', + lastSeen: '2026-05-11T00:00:00.000Z', + p50RuntimeMs: 20, + p95RuntimeMs: 80, + errorRate: 0, + rowsProduced: null, + }, + topUsers: [{ user: 'svc_loader', executions: 5 }], + }); + }, + }; + const sqlAnalysis: SqlAnalysisPort = { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi.fn(async () => + new Map([ + [ + 'shared-bigquery-template', + { + tablesTouched: [tableRef('demo.analytics.orders')], + columnsByClause: { select: ['status'], groupBy: ['status'] }, + }, + ], + ]), + ), + validateReadOnly: vi.fn(async () => ({ ok: true })), + }; + + await stageHistoricSqlAggregatedSnapshot({ + stagedDir, + connectionId: 'warehouse', + queryClient: {}, + reader, + sqlAnalysis, + pullConfig: { + dialect: 'bigquery', + windowDays: 90, + enabledSchemas: ['analytics'], + filters: { + serviceAccounts: { patterns: ['^svc_loader$'], mode: 'exclude' }, + }, + }, + now: new Date('2026-05-11T12:00:00.000Z'), + }); + + const patterns = await readJson>(stagedDir, 'patterns-input.json'); + expect(patterns.templates.map((template: { id: string }) => template.id)).toEqual([ + 'shared-bigquery-template', + ]); + const orders = await readJson>(stagedDir, 'tables/demo.analytics.orders.json'); + expect(orders.topTemplates).toEqual([ + { + id: 'shared-bigquery-template', + canonicalSql: 'select status, count(*) from `demo.analytics.orders` group by status', + topUsers: [{ user: 'svc_loader' }], + }, + ]); + }); + + it('drops service-account-only templates when matched users cover all executions', async () => { + const stagedDir = await tempDir(); + const reader: HistoricSqlReader = { + async probe() { + return { warnings: [], info: [] }; + }, + async *fetchAggregated() { + yield aggregate({ + templateId: 'service-only-template', + canonicalSql: 'merge into analytics.orders using staging.orders_delta on orders.id = orders_delta.id', + stats: { + executions: 12, + distinctUsers: 1, + firstSeen: '2026-05-01T00:00:00.000Z', + lastSeen: '2026-05-11T00:00:00.000Z', + p50RuntimeMs: 20, + p95RuntimeMs: 80, + errorRate: 0, + rowsProduced: 0, + }, + topUsers: [{ user: 'svc_loader', executions: 12 }], + }); + }, + }; + const sqlAnalysis: SqlAnalysisPort = { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi.fn(async () => new Map()), + validateReadOnly: vi.fn(async () => ({ ok: true })), + }; + + await stageHistoricSqlAggregatedSnapshot({ + stagedDir, + connectionId: 'warehouse', + queryClient: {}, + reader, + sqlAnalysis, + pullConfig: { + dialect: 'postgres', + enabledSchemas: ['analytics'], + filters: { + serviceAccounts: { patterns: ['^svc_loader$'], mode: 'exclude' }, + }, + }, + now: new Date('2026-05-11T12:00:00.000Z'), + }); + + expect(sqlAnalysis.analyzeBatch).toHaveBeenCalledWith([], 'postgres', undefined); + const patterns = await readJson>(stagedDir, 'patterns-input.json'); + expect(patterns.templates).toEqual([]); + }); + + it('redacts configured SQL substrings in staged artifacts while analyzing original SQL', async () => { + const stagedDir = await tempDir(); + const originalSql = + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret + const reader: HistoricSqlReader = { + async probe() { + return { warnings: [], info: [] }; + }, + async *fetchAggregated() { + yield aggregate({ + templateId: 'api-events-with-secret', + canonicalSql: originalSql, + stats: { + executions: 15, + distinctUsers: 2, + firstSeen: '2026-05-01T00:00:00.000Z', + lastSeen: '2026-05-11T00:00:00.000Z', + p50RuntimeMs: 12, + p95RuntimeMs: 25, + errorRate: 0, + rowsProduced: 15, + }, + }); + }, + }; + const sqlAnalysis: SqlAnalysisPort = { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi.fn(async () => new Map([ + [ + 'api-events-with-secret', + { + tablesTouched: [tableRef('public.api_events')], + columnsByClause: { + select: [], + where: ['api_key', 'note'], + join: [], + groupBy: [], + }, + }, + ], + ])), + validateReadOnly: vi.fn(async () => ({ ok: true })), + }; + + await stageHistoricSqlAggregatedSnapshot({ + stagedDir, + connectionId: 'warehouse', + queryClient: {}, + reader, + sqlAnalysis, + pullConfig: { + dialect: 'postgres', + enabledSchemas: ['public'], + redactionPatterns: ['sk_live_[A-Za-z0-9]+', '(?i)secret_token_[a-z0-9]+'], + }, + now: new Date('2026-05-11T12:00:00.000Z'), + }); + + expect(sqlAnalysis.analyzeBatch).toHaveBeenCalledWith( + [{ id: 'api-events-with-secret', sql: originalSql }], + 'postgres', + undefined, + ); + + const tableJson = await readFile(join(stagedDir, 'tables/public.api_events.json'), 'utf-8'); + const patternsJson = await readFile(join(stagedDir, 'patterns-input.json'), 'utf-8'); + expect(tableJson).not.toContain('sk_live_abc123'); + expect(tableJson).not.toContain('Secret_Token_9f'); + expect(patternsJson).not.toContain('sk_live_abc123'); + expect(patternsJson).not.toContain('Secret_Token_9f'); + expect(tableJson).toContain('[REDACTED]'); + expect(patternsJson).toContain('[REDACTED]'); + }); + + it('limits staged table artifacts to configured enabled tables', async () => { + const stagedDir = await tempDir(); + const reader: HistoricSqlReader = { + async probe() { + return { warnings: [], info: [] }; + }, + async *fetchAggregated() { + yield aggregate({ + templateId: 'selected-qualified', + canonicalSql: 'select count(*) from orbit_analytics.int_active_contract_arr', + }); + yield aggregate({ + templateId: 'selected-unqualified', + canonicalSql: 'select count(*) from int_customer_health_signals', + }); + yield aggregate({ + templateId: 'unselected', + canonicalSql: 'select count(*) from orbit_raw.accounts', + }); + }, + }; + const sqlAnalysis: SqlAnalysisPort = { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi.fn(async () => new Map([ + [ + 'selected-qualified', + { + tablesTouched: [tableRef('orbit_analytics.int_active_contract_arr')], + columnsByClause: { select: [], where: [], join: [], groupBy: [] }, + }, + ], + [ + 'selected-unqualified', + { + tablesTouched: [tableRef('orbit_analytics.int_customer_health_signals')], + columnsByClause: { select: [], where: [], join: [], groupBy: [] }, + }, + ], + [ + 'unselected', + { + tablesTouched: [tableRef('orbit_raw.accounts')], + columnsByClause: { select: [], where: [], join: [], groupBy: [] }, + }, + ], + ])), + validateReadOnly: vi.fn(async () => ({ ok: true })), + }; + + await stageHistoricSqlAggregatedSnapshot({ + stagedDir, + connectionId: 'warehouse', + queryClient: {}, + reader, + sqlAnalysis, + pullConfig: { + dialect: 'postgres', + enabledTables: [ + tableRef('orbit_analytics.int_active_contract_arr'), + tableRef('orbit_analytics.int_customer_health_signals'), + ], + }, + now: new Date('2026-05-11T12:00:00.000Z'), + }); + + expect(await readdir(join(stagedDir, 'tables'))).toEqual([ + 'orbit_analytics.int_active_contract_arr.json', + 'orbit_analytics.int_customer_health_signals.json', + ]); + const manifest = await readJson>(stagedDir, 'manifest.json'); + expect(manifest.touchedTableCount).toBe(2); + const patterns = await readJson>(stagedDir, 'patterns-input.json'); + expect(patterns.templates.map((entry: any) => entry.id)).toEqual(['selected-qualified', 'selected-unqualified']); + }); + + it('preserves full patterns audit input and writes bounded cross-table pattern shards', async () => { + const stagedDir = await tempDir(); + const largeSql = `select * from public.orders o join public.customers c on c.id = o.customer_id where payload = '${'x'.repeat(8000)}'`; + const reader: HistoricSqlReader = { + async probe() { + return { warnings: [], info: [] }; + }, + async *fetchAggregated() { + yield aggregate({ + templateId: 'orders-customers-a', + canonicalSql: largeSql, + stats: { + executions: 25, + distinctUsers: 4, + firstSeen: '2026-05-01T00:00:00.000Z', + lastSeen: '2026-05-11T00:00:00.000Z', + p50RuntimeMs: 15, + p95RuntimeMs: 90, + errorRate: 0, + rowsProduced: 250, + }, + }); + yield aggregate({ + templateId: 'orders-customers-b', + canonicalSql: largeSql.replace('payload', 'payload_b'), + stats: { + executions: 22, + distinctUsers: 3, + firstSeen: '2026-05-01T00:00:00.000Z', + lastSeen: '2026-05-11T00:00:00.000Z', + p50RuntimeMs: 20, + p95RuntimeMs: 95, + errorRate: 0, + rowsProduced: 220, + }, + }); + yield aggregate({ + templateId: 'orders-single-table', + canonicalSql: 'select count(*) from public.orders', + stats: { + executions: 30, + distinctUsers: 2, + firstSeen: '2026-05-01T00:00:00.000Z', + lastSeen: '2026-05-11T00:00:00.000Z', + p50RuntimeMs: 10, + p95RuntimeMs: 20, + errorRate: 0, + rowsProduced: 30, + }, + }); + }, + }; + const sqlAnalysis: SqlAnalysisPort = { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi.fn(async () => new Map([ + [ + 'orders-customers-a', + { + tablesTouched: [tableRef('public.orders'), tableRef('public.customers')], + columnsByClause: { + select: [], + where: ['payload'], + join: ['customer_id', 'id'], + groupBy: [], + }, + }, + ], + [ + 'orders-customers-b', + { + tablesTouched: [tableRef('public.orders'), tableRef('public.customers')], + columnsByClause: { + select: [], + where: ['payload_b'], + join: ['customer_id', 'id'], + groupBy: [], + }, + }, + ], + [ + 'orders-single-table', + { + tablesTouched: [tableRef('public.orders')], + columnsByClause: { + select: [], + where: [], + join: [], + groupBy: [], + }, + }, + ], + ])), + validateReadOnly: vi.fn(async () => ({ ok: true })), + }; + + await stageHistoricSqlAggregatedSnapshot({ + stagedDir, + connectionId: 'warehouse', + queryClient: {}, + reader, + sqlAnalysis, + pullConfig: { dialect: 'postgres', enabledSchemas: ['public'] }, + now: new Date('2026-05-11T12:00:00.000Z'), + }); + + const audit = await readJson>(stagedDir, 'patterns-input.json'); + expect(audit.templates.map((entry: any) => entry.id)).toEqual([ + 'orders-customers-a', + 'orders-customers-b', + 'orders-single-table', + ]); + + const firstShard = await readJson>(stagedDir, 'patterns-input/part-0001.json'); + expect(firstShard.templates.map((entry: any) => entry.id)).toEqual(['orders-customers-a', 'orders-customers-b']); + expect(firstShard.templates.some((entry: any) => entry.id === 'orders-single-table')).toBe(false); + + const manifest = await readJson>(stagedDir, 'manifest.json'); + expect(manifest.warnings).toEqual([]); + }); + + it("drops ktx's own scan/relationship probes from query history", async () => { + const stagedDir = await tempDir(); + const fkOverlapProbe = + 'select * from (WITH child_values AS ( SELECT DISTINCT "account_id" AS value FROM "account_owners" WHERE "account_id" IS NOT NULL LIMIT $1 ), parent_values AS ( SELECT DISTINCT "account_id" AS value FROM "accounts" WHERE "account_id" IS NOT NULL ) SELECT (SELECT COUNT(*) FROM child_values) AS child_distinct, (SELECT COUNT(*) FROM parent_values) AS parent_distinct) probe'; + const profileProbe = + 'select * from (SELECT $1 AS column_name, (SELECT COUNT(*) FROM "orbit_raw"."accounts") AS total, (SELECT STRING_AGG(CAST(value AS TEXT), CHR(31)) FROM (SELECT DISTINCT "id" AS value FROM "orbit_raw"."accounts" LIMIT $2) AS relationship_profile_values) AS samples) profile'; + const reader: HistoricSqlReader = { + async probe() { + return { warnings: [], info: [] }; + }, + async *fetchAggregated() { + yield aggregate({ + templateId: 'analytic', + canonicalSql: 'select status, count(*) from public.orders group by status', + }); + yield aggregate({ templateId: 'ktx-fk-overlap', canonicalSql: fkOverlapProbe }); + yield aggregate({ templateId: 'ktx-profile', canonicalSql: profileProbe }); + }, + }; + const sqlAnalysis: SqlAnalysisPort = { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi.fn(async () => new Map([ + [ + 'analytic', + { + tablesTouched: [tableRef('public.orders')], + columnsByClause: { select: ['status'], where: [], join: [], groupBy: ['status'] }, + }, + ], + ])), + validateReadOnly: vi.fn(async () => ({ ok: true })), + }; + + await stageHistoricSqlAggregatedSnapshot({ + stagedDir, + connectionId: 'warehouse', + queryClient: {}, + reader, + sqlAnalysis, + pullConfig: { dialect: 'postgres', enabledSchemas: ['public'] }, + now: new Date('2026-05-11T12:00:00.000Z'), + }); + + // ktx scan probes are filtered before SQL analysis, so only the analytic query is parsed. + expect(sqlAnalysis.analyzeBatch).toHaveBeenCalledWith( + [{ id: 'analytic', sql: 'select status, count(*) from public.orders group by status' }], + 'postgres', + undefined, + ); + expect(await readdir(join(stagedDir, 'tables'))).toEqual(['public.orders.json']); + }); + + it('keeps modeled-schema refs and drops unmodeled-schema refs by default', async () => { + const stagedDir = await tempDir(); + const reader: HistoricSqlReader = { + async probe() { + return { warnings: [], info: [] }; + }, + async *fetchAggregated() { + yield aggregate({ templateId: 'modeled', canonicalSql: 'select count(*) from orbit_raw.accounts' }); + yield aggregate({ templateId: 'noise', canonicalSql: 'select count(*) from metabase.application_table' }); + }, + }; + const sqlAnalysis: SqlAnalysisPort = { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi.fn(async () => new Map([ + ['modeled', { tablesTouched: [{ catalog: null, db: 'orbit_raw', name: 'accounts' }], columnsByClause: {} }], + ['noise', { tablesTouched: [{ catalog: null, db: 'metabase', name: 'application_table' }], columnsByClause: {} }], + ])), + validateReadOnly: vi.fn(async () => ({ ok: true })), + }; + + await stageHistoricSqlAggregatedSnapshot({ + stagedDir, + connectionId: 'warehouse', + queryClient: {}, + reader, + sqlAnalysis, + pullConfig: { + dialect: 'postgres', + enabledSchemas: ['orbit_raw'], + modeledTableCatalog: [{ catalog: null, db: 'orbit_raw', name: 'accounts' }], + }, + now: new Date('2026-05-11T12:00:00.000Z'), + }); + + expect(await readdir(join(stagedDir, 'tables'))).toEqual(['orbit_raw.accounts.json']); + const manifest = await readJson>(stagedDir, 'manifest.json'); + expect(manifest.touchedTableCount).toBe(1); + }); + + it('fails open when the implicit modeled scope is empty', async () => { + const stagedDir = await tempDir(); + const reader: HistoricSqlReader = { + async probe() { + return { warnings: [], info: [] }; + }, + async *fetchAggregated() { + yield aggregate({ templateId: 'any-table', canonicalSql: 'select count(*) from metabase.application_table' }); + }, + }; + const sqlAnalysis: SqlAnalysisPort = { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi.fn(async () => new Map([ + ['any-table', { tablesTouched: [{ catalog: null, db: 'metabase', name: 'application_table' }], columnsByClause: {} }], + ])), + validateReadOnly: vi.fn(async () => ({ ok: true })), + }; + + await stageHistoricSqlAggregatedSnapshot({ + stagedDir, + connectionId: 'warehouse', + queryClient: {}, + reader, + sqlAnalysis, + pullConfig: { dialect: 'postgres', enabledSchemas: [], modeledTableCatalog: [] }, + now: new Date('2026-05-11T12:00:00.000Z'), + }); + + expect(await readdir(join(stagedDir, 'tables'))).toEqual(['metabase.application_table.json']); + const manifest = await readJson>(stagedDir, 'manifest.json'); + expect(manifest.warnings).toContain('query_history_scope_floor_disabled:empty_modeled_scope'); + }); + + it('lets enabledSchemas star disable the floor', async () => { + const stagedDir = await tempDir(); + const reader: HistoricSqlReader = { + async probe() { + return { warnings: [], info: [] }; + }, + async *fetchAggregated() { + yield aggregate({ templateId: 'noise', canonicalSql: 'select count(*) from metabase.application_table' }); + }, + }; + const sqlAnalysis: SqlAnalysisPort = { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi.fn(async () => new Map([ + ['noise', { tablesTouched: [{ catalog: null, db: 'metabase', name: 'application_table' }], columnsByClause: {} }], + ])), + validateReadOnly: vi.fn(async () => ({ ok: true })), + }; + + await stageHistoricSqlAggregatedSnapshot({ + stagedDir, + connectionId: 'warehouse', + queryClient: {}, + reader, + sqlAnalysis, + pullConfig: { + dialect: 'postgres', + enabledSchemas: ['*'], + modeledTableCatalog: [{ catalog: null, db: 'orbit_raw', name: 'accounts' }], + }, + now: new Date('2026-05-11T12:00:00.000Z'), + }); + + expect(await readdir(join(stagedDir, 'tables'))).toEqual(['metabase.application_table.json']); + }); + + it('matches BigQuery dataset scope even when refs include a catalog', async () => { + const stagedDir = await tempDir(); + const reader: HistoricSqlReader = { + async probe() { + return { warnings: [], info: [] }; + }, + async *fetchAggregated() { + yield aggregate({ templateId: 'modeled', canonicalSql: 'select count(*) from `demo-project.orbit_analytics.orders`' }); + yield aggregate({ templateId: 'noise', canonicalSql: 'select count(*) from `demo-project.metabase.application_table`' }); + }, + }; + const sqlAnalysis: SqlAnalysisPort = { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi.fn(async () => new Map([ + ['modeled', { tablesTouched: [{ catalog: 'demo-project', db: 'orbit_analytics', name: 'orders' }], columnsByClause: {} }], + ['noise', { tablesTouched: [{ catalog: 'demo-project', db: 'metabase', name: 'application_table' }], columnsByClause: {} }], + ])), + validateReadOnly: vi.fn(async () => ({ ok: true })), + }; + + await stageHistoricSqlAggregatedSnapshot({ + stagedDir, + connectionId: 'warehouse', + queryClient: {}, + reader, + sqlAnalysis, + pullConfig: { + dialect: 'bigquery', + enabledSchemas: ['orbit_analytics'], + modeledTableCatalog: [{ catalog: 'demo-project', db: 'orbit_analytics', name: 'orders' }], + }, + now: new Date('2026-05-11T12:00:00.000Z'), + }); + + expect(await readdir(join(stagedDir, 'tables'))).toEqual(['demo-project.orbit_analytics.orders.json']); + }); + + it('writes propagated scope-floor warnings to the staged manifest', async () => { + const stagedDir = await tempDir(); + const reader: HistoricSqlReader = { + async probe() { + return { warnings: [], info: [] }; + }, + async *fetchAggregated() { + yield aggregate({ templateId: 'any-table', canonicalSql: 'select count(*) from metabase.application_table' }); + }, + }; + const sqlAnalysis: SqlAnalysisPort = { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi.fn(async () => new Map([ + ['any-table', { tablesTouched: [{ catalog: null, db: 'metabase', name: 'application_table' }], columnsByClause: {} }], + ])), + validateReadOnly: vi.fn(async () => ({ ok: true })), + }; + + await stageHistoricSqlAggregatedSnapshot({ + stagedDir, + connectionId: 'warehouse', + queryClient: {}, + reader, + sqlAnalysis, + pullConfig: { + dialect: 'postgres', + enabledSchemas: ['*'], + scopeFloorWarnings: ['query_history_scope_floor_disabled:catalog_unavailable'], + }, + now: new Date('2026-05-11T12:00:00.000Z'), + }); + + const manifest = await readJson>(stagedDir, 'manifest.json'); + expect(manifest.warnings).toContain('query_history_scope_floor_disabled:catalog_unavailable'); + expect(await readdir(join(stagedDir, 'tables'))).toEqual(['metabase.application_table.json']); + }); + + it('retries without the catalog and disables the floor when catalog qualification fails wholesale', async () => { + const stagedDir = await tempDir(); + const reader: HistoricSqlReader = { + async probe() { + return { warnings: [], info: [] }; + }, + async *fetchAggregated() { + yield aggregate({ templateId: 'noise', canonicalSql: 'select count(*) from metabase.application_table' }); + }, + }; + const sqlAnalysis: SqlAnalysisPort = { + analyzeForFingerprint: vi.fn(), + analyzeBatch: vi + .fn() + .mockRejectedValueOnce(new Error('catalog qualification failed')) + .mockResolvedValueOnce( + new Map([ + ['noise', { tablesTouched: [{ catalog: null, db: 'metabase', name: 'application_table' }], columnsByClause: {} }], + ]), + ), + validateReadOnly: vi.fn(async () => ({ ok: true })), + }; + + await stageHistoricSqlAggregatedSnapshot({ + stagedDir, + connectionId: 'warehouse', + queryClient: {}, + reader, + sqlAnalysis, + pullConfig: { + dialect: 'postgres', + enabledSchemas: ['orbit_raw'], + modeledTableCatalog: [{ catalog: null, db: 'orbit_raw', name: 'accounts' }], + }, + now: new Date('2026-05-11T12:00:00.000Z'), + }); + + expect(sqlAnalysis.analyzeBatch).toHaveBeenCalledTimes(2); + expect(sqlAnalysis.analyzeBatch).toHaveBeenNthCalledWith( + 1, + [{ id: 'noise', sql: 'select count(*) from metabase.application_table' }], + 'postgres', + { catalog: { tables: [{ catalog: null, db: 'orbit_raw', name: 'accounts' }] } }, + ); + expect(sqlAnalysis.analyzeBatch).toHaveBeenNthCalledWith( + 2, + [{ id: 'noise', sql: 'select count(*) from metabase.application_table' }], + 'postgres', + undefined, + ); + expect(await readdir(join(stagedDir, 'tables'))).toEqual(['metabase.application_table.json']); + const manifest = await readJson>(stagedDir, 'manifest.json'); + expect(manifest.warnings).toContain('query_history_scope_floor_disabled:catalog_qualification_failed'); + }); +}); diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/types.test.ts b/packages/cli/test/context/ingest/adapters/historic-sql/types.test.ts similarity index 94% rename from packages/cli/src/context/ingest/adapters/historic-sql/types.test.ts rename to packages/cli/test/context/ingest/adapters/historic-sql/types.test.ts index 95b253a8..ca3d7e70 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/types.test.ts +++ b/packages/cli/test/context/ingest/adapters/historic-sql/types.test.ts @@ -5,7 +5,7 @@ import { stagedManifestSchema, stagedPatternsInputSchema, stagedTableInputSchema, -} from './types.js'; +} from '../../../../../src/context/ingest/adapters/historic-sql/types.js'; describe('historic-sql unified contracts', () => { it('parses minExecutions and service-account filters', () => { @@ -59,6 +59,7 @@ describe('historic-sql unified contracts', () => { expect( stagedTableInputSchema.parse({ table: 'public.orders', + tableRef: { catalog: null, db: 'public', name: 'orders' }, stats: { executionsBucket: '10-100', distinctUsersBucket: '2-5', @@ -81,7 +82,7 @@ describe('historic-sql unified contracts', () => { { id: 'pg:123', canonicalSql: 'select * from public.orders', - tablesTouched: ['public.orders'], + tablesTouched: [{ catalog: null, db: 'public', name: 'orders' }], executionsBucket: '10-100', distinctUsersBucket: '2-5', dialect: 'postgres', diff --git a/packages/cli/src/context/ingest/adapters/live-database/chunk.test.ts b/packages/cli/test/context/ingest/adapters/live-database/chunk.test.ts similarity index 92% rename from packages/cli/src/context/ingest/adapters/live-database/chunk.test.ts rename to packages/cli/test/context/ingest/adapters/live-database/chunk.test.ts index 2e38be9a..d3c5207c 100644 --- a/packages/cli/src/context/ingest/adapters/live-database/chunk.test.ts +++ b/packages/cli/test/context/ingest/adapters/live-database/chunk.test.ts @@ -2,9 +2,9 @@ import { mkdtemp } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import type { KtxSchemaSnapshot } from '../../../scan/types.js'; -import { chunkLiveDatabaseStagedDir } from './chunk.js'; -import { liveDatabaseTablePath, writeLiveDatabaseSnapshot } from './stage.js'; +import type { KtxSchemaSnapshot } from '../../../../../src/context/scan/types.js'; +import { chunkLiveDatabaseStagedDir } from '../../../../../src/context/ingest/adapters/live-database/chunk.js'; +import { liveDatabaseTablePath, writeLiveDatabaseSnapshot } from '../../../../../src/context/ingest/adapters/live-database/stage.js'; function snapshot(): KtxSchemaSnapshot { return { diff --git a/packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.test.ts b/packages/cli/test/context/ingest/adapters/live-database/daemon-introspection.test.ts similarity index 97% rename from packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.test.ts rename to packages/cli/test/context/ingest/adapters/live-database/daemon-introspection.test.ts index 9310f148..5cc6affb 100644 --- a/packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.test.ts +++ b/packages/cli/test/context/ingest/adapters/live-database/daemon-introspection.test.ts @@ -1,8 +1,8 @@ import { once } from 'node:events'; import { createServer } from 'node:http'; import { describe, expect, it, vi } from 'vitest'; -import { tableRefSet } from '../../../scan/table-ref.js'; -import { createDaemonLiveDatabaseIntrospection } from './daemon-introspection.js'; +import { tableRefSet } from '../../../../../src/context/scan/table-ref.js'; +import { createDaemonLiveDatabaseIntrospection } from '../../../../../src/context/ingest/adapters/live-database/daemon-introspection.js'; const daemonResponse = { connection_id: 'warehouse', @@ -155,7 +155,7 @@ describe('createDaemonLiveDatabaseIntrospection', () => { const introspection = createDaemonLiveDatabaseIntrospection({ connections: { warehouse: { - driver: 'postgresql', + driver: 'postgres', url: 'postgres://localhost:5432/warehouse', }, }, diff --git a/packages/cli/src/context/ingest/adapters/live-database/live-database.adapter.test.ts b/packages/cli/test/context/ingest/adapters/live-database/live-database.adapter.test.ts similarity index 94% rename from packages/cli/src/context/ingest/adapters/live-database/live-database.adapter.test.ts rename to packages/cli/test/context/ingest/adapters/live-database/live-database.adapter.test.ts index 6cd543e1..72c31446 100644 --- a/packages/cli/src/context/ingest/adapters/live-database/live-database.adapter.test.ts +++ b/packages/cli/test/context/ingest/adapters/live-database/live-database.adapter.test.ts @@ -2,8 +2,8 @@ import { mkdtemp, readdir, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it, vi } from 'vitest'; -import { tableRefSet, type KtxTableRefKey } from '../../../scan/table-ref.js'; -import { LiveDatabaseSourceAdapter } from './live-database.adapter.js'; +import { tableRefSet, type KtxTableRefKey } from '../../../../../src/context/scan/table-ref.js'; +import { LiveDatabaseSourceAdapter } from '../../../../../src/context/ingest/adapters/live-database/live-database.adapter.js'; describe('LiveDatabaseSourceAdapter', () => { it('fetches a schema snapshot through the introspection port', async () => { diff --git a/packages/cli/src/context/ingest/adapters/live-database/manifest.test.ts b/packages/cli/test/context/ingest/adapters/live-database/manifest.test.ts similarity index 99% rename from packages/cli/src/context/ingest/adapters/live-database/manifest.test.ts rename to packages/cli/test/context/ingest/adapters/live-database/manifest.test.ts index a97140a9..d32868ec 100644 --- a/packages/cli/src/context/ingest/adapters/live-database/manifest.test.ts +++ b/packages/cli/test/context/ingest/adapters/live-database/manifest.test.ts @@ -4,7 +4,7 @@ import { type LiveDatabaseManifestExistingDescriptions, type LiveDatabaseManifestJoinEntry, type LiveDatabaseManifestShard, -} from './manifest.js'; +} from '../../../../../src/context/ingest/adapters/live-database/manifest.js'; function shardObject(shards: Map): Record { return Object.fromEntries([...shards.entries()].sort(([a], [b]) => a.localeCompare(b))); diff --git a/packages/cli/src/context/ingest/adapters/live-database/stage.test.ts b/packages/cli/test/context/ingest/adapters/live-database/stage.test.ts similarity index 81% rename from packages/cli/src/context/ingest/adapters/live-database/stage.test.ts rename to packages/cli/test/context/ingest/adapters/live-database/stage.test.ts index 297071ae..b2382775 100644 --- a/packages/cli/src/context/ingest/adapters/live-database/stage.test.ts +++ b/packages/cli/test/context/ingest/adapters/live-database/stage.test.ts @@ -6,11 +6,12 @@ import { detectLiveDatabaseStagedDir, LIVE_DATABASE_FOREIGN_KEYS_FILE, LIVE_DATABASE_META_FILE, + LIVE_DATABASE_WARNINGS_FILE, liveDatabaseTablePath, readLiveDatabaseTableFiles, writeLiveDatabaseSnapshot, -} from './stage.js'; -import type { KtxSchemaSnapshot } from '../../../scan/types.js'; +} from '../../../../../src/context/ingest/adapters/live-database/stage.js'; +import type { KtxSchemaSnapshot } from '../../../../../src/context/scan/types.js'; function snapshot(): KtxSchemaSnapshot { return { @@ -145,6 +146,31 @@ describe('live-database staged snapshot files', () => { expect(connectionJson).not.toContain('pem-value'); }); + it('writes redacted scan warnings next to live database metadata', async () => { + const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-warning-stage-')); + await writeLiveDatabaseSnapshot(dir, { + ...snapshot(), + warnings: [ + { + code: 'constraint_discovery_unauthorized', + message: 'Skipped primary-key discovery in public (insufficient grants on system catalogs)', + recoverable: true, + metadata: { + schema: 'public', + kind: 'primary_key', + url: 'postgres://reader:secret@example.test/db', // pragma: allowlist secret + }, + }, + ], + }); + + const warningsJson = await readFile(join(dir, LIVE_DATABASE_WARNINGS_FILE), 'utf8'); + expect(warningsJson).toContain('"constraint_discovery_unauthorized"'); + expect(warningsJson).toContain('"schema": "public"'); + expect(warningsJson).toContain('"url": ""'); + expect(warningsJson).not.toContain('postgres://reader:secret@example.test/db'); // pragma: allowlist secret + }); + it('returns false for a directory that is missing live database metadata', async () => { const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-empty-')); expect(await detectLiveDatabaseStagedDir(dir)).toBe(false); diff --git a/packages/cli/src/context/ingest/adapters/looker/chunk.test.ts b/packages/cli/test/context/ingest/adapters/looker/chunk.test.ts similarity index 96% rename from packages/cli/src/context/ingest/adapters/looker/chunk.test.ts rename to packages/cli/test/context/ingest/adapters/looker/chunk.test.ts index 9d41d37a..f55a734b 100644 --- a/packages/cli/src/context/ingest/adapters/looker/chunk.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/chunk.test.ts @@ -2,8 +2,8 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { chunkLookerStagedDir } from './chunk.js'; -import { writeLookerEvidenceDocuments } from './evidence-documents.js'; +import { chunkLookerStagedDir } from '../../../../../src/context/ingest/adapters/looker/chunk.js'; +import { writeLookerEvidenceDocuments } from '../../../../../src/context/ingest/adapters/looker/evidence-documents.js'; async function writeJson(stagedDir: string, relPath: string, value: unknown): Promise { const abs = join(stagedDir, relPath); diff --git a/packages/cli/src/context/ingest/adapters/looker/client-boundary.test.ts b/packages/cli/test/context/ingest/adapters/looker/client-boundary.test.ts similarity index 77% rename from packages/cli/src/context/ingest/adapters/looker/client-boundary.test.ts rename to packages/cli/test/context/ingest/adapters/looker/client-boundary.test.ts index 9172e23f..48cd4e4b 100644 --- a/packages/cli/src/context/ingest/adapters/looker/client-boundary.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/client-boundary.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from 'vitest'; describe('LookerClient boundary', () => { it('does not import server or NestJS modules', async () => { - const source = await readFile(new URL('./client.ts', import.meta.url), 'utf-8'); + const source = await readFile(new URL('../../../../../src/context/ingest/adapters/looker/client.ts', import.meta.url), 'utf-8'); expect(source).not.toMatch(/@nestjs\/common/); expect(source).not.toMatch(/DataSourceClient/); diff --git a/packages/cli/src/context/ingest/adapters/looker/client.test.ts b/packages/cli/test/context/ingest/adapters/looker/client.test.ts similarity index 99% rename from packages/cli/src/context/ingest/adapters/looker/client.test.ts rename to packages/cli/test/context/ingest/adapters/looker/client.test.ts index 3b1822e0..e9aacb11 100644 --- a/packages/cli/src/context/ingest/adapters/looker/client.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/client.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { LookerClient, type LookerSdkPort } from './client.js'; +import { LookerClient, type LookerSdkPort } from '../../../../../src/context/ingest/adapters/looker/client.js'; const clientSecretParam = 'client_secret'; // pragma: allowlist secret diff --git a/packages/cli/src/context/ingest/adapters/looker/daemon-table-identifier-parser.test.ts b/packages/cli/test/context/ingest/adapters/looker/daemon-table-identifier-parser.test.ts similarity index 90% rename from packages/cli/src/context/ingest/adapters/looker/daemon-table-identifier-parser.test.ts rename to packages/cli/test/context/ingest/adapters/looker/daemon-table-identifier-parser.test.ts index 0da13d53..b8ae5b73 100644 --- a/packages/cli/src/context/ingest/adapters/looker/daemon-table-identifier-parser.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/daemon-table-identifier-parser.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { createDaemonLookerTableIdentifierParser } from './daemon-table-identifier-parser.js'; +import { createDaemonLookerTableIdentifierParser } from '../../../../../src/context/ingest/adapters/looker/daemon-table-identifier-parser.js'; describe('createDaemonLookerTableIdentifierParser', () => { it('posts parse items to the daemon endpoint', async () => { diff --git a/packages/cli/src/context/ingest/adapters/looker/detect.test.ts b/packages/cli/test/context/ingest/adapters/looker/detect.test.ts similarity index 94% rename from packages/cli/src/context/ingest/adapters/looker/detect.test.ts rename to packages/cli/test/context/ingest/adapters/looker/detect.test.ts index 1490bcfa..08e8472b 100644 --- a/packages/cli/src/context/ingest/adapters/looker/detect.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/detect.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { detectLookerStagedDir } from './detect.js'; +import { detectLookerStagedDir } from '../../../../../src/context/ingest/adapters/looker/detect.js'; async function touch(stagedDir: string, relPath: string, body = '{}\n'): Promise { const abs = join(stagedDir, relPath); diff --git a/packages/cli/src/context/ingest/adapters/looker/evidence-documents.test.ts b/packages/cli/test/context/ingest/adapters/looker/evidence-documents.test.ts similarity index 98% rename from packages/cli/src/context/ingest/adapters/looker/evidence-documents.test.ts rename to packages/cli/test/context/ingest/adapters/looker/evidence-documents.test.ts index 6d4545ca..55da5fc9 100644 --- a/packages/cli/src/context/ingest/adapters/looker/evidence-documents.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/evidence-documents.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { getLookerTriageSignals, writeLookerEvidenceDocuments } from './evidence-documents.js'; +import { getLookerTriageSignals, writeLookerEvidenceDocuments } from '../../../../../src/context/ingest/adapters/looker/evidence-documents.js'; async function writeJson(root: string, relPath: string, value: unknown): Promise { const target = join(root, relPath); diff --git a/packages/cli/src/context/ingest/adapters/looker/factory.test.ts b/packages/cli/test/context/ingest/adapters/looker/factory.test.ts similarity index 86% rename from packages/cli/src/context/ingest/adapters/looker/factory.test.ts rename to packages/cli/test/context/ingest/adapters/looker/factory.test.ts index d68be942..ed5c5633 100644 --- a/packages/cli/src/context/ingest/adapters/looker/factory.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/factory.test.ts @@ -1,13 +1,13 @@ import { describe, expect, it, vi } from 'vitest'; -import type { FetchContext } from '../../types.js'; -import type { LookerSdkPort } from './client.js'; +import type { FetchContext } from '../../../../../src/context/ingest/types.js'; +import type { LookerSdkPort } from '../../../../../src/context/ingest/adapters/looker/client.js'; import { DefaultLookerClientFactory, DefaultLookerConnectionClientFactory, type LookerCredentialResolver, -} from './factory.js'; -import type { LookerRuntimeClient } from './fetch.js'; -import type { LookerPullConfig } from './types.js'; +} from '../../../../../src/context/ingest/adapters/looker/factory.js'; +import type { LookerRuntimeClient } from '../../../../../src/context/ingest/adapters/looker/fetch.js'; +import type { LookerPullConfig } from '../../../../../src/context/ingest/adapters/looker/types.js'; function sdk(): LookerSdkPort { return { diff --git a/packages/cli/src/context/ingest/adapters/looker/fetch-report.test.ts b/packages/cli/test/context/ingest/adapters/looker/fetch-report.test.ts similarity index 97% rename from packages/cli/src/context/ingest/adapters/looker/fetch-report.test.ts rename to packages/cli/test/context/ingest/adapters/looker/fetch-report.test.ts index 157a6770..f9f5bcd3 100644 --- a/packages/cli/src/context/ingest/adapters/looker/fetch-report.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/fetch-report.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { readLookerFetchReport, writeLookerFetchReport } from './fetch-report.js'; +import { readLookerFetchReport, writeLookerFetchReport } from '../../../../../src/context/ingest/adapters/looker/fetch-report.js'; describe('Looker staged fetch report', () => { let stagedDir: string; diff --git a/packages/cli/src/context/ingest/adapters/looker/fetch.test.ts b/packages/cli/test/context/ingest/adapters/looker/fetch.test.ts similarity index 99% rename from packages/cli/src/context/ingest/adapters/looker/fetch.test.ts rename to packages/cli/test/context/ingest/adapters/looker/fetch.test.ts index 2b18a3dd..95edb382 100644 --- a/packages/cli/src/context/ingest/adapters/looker/fetch.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/fetch.test.ts @@ -2,8 +2,8 @@ import { mkdtemp, readdir, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { chunkLookerStagedDir } from './chunk.js'; -import { fetchLookerRuntimeBundle, type LookerRuntimeClient } from './fetch.js'; +import { chunkLookerStagedDir } from '../../../../../src/context/ingest/adapters/looker/chunk.js'; +import { fetchLookerRuntimeBundle, type LookerRuntimeClient } from '../../../../../src/context/ingest/adapters/looker/fetch.js'; const connectionId = '11111111-1111-4111-8111-111111111111'; diff --git a/packages/cli/src/context/ingest/adapters/looker/local-runtime-store.test.ts b/packages/cli/test/context/ingest/adapters/looker/local-runtime-store.test.ts similarity index 96% rename from packages/cli/src/context/ingest/adapters/looker/local-runtime-store.test.ts rename to packages/cli/test/context/ingest/adapters/looker/local-runtime-store.test.ts index 3f9bbdc5..ff7b1f5c 100644 --- a/packages/cli/src/context/ingest/adapters/looker/local-runtime-store.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/local-runtime-store.test.ts @@ -2,7 +2,7 @@ import { mkdtemp } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { LocalLookerRuntimeStore } from './local-runtime-store.js'; +import { LocalLookerRuntimeStore } from '../../../../../src/context/ingest/adapters/looker/local-runtime-store.js'; describe('LocalLookerRuntimeStore', () => { async function store() { diff --git a/packages/cli/src/context/ingest/adapters/looker/looker.adapter.test.ts b/packages/cli/test/context/ingest/adapters/looker/looker.adapter.test.ts similarity index 95% rename from packages/cli/src/context/ingest/adapters/looker/looker.adapter.test.ts rename to packages/cli/test/context/ingest/adapters/looker/looker.adapter.test.ts index 64a35622..90623a37 100644 --- a/packages/cli/src/context/ingest/adapters/looker/looker.adapter.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/looker.adapter.test.ts @@ -2,8 +2,8 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { LookerRuntimeClient } from './fetch.js'; -import { LookerSourceAdapter } from './looker.adapter.js'; +import type { LookerRuntimeClient } from '../../../../../src/context/ingest/adapters/looker/fetch.js'; +import { LookerSourceAdapter } from '../../../../../src/context/ingest/adapters/looker/looker.adapter.js'; const connectionId = '11111111-1111-4111-8111-111111111111'; diff --git a/packages/cli/src/context/ingest/adapters/looker/mapping.test.ts b/packages/cli/test/context/ingest/adapters/looker/mapping.test.ts similarity index 97% rename from packages/cli/src/context/ingest/adapters/looker/mapping.test.ts rename to packages/cli/test/context/ingest/adapters/looker/mapping.test.ts index 796a9f05..0ac9c067 100644 --- a/packages/cli/src/context/ingest/adapters/looker/mapping.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/mapping.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import type { StagedExploreFile, StagedLookmlModelsFile } from './types.js'; +import type { StagedExploreFile, StagedLookmlModelsFile } from '../../../../../src/context/ingest/adapters/looker/types.js'; import { buildLookerPullConfigFromInputs, collectExploreParseItems, @@ -12,7 +12,7 @@ import { suggestKtxConnectionForLookerConnection, validateLookerMappings, validateLookerWarehouseTarget, -} from './mapping.js'; +} from '../../../../../src/context/ingest/adapters/looker/mapping.js'; const liveConnections = [ { @@ -72,7 +72,8 @@ describe('looker dialect and target validation helpers', () => { it('maps Looker dialect names to KTX connection types', () => { expect(lookerDialectToConnectionType('bigquery_standard_sql')).toBe('BIGQUERY'); expect(lookerDialectToConnectionType('postgres')).toBe('POSTGRESQL'); - expect(lookerDialectToConnectionType('mssql')).toBe('SQLSERVER'); + expect(lookerDialectToConnectionType('mssql')).toBeNull(); + expect(lookerDialectToConnectionType('tsql')).toBeNull(); expect(lookerDialectToConnectionType('unknown')).toBeNull(); }); diff --git a/packages/cli/src/context/ingest/adapters/looker/reconcile.test.ts b/packages/cli/test/context/ingest/adapters/looker/reconcile.test.ts similarity index 84% rename from packages/cli/src/context/ingest/adapters/looker/reconcile.test.ts rename to packages/cli/test/context/ingest/adapters/looker/reconcile.test.ts index 09e8685f..2ffcdaeb 100644 --- a/packages/cli/src/context/ingest/adapters/looker/reconcile.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/reconcile.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { buildLookerReconcileNotes } from './reconcile.js'; +import { buildLookerReconcileNotes } from '../../../../../src/context/ingest/adapters/looker/reconcile.js'; describe('buildLookerReconcileNotes', () => { it('instructs reconciliation to record subsumed provenance', () => { diff --git a/packages/cli/src/context/ingest/adapters/looker/scope.test.ts b/packages/cli/test/context/ingest/adapters/looker/scope.test.ts similarity index 98% rename from packages/cli/src/context/ingest/adapters/looker/scope.test.ts rename to packages/cli/test/context/ingest/adapters/looker/scope.test.ts index d7c2c56e..55592761 100644 --- a/packages/cli/src/context/ingest/adapters/looker/scope.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/scope.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { describeLookerScope, hashLookerScope, isPathInLookerScope } from './scope.js'; +import { describeLookerScope, hashLookerScope, isPathInLookerScope } from '../../../../../src/context/ingest/adapters/looker/scope.js'; async function writeJson(stagedDir: string, relPath: string, value: unknown): Promise { const abs = join(stagedDir, relPath); diff --git a/packages/cli/src/context/ingest/adapters/looker/target-connections.test.ts b/packages/cli/test/context/ingest/adapters/looker/target-connections.test.ts similarity index 95% rename from packages/cli/src/context/ingest/adapters/looker/target-connections.test.ts rename to packages/cli/test/context/ingest/adapters/looker/target-connections.test.ts index 10b2d892..5497914d 100644 --- a/packages/cli/src/context/ingest/adapters/looker/target-connections.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/target-connections.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { listLookerTargetConnectionIds } from './target-connections.js'; +import { listLookerTargetConnectionIds } from '../../../../../src/context/ingest/adapters/looker/target-connections.js'; describe('listLookerTargetConnectionIds', () => { let stagedDir: string; diff --git a/packages/cli/src/context/ingest/adapters/looker/tools/looker-query-to-sl.tool.test.ts b/packages/cli/test/context/ingest/adapters/looker/tools/looker-query-to-sl.tool.test.ts similarity index 97% rename from packages/cli/src/context/ingest/adapters/looker/tools/looker-query-to-sl.tool.test.ts rename to packages/cli/test/context/ingest/adapters/looker/tools/looker-query-to-sl.tool.test.ts index d4cd857b..2f8dc4a4 100644 --- a/packages/cli/src/context/ingest/adapters/looker/tools/looker-query-to-sl.tool.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/tools/looker-query-to-sl.tool.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import type { ToolOutput } from '../../../../../context/tools/base-tool.js'; -import { buildLookerSlProposal, createLookerQueryToSlTool, type LookerSlProposal } from './looker-query-to-sl.tool.js'; +import type { ToolOutput } from '../../../../../../src/context/tools/base-tool.js'; +import { buildLookerSlProposal, createLookerQueryToSlTool, type LookerSlProposal } from '../../../../../../src/context/ingest/adapters/looker/tools/looker-query-to-sl.tool.js'; describe('buildLookerSlProposal', () => { it('suggests a measure and segment for an aggregated filtered Looker query', () => { diff --git a/packages/cli/src/context/ingest/adapters/looker/types.test.ts b/packages/cli/test/context/ingest/adapters/looker/types.test.ts similarity index 98% rename from packages/cli/src/context/ingest/adapters/looker/types.test.ts rename to packages/cli/test/context/ingest/adapters/looker/types.test.ts index 2a4c2b8c..113f9fe3 100644 --- a/packages/cli/src/context/ingest/adapters/looker/types.test.ts +++ b/packages/cli/test/context/ingest/adapters/looker/types.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { parsedTargetTableSchema } from '../../parsed-target-table.js'; +import { parsedTargetTableSchema } from '../../../../../src/context/ingest/parsed-target-table.js'; import { lookerPullConfigSchema, parseLookerPullConfig, @@ -11,7 +11,7 @@ import { stagedLookerSignalsFileSchema, stagedLookFileSchema, stagedSyncConfigSchema, -} from './types.js'; +} from '../../../../../src/context/ingest/adapters/looker/types.js'; describe('Looker staged runtime schemas', () => { it('parses pull config and staged sync config', () => { diff --git a/packages/cli/src/context/ingest/adapters/lookml/chunk.test.ts b/packages/cli/test/context/ingest/adapters/lookml/chunk.test.ts similarity index 97% rename from packages/cli/src/context/ingest/adapters/lookml/chunk.test.ts rename to packages/cli/test/context/ingest/adapters/lookml/chunk.test.ts index e9a8b5f3..5898a2a9 100644 --- a/packages/cli/src/context/ingest/adapters/lookml/chunk.test.ts +++ b/packages/cli/test/context/ingest/adapters/lookml/chunk.test.ts @@ -1,9 +1,9 @@ import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { chunkLookmlProject } from './chunk.js'; -import { type ParsedLookmlProject, parseLookmlStagedDir } from './parse.js'; +import { chunkLookmlProject } from '../../../../../src/context/ingest/adapters/lookml/chunk.js'; +import { type ParsedLookmlProject, parseLookmlStagedDir } from '../../../../../src/context/ingest/adapters/lookml/parse.js'; -const FIXTURE_ROOT = join(__dirname, '../../../../test/fixtures/lookml'); +const FIXTURE_ROOT = join(__dirname, '../../../../fixtures/lookml'); describe('chunkLookmlProject — first run', () => { it('single-model bundle → 1 WU with model + all views in rawFiles', async () => { diff --git a/packages/cli/src/context/ingest/adapters/lookml/detect.test.ts b/packages/cli/test/context/ingest/adapters/lookml/detect.test.ts similarity index 94% rename from packages/cli/src/context/ingest/adapters/lookml/detect.test.ts rename to packages/cli/test/context/ingest/adapters/lookml/detect.test.ts index 040c1788..12640d07 100644 --- a/packages/cli/src/context/ingest/adapters/lookml/detect.test.ts +++ b/packages/cli/test/context/ingest/adapters/lookml/detect.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { detectLookmlStagedDir } from './detect.js'; +import { detectLookmlStagedDir } from '../../../../../src/context/ingest/adapters/lookml/detect.js'; describe('detectLookmlStagedDir', () => { let stagedDir: string; diff --git a/packages/cli/src/context/ingest/adapters/lookml/fetch-report.test.ts b/packages/cli/test/context/ingest/adapters/lookml/fetch-report.test.ts similarity index 95% rename from packages/cli/src/context/ingest/adapters/lookml/fetch-report.test.ts rename to packages/cli/test/context/ingest/adapters/lookml/fetch-report.test.ts index ffeb52fb..4aa91e82 100644 --- a/packages/cli/src/context/ingest/adapters/lookml/fetch-report.test.ts +++ b/packages/cli/test/context/ingest/adapters/lookml/fetch-report.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import type { ParsedLookmlProject } from './parse.js'; +import type { ParsedLookmlProject } from '../../../../../src/context/ingest/adapters/lookml/parse.js'; import { LOOKML_FETCH_REPORT_FILE, LOOKML_MISMATCHED_MODELS_FILE, @@ -10,7 +10,7 @@ import { readLookmlFetchReport, readLookmlMismatchedModelNames, writeLookmlValidationArtifacts, -} from './fetch-report.js'; +} from '../../../../../src/context/ingest/adapters/lookml/fetch-report.js'; function project(models: ParsedLookmlProject['models']): ParsedLookmlProject { return { models, views: [], dashboards: [], allPaths: models.map((m) => m.path) }; diff --git a/packages/cli/src/context/ingest/adapters/lookml/fetch.test.ts b/packages/cli/test/context/ingest/adapters/lookml/fetch.test.ts similarity index 95% rename from packages/cli/src/context/ingest/adapters/lookml/fetch.test.ts rename to packages/cli/test/context/ingest/adapters/lookml/fetch.test.ts index a0c293e7..05ee9bfc 100644 --- a/packages/cli/src/context/ingest/adapters/lookml/fetch.test.ts +++ b/packages/cli/test/context/ingest/adapters/lookml/fetch.test.ts @@ -3,10 +3,10 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { makeLocalGitRepo } from '../../../test/make-local-git-repo.js'; -import { fetchLookmlRepo } from './fetch.js'; -import type { LookmlPullConfig } from './pull-config.js'; +import { fetchLookmlRepo } from '../../../../../src/context/ingest/adapters/lookml/fetch.js'; +import type { LookmlPullConfig } from '../../../../../src/context/ingest/adapters/lookml/pull-config.js'; -const FIXTURE_ROOT = join(__dirname, '../../../../test/fixtures/lookml'); +const FIXTURE_ROOT = join(__dirname, '../../../../fixtures/lookml'); function pullConfig(overrides: Partial & Pick): LookmlPullConfig { return { diff --git a/packages/cli/src/context/ingest/adapters/lookml/graph.test.ts b/packages/cli/test/context/ingest/adapters/lookml/graph.test.ts similarity index 96% rename from packages/cli/src/context/ingest/adapters/lookml/graph.test.ts rename to packages/cli/test/context/ingest/adapters/lookml/graph.test.ts index c1efd701..d0df93e3 100644 --- a/packages/cli/src/context/ingest/adapters/lookml/graph.test.ts +++ b/packages/cli/test/context/ingest/adapters/lookml/graph.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { buildLookmlGraph } from './graph.js'; -import type { ParsedLookmlProject } from './parse.js'; +import { buildLookmlGraph } from '../../../../../src/context/ingest/adapters/lookml/graph.js'; +import type { ParsedLookmlProject } from '../../../../../src/context/ingest/adapters/lookml/parse.js'; type LooseParsedLookmlProject = Omit, 'models' | 'views'> & { models?: Array & { connectionName?: string | null }>; diff --git a/packages/cli/src/context/ingest/adapters/lookml/lookml.adapter.test.ts b/packages/cli/test/context/ingest/adapters/lookml/lookml.adapter.test.ts similarity index 92% rename from packages/cli/src/context/ingest/adapters/lookml/lookml.adapter.test.ts rename to packages/cli/test/context/ingest/adapters/lookml/lookml.adapter.test.ts index d22597b9..0d23cc95 100644 --- a/packages/cli/src/context/ingest/adapters/lookml/lookml.adapter.test.ts +++ b/packages/cli/test/context/ingest/adapters/lookml/lookml.adapter.test.ts @@ -3,8 +3,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { makeLocalGitRepo } from '../../../test/make-local-git-repo.js'; -import { LOOKML_FETCH_REPORT_FILE } from './fetch-report.js'; -import { LookmlSourceAdapter } from './lookml.adapter.js'; +import { LOOKML_FETCH_REPORT_FILE } from '../../../../../src/context/ingest/adapters/lookml/fetch-report.js'; +import { LookmlSourceAdapter } from '../../../../../src/context/ingest/adapters/lookml/lookml.adapter.js'; describe('LookmlSourceAdapter validation sidecars', () => { let tmpRoot: string; diff --git a/packages/cli/src/context/ingest/adapters/lookml/parse.test.ts b/packages/cli/test/context/ingest/adapters/lookml/parse.test.ts similarity index 97% rename from packages/cli/src/context/ingest/adapters/lookml/parse.test.ts rename to packages/cli/test/context/ingest/adapters/lookml/parse.test.ts index 84ce5b5a..5372dc63 100644 --- a/packages/cli/src/context/ingest/adapters/lookml/parse.test.ts +++ b/packages/cli/test/context/ingest/adapters/lookml/parse.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { parseLookmlStagedDir } from './parse.js'; +import { parseLookmlStagedDir } from '../../../../../src/context/ingest/adapters/lookml/parse.js'; describe('parseLookmlStagedDir', () => { let stagedDir: string; diff --git a/packages/cli/src/context/ingest/adapters/lookml/pull-config.test.ts b/packages/cli/test/context/ingest/adapters/lookml/pull-config.test.ts similarity index 98% rename from packages/cli/src/context/ingest/adapters/lookml/pull-config.test.ts rename to packages/cli/test/context/ingest/adapters/lookml/pull-config.test.ts index 2edec99f..34cd9d8e 100644 --- a/packages/cli/src/context/ingest/adapters/lookml/pull-config.test.ts +++ b/packages/cli/test/context/ingest/adapters/lookml/pull-config.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { parseLookmlPullConfig, pullConfigFromIntegrationConfig } from './pull-config.js'; +import { parseLookmlPullConfig, pullConfigFromIntegrationConfig } from '../../../../../src/context/ingest/adapters/lookml/pull-config.js'; describe('lookml pull config', () => { it('parses a minimal valid config with defaulted branch', () => { diff --git a/packages/cli/src/context/ingest/adapters/metabase/card-references.test.ts b/packages/cli/test/context/ingest/adapters/metabase/card-references.test.ts similarity index 96% rename from packages/cli/src/context/ingest/adapters/metabase/card-references.test.ts rename to packages/cli/test/context/ingest/adapters/metabase/card-references.test.ts index 8c179710..f619490f 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/card-references.test.ts +++ b/packages/cli/test/context/ingest/adapters/metabase/card-references.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { CardReferenceCycleError, expandCardReferences } from './card-references.js'; +import { CardReferenceCycleError, expandCardReferences } from '../../../../../src/context/ingest/adapters/metabase/card-references.js'; describe('expandCardReferences', () => { const fetchCard = (id: number): Promise<{ native_query: string }> => { diff --git a/packages/cli/src/context/ingest/adapters/metabase/chunk.test.ts b/packages/cli/test/context/ingest/adapters/metabase/chunk.test.ts similarity index 97% rename from packages/cli/src/context/ingest/adapters/metabase/chunk.test.ts rename to packages/cli/test/context/ingest/adapters/metabase/chunk.test.ts index 1991e147..333faf26 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/chunk.test.ts +++ b/packages/cli/test/context/ingest/adapters/metabase/chunk.test.ts @@ -2,10 +2,10 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { chunkMetabaseStagedDir } from './chunk.js'; -import { stagedSyncConfigSchema } from './types.js'; +import { chunkMetabaseStagedDir } from '../../../../../src/context/ingest/adapters/metabase/chunk.js'; +import { stagedSyncConfigSchema } from '../../../../../src/context/ingest/adapters/metabase/types.js'; -const FIXTURES = resolve(__dirname, '../../../../test/fixtures/metabase'); +const FIXTURES = resolve(__dirname, '../../../../fixtures/metabase'); const SIMPLE = join(FIXTURES, 'simple'); const MULTI = join(FIXTURES, 'multi-collection'); const CARD_REF = join(FIXTURES, 'card-ref'); diff --git a/packages/cli/src/context/ingest/adapters/metabase/client-boundary.test.ts b/packages/cli/test/context/ingest/adapters/metabase/client-boundary.test.ts similarity index 92% rename from packages/cli/src/context/ingest/adapters/metabase/client-boundary.test.ts rename to packages/cli/test/context/ingest/adapters/metabase/client-boundary.test.ts index 7df6691e..7e8a0e2e 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/client-boundary.test.ts +++ b/packages/cli/test/context/ingest/adapters/metabase/client-boundary.test.ts @@ -1,9 +1,9 @@ import { readFile } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; +import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; -const metabaseDir = dirname(fileURLToPath(import.meta.url)); +const metabaseDir = fileURLToPath(new URL('../../../../../src/context/ingest/adapters/metabase/', import.meta.url)); async function readMetabaseFile(name: string): Promise { return readFile(join(metabaseDir, name), 'utf-8'); diff --git a/packages/cli/src/context/ingest/adapters/metabase/client-port.test.ts b/packages/cli/test/context/ingest/adapters/metabase/client-port.test.ts similarity index 92% rename from packages/cli/src/context/ingest/adapters/metabase/client-port.test.ts rename to packages/cli/test/context/ingest/adapters/metabase/client-port.test.ts index 8f775b56..3a9e2732 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/client-port.test.ts +++ b/packages/cli/test/context/ingest/adapters/metabase/client-port.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import type { FetchContext } from '../../types.js'; +import type { FetchContext } from '../../../../../src/context/ingest/types.js'; import { IngestMetabaseClientFactory, type MetabaseCard, @@ -8,8 +8,8 @@ import { type MetabaseRuntimeClient, type MetabaseTemplateTag, type TestConnectionResult, -} from './client-port.js'; -import type { MetabasePullConfig } from './types.js'; +} from '../../../../../src/context/ingest/adapters/metabase/client-port.js'; +import type { MetabasePullConfig } from '../../../../../src/context/ingest/adapters/metabase/types.js'; function makeRuntimeClient(): MetabaseRuntimeClient { return { diff --git a/packages/cli/src/context/ingest/adapters/metabase/client.test.ts b/packages/cli/test/context/ingest/adapters/metabase/client.test.ts similarity index 98% rename from packages/cli/src/context/ingest/adapters/metabase/client.test.ts rename to packages/cli/test/context/ingest/adapters/metabase/client.test.ts index 3d45a276..5ab2fd09 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/client.test.ts +++ b/packages/cli/test/context/ingest/adapters/metabase/client.test.ts @@ -5,8 +5,8 @@ import { getDummyValueForWidgetType, MetabaseClient, stripOptionalClauses, -} from './client.js'; -import type { MetabaseCard, MetabaseTemplateTag } from './client-port.js'; +} from '../../../../../src/context/ingest/adapters/metabase/client.js'; +import type { MetabaseCard, MetabaseTemplateTag } from '../../../../../src/context/ingest/adapters/metabase/client-port.js'; const runtime = { apiUrl: 'https://metabase.example.test/api', diff --git a/packages/cli/src/context/ingest/adapters/metabase/detect.test.ts b/packages/cli/test/context/ingest/adapters/metabase/detect.test.ts similarity index 94% rename from packages/cli/src/context/ingest/adapters/metabase/detect.test.ts rename to packages/cli/test/context/ingest/adapters/metabase/detect.test.ts index 816bbef6..44abe951 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/detect.test.ts +++ b/packages/cli/test/context/ingest/adapters/metabase/detect.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { detectMetabaseStagedDir } from './detect.js'; +import { detectMetabaseStagedDir } from '../../../../../src/context/ingest/adapters/metabase/detect.js'; async function touch(stagedDir: string, relPath: string, body: string): Promise { const abs = join(stagedDir, relPath); diff --git a/packages/cli/src/context/ingest/adapters/metabase/fanout-planner.test.ts b/packages/cli/test/context/ingest/adapters/metabase/fanout-planner.test.ts similarity index 94% rename from packages/cli/src/context/ingest/adapters/metabase/fanout-planner.test.ts rename to packages/cli/test/context/ingest/adapters/metabase/fanout-planner.test.ts index cb275472..d1e9e7e5 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/fanout-planner.test.ts +++ b/packages/cli/test/context/ingest/adapters/metabase/fanout-planner.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { planMetabaseFanoutChildren } from './fanout-planner.js'; +import { planMetabaseFanoutChildren } from '../../../../../src/context/ingest/adapters/metabase/fanout-planner.js'; describe('planMetabaseFanoutChildren', () => { it('builds ordered child plans for sync-enabled mapped Metabase databases', () => { diff --git a/packages/cli/src/context/ingest/adapters/metabase/fetch-scope.test.ts b/packages/cli/test/context/ingest/adapters/metabase/fetch-scope.test.ts similarity index 96% rename from packages/cli/src/context/ingest/adapters/metabase/fetch-scope.test.ts rename to packages/cli/test/context/ingest/adapters/metabase/fetch-scope.test.ts index 9768c0c9..e2b1c6e7 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/fetch-scope.test.ts +++ b/packages/cli/test/context/ingest/adapters/metabase/fetch-scope.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { computeFetchScope, type FetchScope, hashScope, isPathInMetabaseScope } from './fetch-scope.js'; -import type { StagedSyncConfig } from './types.js'; +import { computeFetchScope, type FetchScope, hashScope, isPathInMetabaseScope } from '../../../../../src/context/ingest/adapters/metabase/fetch-scope.js'; +import type { StagedSyncConfig } from '../../../../../src/context/ingest/adapters/metabase/types.js'; const BASE_CONFIG = { metabaseConnectionId: 'a1b2c3d4-e5f6-4789-9abc-def012345678', diff --git a/packages/cli/src/context/ingest/adapters/metabase/fetch.test.ts b/packages/cli/test/context/ingest/adapters/metabase/fetch.test.ts similarity index 99% rename from packages/cli/src/context/ingest/adapters/metabase/fetch.test.ts rename to packages/cli/test/context/ingest/adapters/metabase/fetch.test.ts index 1f93765e..4e067663 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/fetch.test.ts +++ b/packages/cli/test/context/ingest/adapters/metabase/fetch.test.ts @@ -2,8 +2,8 @@ import { mkdtemp, readdir, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { FetchContext } from '../../types.js'; -import { fetchMetabaseBundle } from './fetch.js'; +import type { FetchContext } from '../../../../../src/context/ingest/types.js'; +import { fetchMetabaseBundle } from '../../../../../src/context/ingest/adapters/metabase/fetch.js'; const metabaseConnectionId = 'a1b2c3d4-e5f6-4789-9abc-def012345678'; const targetConnectionId = 'b2c3d4e5-f6a7-4890-abcd-ef0123456789'; diff --git a/packages/cli/src/context/ingest/adapters/metabase/local-metabase.adapter.test.ts b/packages/cli/test/context/ingest/adapters/metabase/local-metabase.adapter.test.ts similarity index 91% rename from packages/cli/src/context/ingest/adapters/metabase/local-metabase.adapter.test.ts rename to packages/cli/test/context/ingest/adapters/metabase/local-metabase.adapter.test.ts index c20a65ac..1f860557 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/local-metabase.adapter.test.ts +++ b/packages/cli/test/context/ingest/adapters/metabase/local-metabase.adapter.test.ts @@ -2,8 +2,8 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import type { KtxProjectConnectionConfig } from '../../../../context/project/config.js'; -import { metabaseRuntimeConfigFromLocalConnection } from './local-metabase.adapter.js'; +import type { KtxProjectConnectionConfig } from '../../../../../src/context/project/config.js'; +import { metabaseRuntimeConfigFromLocalConnection } from '../../../../../src/context/ingest/adapters/metabase/local-metabase.adapter.js'; describe('metabaseRuntimeConfigFromLocalConnection', () => { let tempDir: string; diff --git a/packages/cli/src/context/ingest/adapters/metabase/local-source-state-store.test.ts b/packages/cli/test/context/ingest/adapters/metabase/local-source-state-store.test.ts similarity index 93% rename from packages/cli/src/context/ingest/adapters/metabase/local-source-state-store.test.ts rename to packages/cli/test/context/ingest/adapters/metabase/local-source-state-store.test.ts index 0225f398..3f80f7af 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/local-source-state-store.test.ts +++ b/packages/cli/test/context/ingest/adapters/metabase/local-source-state-store.test.ts @@ -2,9 +2,9 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { buildDefaultKtxProjectConfig } from '../../../../context/project/config.js'; -import { connectionConfigSchema } from '../../../project/driver-schemas.js'; -import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './local-source-state-store.js'; +import { buildDefaultKtxProjectConfig } from '../../../../../src/context/project/config.js'; +import { connectionConfigSchema } from '../../../../../src/context/project/driver-schemas.js'; +import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from '../../../../../src/context/ingest/adapters/metabase/local-source-state-store.js'; describe('Metabase YAML source state and discovery cache', () => { let tempDir: string; diff --git a/packages/cli/src/context/ingest/adapters/metabase/mapping.test.ts b/packages/cli/test/context/ingest/adapters/metabase/mapping.test.ts similarity index 98% rename from packages/cli/src/context/ingest/adapters/metabase/mapping.test.ts rename to packages/cli/test/context/ingest/adapters/metabase/mapping.test.ts index e347390c..b6f1f354 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/mapping.test.ts +++ b/packages/cli/test/context/ingest/adapters/metabase/mapping.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import type { MetabaseRuntimeClient } from './client-port.js'; +import type { MetabaseRuntimeClient } from '../../../../../src/context/ingest/adapters/metabase/client-port.js'; import { METABASE_ENGINE_TO_CONNECTION_TYPE, computeMetabaseMappingDrift, @@ -9,7 +9,7 @@ import { refreshMetabaseMapping, validateMappingPhysicalMatch, validateMetabaseMappings, -} from './mapping.js'; +} from '../../../../../src/context/ingest/adapters/metabase/mapping.js'; describe('discoverMetabaseDatabases', () => { it('filters sample databases and extracts host plus database names from Metabase details', async () => { diff --git a/packages/cli/src/context/ingest/adapters/metabase/metabase.adapter.test.ts b/packages/cli/test/context/ingest/adapters/metabase/metabase.adapter.test.ts similarity index 97% rename from packages/cli/src/context/ingest/adapters/metabase/metabase.adapter.test.ts rename to packages/cli/test/context/ingest/adapters/metabase/metabase.adapter.test.ts index a22c1f3b..a22e8b28 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/metabase.adapter.test.ts +++ b/packages/cli/test/context/ingest/adapters/metabase/metabase.adapter.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { MetabaseSourceAdapter } from './metabase.adapter.js'; +import { MetabaseSourceAdapter } from '../../../../../src/context/ingest/adapters/metabase/metabase.adapter.js'; describe('MetabaseSourceAdapter', () => { let stagedDir: string; diff --git a/packages/cli/src/context/ingest/adapters/metabase/serialize-card.test.ts b/packages/cli/test/context/ingest/adapters/metabase/serialize-card.test.ts similarity index 98% rename from packages/cli/src/context/ingest/adapters/metabase/serialize-card.test.ts rename to packages/cli/test/context/ingest/adapters/metabase/serialize-card.test.ts index ff10ce59..c743b74d 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/serialize-card.test.ts +++ b/packages/cli/test/context/ingest/adapters/metabase/serialize-card.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { extractReferencedCardIds, serializeCard } from './serialize-card.js'; +import { extractReferencedCardIds, serializeCard } from '../../../../../src/context/ingest/adapters/metabase/serialize-card.js'; describe('extractReferencedCardIds', () => { it('pulls ids out of template tags with type=card', () => { diff --git a/packages/cli/src/context/ingest/adapters/metabase/types.test.ts b/packages/cli/test/context/ingest/adapters/metabase/types.test.ts similarity index 97% rename from packages/cli/src/context/ingest/adapters/metabase/types.test.ts rename to packages/cli/test/context/ingest/adapters/metabase/types.test.ts index 4a445d89..fd92090b 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/types.test.ts +++ b/packages/cli/test/context/ingest/adapters/metabase/types.test.ts @@ -4,7 +4,7 @@ import { parseMetabasePullConfig, stagedCardFileSchema, stagedSyncConfigSchema, -} from './types.js'; +} from '../../../../../src/context/ingest/adapters/metabase/types.js'; describe('metabase adapter types', () => { it('parses a valid MetabasePullConfig', () => { diff --git a/packages/cli/src/context/ingest/adapters/metricflow/chunk.test.ts b/packages/cli/test/context/ingest/adapters/metricflow/chunk.test.ts similarity index 94% rename from packages/cli/src/context/ingest/adapters/metricflow/chunk.test.ts rename to packages/cli/test/context/ingest/adapters/metricflow/chunk.test.ts index 88062fb3..b783087b 100644 --- a/packages/cli/src/context/ingest/adapters/metricflow/chunk.test.ts +++ b/packages/cli/test/context/ingest/adapters/metricflow/chunk.test.ts @@ -1,9 +1,9 @@ import { join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { chunkMetricFlowProject } from './chunk.js'; -import { parseMetricFlowStagedDir } from './parse.js'; +import { chunkMetricFlowProject } from '../../../../../src/context/ingest/adapters/metricflow/chunk.js'; +import { parseMetricFlowStagedDir } from '../../../../../src/context/ingest/adapters/metricflow/parse.js'; -const FIXTURES = resolve(__dirname, '../../../../test/fixtures/metricflow'); +const FIXTURES = resolve(__dirname, '../../../../fixtures/metricflow'); const SINGLE = join(FIXTURES, 'single-model'); const EXTENDS_CHAIN = join(FIXTURES, 'extends-chain'); const MULTI = join(FIXTURES, 'multi-component'); diff --git a/packages/cli/src/context/ingest/adapters/metricflow/deep-parse.test.ts b/packages/cli/test/context/ingest/adapters/metricflow/deep-parse.test.ts similarity index 99% rename from packages/cli/src/context/ingest/adapters/metricflow/deep-parse.test.ts rename to packages/cli/test/context/ingest/adapters/metricflow/deep-parse.test.ts index 8896db68..e6747db1 100644 --- a/packages/cli/src/context/ingest/adapters/metricflow/deep-parse.test.ts +++ b/packages/cli/test/context/ingest/adapters/metricflow/deep-parse.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, it } from 'vitest'; -import { parseMetricflowFiles, translateMetricflowJinjaFilter } from './deep-parse.js'; +import { parseMetricflowFiles, translateMetricflowJinjaFilter } from '../../../../../src/context/ingest/adapters/metricflow/deep-parse.js'; function yaml(strings: TemplateStringsArray, ...values: unknown[]): string { return String.raw(strings, ...values); diff --git a/packages/cli/src/context/ingest/adapters/metricflow/detect.test.ts b/packages/cli/test/context/ingest/adapters/metricflow/detect.test.ts similarity index 94% rename from packages/cli/src/context/ingest/adapters/metricflow/detect.test.ts rename to packages/cli/test/context/ingest/adapters/metricflow/detect.test.ts index a8df434f..77923eb7 100644 --- a/packages/cli/src/context/ingest/adapters/metricflow/detect.test.ts +++ b/packages/cli/test/context/ingest/adapters/metricflow/detect.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { detectMetricFlowStagedDir } from './detect.js'; +import { detectMetricFlowStagedDir } from '../../../../../src/context/ingest/adapters/metricflow/detect.js'; async function touch(stagedDir: string, relPath: string, body = ''): Promise { const abs = join(stagedDir, relPath); diff --git a/packages/cli/src/context/ingest/adapters/metricflow/fetch.test.ts b/packages/cli/test/context/ingest/adapters/metricflow/fetch.test.ts similarity index 97% rename from packages/cli/src/context/ingest/adapters/metricflow/fetch.test.ts rename to packages/cli/test/context/ingest/adapters/metricflow/fetch.test.ts index 70568be2..0fb67072 100644 --- a/packages/cli/src/context/ingest/adapters/metricflow/fetch.test.ts +++ b/packages/cli/test/context/ingest/adapters/metricflow/fetch.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { makeLocalGitRepo } from '../../../test/make-local-git-repo.js'; -import { fetchMetricflowRepo } from './fetch.js'; +import { fetchMetricflowRepo } from '../../../../../src/context/ingest/adapters/metricflow/fetch.js'; async function exists(path: string): Promise { try { diff --git a/packages/cli/src/context/ingest/adapters/metricflow/graph.test.ts b/packages/cli/test/context/ingest/adapters/metricflow/graph.test.ts similarity index 97% rename from packages/cli/src/context/ingest/adapters/metricflow/graph.test.ts rename to packages/cli/test/context/ingest/adapters/metricflow/graph.test.ts index 93a3a6c6..1b252bc3 100644 --- a/packages/cli/src/context/ingest/adapters/metricflow/graph.test.ts +++ b/packages/cli/test/context/ingest/adapters/metricflow/graph.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { buildMetricFlowGraph } from './graph.js'; -import type { ParsedMetricFlowProject } from './parse.js'; +import { buildMetricFlowGraph } from '../../../../../src/context/ingest/adapters/metricflow/graph.js'; +import type { ParsedMetricFlowProject } from '../../../../../src/context/ingest/adapters/metricflow/parse.js'; function project(parts: Partial): ParsedMetricFlowProject { return { diff --git a/packages/cli/src/context/ingest/adapters/metricflow/import-semantic-models.test.ts b/packages/cli/test/context/ingest/adapters/metricflow/import-semantic-models.test.ts similarity index 98% rename from packages/cli/src/context/ingest/adapters/metricflow/import-semantic-models.test.ts rename to packages/cli/test/context/ingest/adapters/metricflow/import-semantic-models.test.ts index d5a7e3c5..a247af55 100644 --- a/packages/cli/src/context/ingest/adapters/metricflow/import-semantic-models.test.ts +++ b/packages/cli/test/context/ingest/adapters/metricflow/import-semantic-models.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import type { MetricFlowParseResult } from './deep-parse.js'; -import { importMetricflowSemanticModels } from './import-semantic-models.js'; +import type { MetricFlowParseResult } from '../../../../../src/context/ingest/adapters/metricflow/deep-parse.js'; +import { importMetricflowSemanticModels } from '../../../../../src/context/ingest/adapters/metricflow/import-semantic-models.js'; const DBT_SYSTEM_EMAIL = ['system@kae', 'lio.dev'].join(''); diff --git a/packages/cli/src/context/ingest/adapters/metricflow/metricflow.adapter.test.ts b/packages/cli/test/context/ingest/adapters/metricflow/metricflow.adapter.test.ts similarity index 95% rename from packages/cli/src/context/ingest/adapters/metricflow/metricflow.adapter.test.ts rename to packages/cli/test/context/ingest/adapters/metricflow/metricflow.adapter.test.ts index 232624a5..099a666f 100644 --- a/packages/cli/src/context/ingest/adapters/metricflow/metricflow.adapter.test.ts +++ b/packages/cli/test/context/ingest/adapters/metricflow/metricflow.adapter.test.ts @@ -3,10 +3,10 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { makeLocalGitRepo } from '../../../test/make-local-git-repo.js'; -import type { SourceAdapter } from '../../types.js'; -import type { MetricFlowParseResult } from './deep-parse.js'; -import { MetricflowSourceAdapter } from './metricflow.adapter.js'; -import { readMetricflowProjectionConfig, writeMetricflowProjectionConfig } from './projection-config.js'; +import type { SourceAdapter } from '../../../../../src/context/ingest/types.js'; +import type { MetricFlowParseResult } from '../../../../../src/context/ingest/adapters/metricflow/deep-parse.js'; +import { MetricflowSourceAdapter } from '../../../../../src/context/ingest/adapters/metricflow/metricflow.adapter.js'; +import { readMetricflowProjectionConfig, writeMetricflowProjectionConfig } from '../../../../../src/context/ingest/adapters/metricflow/projection-config.js'; function compileOnlyRequiredDepsCheck(): void { // @ts-expect-error MetricflowSourceAdapter requires an explicit cache home. diff --git a/packages/cli/src/context/ingest/adapters/metricflow/parse.test.ts b/packages/cli/test/context/ingest/adapters/metricflow/parse.test.ts similarity index 98% rename from packages/cli/src/context/ingest/adapters/metricflow/parse.test.ts rename to packages/cli/test/context/ingest/adapters/metricflow/parse.test.ts index 72a94472..9c2e071f 100644 --- a/packages/cli/src/context/ingest/adapters/metricflow/parse.test.ts +++ b/packages/cli/test/context/ingest/adapters/metricflow/parse.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { parseMetricFlowStagedDir } from './parse.js'; +import { parseMetricFlowStagedDir } from '../../../../../src/context/ingest/adapters/metricflow/parse.js'; async function writeFixture(stagedDir: string, relPath: string, body: string): Promise { const abs = join(stagedDir, relPath); diff --git a/packages/cli/src/context/ingest/adapters/metricflow/pull-config.test.ts b/packages/cli/test/context/ingest/adapters/metricflow/pull-config.test.ts similarity index 95% rename from packages/cli/src/context/ingest/adapters/metricflow/pull-config.test.ts rename to packages/cli/test/context/ingest/adapters/metricflow/pull-config.test.ts index 5137a4e6..a12cdccd 100644 --- a/packages/cli/src/context/ingest/adapters/metricflow/pull-config.test.ts +++ b/packages/cli/test/context/ingest/adapters/metricflow/pull-config.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { parseMetricflowPullConfig, pullConfigFromMetricflowIntegration } from './pull-config.js'; +import { parseMetricflowPullConfig, pullConfigFromMetricflowIntegration } from '../../../../../src/context/ingest/adapters/metricflow/pull-config.js'; describe('metricflow pull config', () => { it('applies defaults for optional git fields', () => { diff --git a/packages/cli/src/context/ingest/adapters/metricflow/semantic-models.test.ts b/packages/cli/test/context/ingest/adapters/metricflow/semantic-models.test.ts similarity index 96% rename from packages/cli/src/context/ingest/adapters/metricflow/semantic-models.test.ts rename to packages/cli/test/context/ingest/adapters/metricflow/semantic-models.test.ts index c22ac97d..0796ff0e 100644 --- a/packages/cli/src/context/ingest/adapters/metricflow/semantic-models.test.ts +++ b/packages/cli/test/context/ingest/adapters/metricflow/semantic-models.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; -import { composeOverlay } from '../../../../context/sl/semantic-layer.service.js'; -import type { SemanticLayerSource } from '../../../../context/sl/types.js'; -import type { ParsedCrossModelMetric, ParsedMetricflowRelationship, ParsedSemanticModel } from './deep-parse.js'; +import { composeOverlay } from '../../../../../src/context/sl/semantic-layer.service.js'; +import type { SemanticLayerSource } from '../../../../../src/context/sl/types.js'; +import type { ParsedCrossModelMetric, ParsedMetricflowRelationship, ParsedSemanticModel } from '../../../../../src/context/ingest/adapters/metricflow/deep-parse.js'; import { buildMetricflowColumns, buildMetricflowJoinsForModel, @@ -13,7 +13,7 @@ import { resolveMetricflowSemanticModelSourceName, rewriteMetricflowManifestJoins, toKebabCaseMetricflowName, -} from './semantic-models.js'; +} from '../../../../../src/context/ingest/adapters/metricflow/semantic-models.js'; const ordersModel: ParsedSemanticModel = { name: 'orders', diff --git a/packages/cli/src/context/ingest/adapters/notion/cluster.test.ts b/packages/cli/test/context/ingest/adapters/notion/cluster.test.ts similarity index 94% rename from packages/cli/src/context/ingest/adapters/notion/cluster.test.ts rename to packages/cli/test/context/ingest/adapters/notion/cluster.test.ts index ad41b571..ca2dfc0e 100644 --- a/packages/cli/src/context/ingest/adapters/notion/cluster.test.ts +++ b/packages/cli/test/context/ingest/adapters/notion/cluster.test.ts @@ -2,9 +2,9 @@ import { mkdir, mkdtemp, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, test } from 'vitest'; -import type { KtxEmbeddingPort } from '../../../core/embedding.js'; -import type { WorkUnit } from '../../types.js'; -import { clusterNotionWorkUnits, MIN_PAGES_TO_CLUSTER } from './cluster.js'; +import type { KtxEmbeddingPort } from '../../../../../src/context/core/embedding.js'; +import type { WorkUnit } from '../../../../../src/context/ingest/types.js'; +import { clusterNotionWorkUnits, MIN_PAGES_TO_CLUSTER } from '../../../../../src/context/ingest/adapters/notion/cluster.js'; function fakeEmbedding(text: string): number[] { const v = [0, 0, 0, 0]; diff --git a/packages/cli/src/context/ingest/adapters/notion/fetch.test.ts b/packages/cli/test/context/ingest/adapters/notion/fetch.test.ts similarity index 98% rename from packages/cli/src/context/ingest/adapters/notion/fetch.test.ts rename to packages/cli/test/context/ingest/adapters/notion/fetch.test.ts index b60170f7..10140074 100644 --- a/packages/cli/src/context/ingest/adapters/notion/fetch.test.ts +++ b/packages/cli/test/context/ingest/adapters/notion/fetch.test.ts @@ -2,8 +2,8 @@ import { mkdtemp, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { fetchNotionSnapshot } from './fetch.js'; -import type { NotionApi } from './notion-client.js'; +import { fetchNotionSnapshot } from '../../../../../src/context/ingest/adapters/notion/fetch.js'; +import type { NotionApi } from '../../../../../src/context/ingest/adapters/notion/notion-client.js'; describe('fetchNotionSnapshot', () => { let stagedDir: string; diff --git a/packages/cli/src/context/ingest/adapters/notion/local-state-store.test.ts b/packages/cli/test/context/ingest/adapters/notion/local-state-store.test.ts similarity index 91% rename from packages/cli/src/context/ingest/adapters/notion/local-state-store.test.ts rename to packages/cli/test/context/ingest/adapters/notion/local-state-store.test.ts index 892ea6c1..da5d51c2 100644 --- a/packages/cli/src/context/ingest/adapters/notion/local-state-store.test.ts +++ b/packages/cli/test/context/ingest/adapters/notion/local-state-store.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { LocalNotionRuntimeStore } from './local-state-store.js'; +import { LocalNotionRuntimeStore } from '../../../../../src/context/ingest/adapters/notion/local-state-store.js'; describe('LocalNotionRuntimeStore', () => { let tempDir: string; diff --git a/packages/cli/src/context/ingest/adapters/notion/normalize.test.ts b/packages/cli/test/context/ingest/adapters/notion/normalize.test.ts similarity index 96% rename from packages/cli/src/context/ingest/adapters/notion/normalize.test.ts rename to packages/cli/test/context/ingest/adapters/notion/normalize.test.ts index 3b90c4de..dcc4621e 100644 --- a/packages/cli/src/context/ingest/adapters/notion/normalize.test.ts +++ b/packages/cli/test/context/ingest/adapters/notion/normalize.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { normalizeNotionBlocksToMarkdown, normalizeNotionPageMetadata, propertyValueToText } from './normalize.js'; +import { normalizeNotionBlocksToMarkdown, normalizeNotionPageMetadata, propertyValueToText } from '../../../../../src/context/ingest/adapters/notion/normalize.js'; describe('Notion normalization', () => { it('converts common blocks into stable markdown', () => { diff --git a/packages/cli/src/context/ingest/adapters/notion/notion-client.test.ts b/packages/cli/test/context/ingest/adapters/notion/notion-client.test.ts similarity index 95% rename from packages/cli/src/context/ingest/adapters/notion/notion-client.test.ts rename to packages/cli/test/context/ingest/adapters/notion/notion-client.test.ts index fd3d54eb..bb3bdb0b 100644 --- a/packages/cli/src/context/ingest/adapters/notion/notion-client.test.ts +++ b/packages/cli/test/context/ingest/adapters/notion/notion-client.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { retryNotionRequest } from './notion-client.js'; +import { retryNotionRequest } from '../../../../../src/context/ingest/adapters/notion/notion-client.js'; describe('Notion client retry helper', () => { it('retries rate-limited requests and then returns the response', async () => { diff --git a/packages/cli/src/context/ingest/adapters/notion/notion.adapter.test.ts b/packages/cli/test/context/ingest/adapters/notion/notion.adapter.test.ts similarity index 97% rename from packages/cli/src/context/ingest/adapters/notion/notion.adapter.test.ts rename to packages/cli/test/context/ingest/adapters/notion/notion.adapter.test.ts index 0f500d5e..e02cf10b 100644 --- a/packages/cli/src/context/ingest/adapters/notion/notion.adapter.test.ts +++ b/packages/cli/test/context/ingest/adapters/notion/notion.adapter.test.ts @@ -2,9 +2,9 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { DiffSetService } from '../../diff-set.service.js'; -import { NOTION_ORG_KNOWLEDGE_WARNING } from './chunk.js'; -import { NotionSourceAdapter } from './notion.adapter.js'; +import { DiffSetService } from '../../../../../src/context/ingest/diff-set.service.js'; +import { NOTION_ORG_KNOWLEDGE_WARNING } from '../../../../../src/context/ingest/adapters/notion/chunk.js'; +import { NotionSourceAdapter } from '../../../../../src/context/ingest/adapters/notion/notion.adapter.js'; describe('NotionSourceAdapter', () => { let stagedDir: string; diff --git a/packages/cli/src/context/ingest/artifact-gates.test.ts b/packages/cli/test/context/ingest/artifact-gates.test.ts similarity index 99% rename from packages/cli/src/context/ingest/artifact-gates.test.ts rename to packages/cli/test/context/ingest/artifact-gates.test.ts index cc786409..c93a24e5 100644 --- a/packages/cli/src/context/ingest/artifact-gates.test.ts +++ b/packages/cli/test/context/ingest/artifact-gates.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { validateFinalIngestArtifacts, validateProvenanceRawPaths } from './artifact-gates.js'; +import { validateFinalIngestArtifacts, validateProvenanceRawPaths } from '../../../src/context/ingest/artifact-gates.js'; function wikiServiceWithPages( pages: Record, diff --git a/packages/cli/src/context/ingest/canonical-pins.test.ts b/packages/cli/test/context/ingest/canonical-pins.test.ts similarity index 92% rename from packages/cli/src/context/ingest/canonical-pins.test.ts rename to packages/cli/test/context/ingest/canonical-pins.test.ts index dec62360..ae645b24 100644 --- a/packages/cli/src/context/ingest/canonical-pins.test.ts +++ b/packages/cli/test/context/ingest/canonical-pins.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { buildCanonicalPinsPromptBlock, type CanonicalPin, selectRelevantCanonicalPins } from './canonical-pins.js'; -import type { StageIndex } from './stages/stage-index.types.js'; +import { buildCanonicalPinsPromptBlock, type CanonicalPin, selectRelevantCanonicalPins } from '../../../src/context/ingest/canonical-pins.js'; +import type { StageIndex } from '../../../src/context/ingest/stages/stage-index.types.js'; function makeStageIndex(): StageIndex { return { diff --git a/packages/cli/src/context/ingest/clustering/kmeans.test.ts b/packages/cli/test/context/ingest/clustering/kmeans.test.ts similarity index 95% rename from packages/cli/src/context/ingest/clustering/kmeans.test.ts rename to packages/cli/test/context/ingest/clustering/kmeans.test.ts index 3cda76d1..a0346309 100644 --- a/packages/cli/src/context/ingest/clustering/kmeans.test.ts +++ b/packages/cli/test/context/ingest/clustering/kmeans.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { kmeans, pickK } from './kmeans.js'; +import { kmeans, pickK } from '../../../../src/context/ingest/clustering/kmeans.js'; describe('pickK', () => { test('uses ceil(N/8) heuristic clamped to [1, 10]', () => { diff --git a/packages/cli/src/context/ingest/context-candidates/candidate-dedup.service.test.ts b/packages/cli/test/context/ingest/context-candidates/candidate-dedup.service.test.ts similarity index 96% rename from packages/cli/src/context/ingest/context-candidates/candidate-dedup.service.test.ts rename to packages/cli/test/context/ingest/context-candidates/candidate-dedup.service.test.ts index a7b5520e..f69ab4db 100644 --- a/packages/cli/src/context/ingest/context-candidates/candidate-dedup.service.test.ts +++ b/packages/cli/test/context/ingest/context-candidates/candidate-dedup.service.test.ts @@ -1,8 +1,8 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import type { ContextCandidateForDedup } from '../ports.js'; -import { CandidateDedupService } from './candidate-dedup.service.js'; -import type { ContextCandidateStorePort } from './store.js'; -import type { ContextCandidateEmbeddingPort } from './types.js'; +import type { ContextCandidateForDedup } from '../../../../src/context/ingest/ports.js'; +import { CandidateDedupService } from '../../../../src/context/ingest/context-candidates/candidate-dedup.service.js'; +import type { ContextCandidateStorePort } from '../../../../src/context/ingest/context-candidates/store.js'; +import type { ContextCandidateEmbeddingPort } from '../../../../src/context/ingest/context-candidates/types.js'; const vector = (...values: number[]): string => JSON.stringify(values); diff --git a/packages/cli/src/context/ingest/context-candidates/context-candidate-carryforward.service.test.ts b/packages/cli/test/context/ingest/context-candidates/context-candidate-carryforward.service.test.ts similarity index 94% rename from packages/cli/src/context/ingest/context-candidates/context-candidate-carryforward.service.test.ts rename to packages/cli/test/context/ingest/context-candidates/context-candidate-carryforward.service.test.ts index df452ca7..229b4a34 100644 --- a/packages/cli/src/context/ingest/context-candidates/context-candidate-carryforward.service.test.ts +++ b/packages/cli/test/context/ingest/context-candidates/context-candidate-carryforward.service.test.ts @@ -1,8 +1,8 @@ import { createHash } from 'node:crypto'; import { describe, expect, it, vi } from 'vitest'; -import { ContextCandidateCarryforwardService } from './context-candidate-carryforward.service.js'; -import type { ContextCandidateStorePort } from './store.js'; -import type { BudgetExhaustedCandidateForCarryForward, CurrentRunEvidenceChunkForCarryForward } from './types.js'; +import { ContextCandidateCarryforwardService } from '../../../../src/context/ingest/context-candidates/context-candidate-carryforward.service.js'; +import type { ContextCandidateStorePort } from '../../../../src/context/ingest/context-candidates/store.js'; +import type { BudgetExhaustedCandidateForCarryForward, CurrentRunEvidenceChunkForCarryForward } from '../../../../src/context/ingest/context-candidates/types.js'; function candidate( overrides: Partial = {}, diff --git a/packages/cli/src/context/ingest/context-candidates/curator-pagination.service.test.ts b/packages/cli/test/context/ingest/context-candidates/curator-pagination.service.test.ts similarity index 96% rename from packages/cli/src/context/ingest/context-candidates/curator-pagination.service.test.ts rename to packages/cli/test/context/ingest/context-candidates/curator-pagination.service.test.ts index bf1876a3..7341f2e9 100644 --- a/packages/cli/src/context/ingest/context-candidates/curator-pagination.service.test.ts +++ b/packages/cli/test/context/ingest/context-candidates/curator-pagination.service.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ContextCandidateForDedup } from '../ports.js'; -import { type CuratorPaginationInput, CuratorPaginationService } from './curator-pagination.service.js'; -import type { ContextCandidateStorePort } from './store.js'; +import type { ContextCandidateForDedup } from '../../../../src/context/ingest/ports.js'; +import { type CuratorPaginationInput, CuratorPaginationService } from '../../../../src/context/ingest/context-candidates/curator-pagination.service.js'; +import type { ContextCandidateStorePort } from '../../../../src/context/ingest/context-candidates/store.js'; const candidate = (key: string, score: number): ContextCandidateForDedup => ({ id: `id-${key}`, diff --git a/packages/cli/src/context/ingest/context-candidates/embedding-text.test.ts b/packages/cli/test/context/ingest/context-candidates/embedding-text.test.ts similarity index 78% rename from packages/cli/src/context/ingest/context-candidates/embedding-text.test.ts rename to packages/cli/test/context/ingest/context-candidates/embedding-text.test.ts index e3e2e728..65857780 100644 --- a/packages/cli/src/context/ingest/context-candidates/embedding-text.test.ts +++ b/packages/cli/test/context/ingest/context-candidates/embedding-text.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { buildContextCandidateEmbeddingText } from './embedding-text.js'; +import { buildContextCandidateEmbeddingText } from '../../../../src/context/ingest/context-candidates/embedding-text.js'; describe('buildContextCandidateEmbeddingText', () => { it('matches the existing dedup embedding input format', () => { diff --git a/packages/cli/src/context/ingest/context-candidates/store.test.ts b/packages/cli/test/context/ingest/context-candidates/store.test.ts similarity index 89% rename from packages/cli/src/context/ingest/context-candidates/store.test.ts rename to packages/cli/test/context/ingest/context-candidates/store.test.ts index 1c2311ad..3dd3cbaf 100644 --- a/packages/cli/src/context/ingest/context-candidates/store.test.ts +++ b/packages/cli/test/context/ingest/context-candidates/store.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ContextCandidateForDedup } from '../ports.js'; -import type { ContextCandidateStorePort } from './store.js'; -import type { InsertContextCandidateInput } from './types.js'; +import type { ContextCandidateForDedup } from '../../../../src/context/ingest/ports.js'; +import type { ContextCandidateStorePort } from '../../../../src/context/ingest/context-candidates/store.js'; +import type { InsertContextCandidateInput } from '../../../../src/context/ingest/context-candidates/types.js'; const candidate: ContextCandidateForDedup = { id: 'candidate-1', diff --git a/packages/cli/src/context/ingest/context-evidence/context-evidence-index.service.test.ts b/packages/cli/test/context/ingest/context-evidence/context-evidence-index.service.test.ts similarity index 97% rename from packages/cli/src/context/ingest/context-evidence/context-evidence-index.service.test.ts rename to packages/cli/test/context/ingest/context-evidence/context-evidence-index.service.test.ts index d62c7b53..80caeec0 100644 --- a/packages/cli/src/context/ingest/context-evidence/context-evidence-index.service.test.ts +++ b/packages/cli/test/context/ingest/context-evidence/context-evidence-index.service.test.ts @@ -2,9 +2,9 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { ContextEvidenceIndexService } from './context-evidence-index.service.js'; -import type { ContextEvidenceIndexStorePort } from './store.js'; -import type { ContextEvidenceEmbeddingPort } from './types.js'; +import { ContextEvidenceIndexService } from '../../../../src/context/ingest/context-evidence/context-evidence-index.service.js'; +import type { ContextEvidenceIndexStorePort } from '../../../../src/context/ingest/context-evidence/store.js'; +import type { ContextEvidenceEmbeddingPort } from '../../../../src/context/ingest/context-evidence/types.js'; const vector384 = (first: number): number[] => [first, ...Array.from({ length: 383 }, () => 0)]; diff --git a/packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.test.ts b/packages/cli/test/context/ingest/context-evidence/sqlite-context-evidence-store.test.ts similarity index 98% rename from packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.test.ts rename to packages/cli/test/context/ingest/context-evidence/sqlite-context-evidence-store.test.ts index 6b575c00..6041c412 100644 --- a/packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.test.ts +++ b/packages/cli/test/context/ingest/context-evidence/sqlite-context-evidence-store.test.ts @@ -2,9 +2,9 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import type { InsertContextCandidateInput } from '../../../context/ingest/context-candidates/types.js'; -import type { JsonValue } from '../ports.js'; -import { SqliteContextEvidenceStore } from './sqlite-context-evidence-store.js'; +import type { InsertContextCandidateInput } from '../../../../src/context/ingest/context-candidates/types.js'; +import type { JsonValue } from '../../../../src/context/ingest/ports.js'; +import { SqliteContextEvidenceStore } from '../../../../src/context/ingest/context-evidence/sqlite-context-evidence-store.js'; describe('SqliteContextEvidenceStore', () => { let tempDir: string; diff --git a/packages/cli/src/context/ingest/context-evidence/store.test.ts b/packages/cli/test/context/ingest/context-evidence/store.test.ts similarity index 92% rename from packages/cli/src/context/ingest/context-evidence/store.test.ts rename to packages/cli/test/context/ingest/context-evidence/store.test.ts index 9c2d281a..268aefec 100644 --- a/packages/cli/src/context/ingest/context-evidence/store.test.ts +++ b/packages/cli/test/context/ingest/context-evidence/store.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ContextEvidenceIndexStorePort } from './store.js'; -import type { ReplaceContextEvidenceChunk, UpsertContextEvidenceDocument } from './types.js'; +import type { ContextEvidenceIndexStorePort } from '../../../../src/context/ingest/context-evidence/store.js'; +import type { ReplaceContextEvidenceChunk, UpsertContextEvidenceDocument } from '../../../../src/context/ingest/context-evidence/types.js'; const documentInput: UpsertContextEvidenceDocument = { runId: 'run-1', diff --git a/packages/cli/src/context/ingest/dbt-shared/project-vars.test.ts b/packages/cli/test/context/ingest/dbt-shared/project-vars.test.ts similarity index 98% rename from packages/cli/src/context/ingest/dbt-shared/project-vars.test.ts rename to packages/cli/test/context/ingest/dbt-shared/project-vars.test.ts index 3b9ed6b3..c996e7aa 100644 --- a/packages/cli/src/context/ingest/dbt-shared/project-vars.test.ts +++ b/packages/cli/test/context/ingest/dbt-shared/project-vars.test.ts @@ -7,7 +7,7 @@ import { parseProjectName, parseProjectVars, resolveJinjaVariables, -} from './project-vars.js'; +} from '../../../../src/context/ingest/dbt-shared/project-vars.js'; function entries(map: Map): Record { return Object.fromEntries([...map.entries()].sort(([a], [b]) => a.localeCompare(b))); diff --git a/packages/cli/src/context/ingest/dbt-shared/schema-files.test.ts b/packages/cli/test/context/ingest/dbt-shared/schema-files.test.ts similarity index 93% rename from packages/cli/src/context/ingest/dbt-shared/schema-files.test.ts rename to packages/cli/test/context/ingest/dbt-shared/schema-files.test.ts index f55851f6..fd39c776 100644 --- a/packages/cli/src/context/ingest/dbt-shared/schema-files.test.ts +++ b/packages/cli/test/context/ingest/dbt-shared/schema-files.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { findDbtSchemaFiles, loadDbtSchemaFiles } from './schema-files.js'; +import { findDbtSchemaFiles, loadDbtSchemaFiles } from '../../../../src/context/ingest/dbt-shared/schema-files.js'; describe('dbt shared schema files', () => { let tmpRoot: string; diff --git a/packages/cli/src/context/ingest/diff-set.service.test.ts b/packages/cli/test/context/ingest/diff-set.service.test.ts similarity index 97% rename from packages/cli/src/context/ingest/diff-set.service.test.ts rename to packages/cli/test/context/ingest/diff-set.service.test.ts index 4eb3ceaa..9f198c54 100644 --- a/packages/cli/src/context/ingest/diff-set.service.test.ts +++ b/packages/cli/test/context/ingest/diff-set.service.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { computeDiffSetFromHashes, DiffSetService } from './diff-set.service.js'; +import { computeDiffSetFromHashes, DiffSetService } from '../../../src/context/ingest/diff-set.service.js'; function makeRepo(latest: Map) { return { diff --git a/packages/cli/src/context/ingest/final-gate-repair.test.ts b/packages/cli/test/context/ingest/final-gate-repair.test.ts similarity index 96% rename from packages/cli/src/context/ingest/final-gate-repair.test.ts rename to packages/cli/test/context/ingest/final-gate-repair.test.ts index 90ad707d..1a52442c 100644 --- a/packages/cli/src/context/ingest/final-gate-repair.test.ts +++ b/packages/cli/test/context/ingest/final-gate-repair.test.ts @@ -2,8 +2,8 @@ import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it, vi } from 'vitest'; -import { finalGateRepairPaths, repairFinalGateFailure } from './final-gate-repair.js'; -import { FileIngestTraceWriter } from './ingest-trace.js'; +import { finalGateRepairPaths, repairFinalGateFailure } from '../../../src/context/ingest/final-gate-repair.js'; +import { FileIngestTraceWriter } from '../../../src/context/ingest/ingest-trace.js'; async function makeHarness() { const root = await mkdtemp(join(tmpdir(), 'ktx-final-gate-repair-')); diff --git a/packages/cli/src/context/ingest/finalization-scope.test.ts b/packages/cli/test/context/ingest/finalization-scope.test.ts similarity index 98% rename from packages/cli/src/context/ingest/finalization-scope.test.ts rename to packages/cli/test/context/ingest/finalization-scope.test.ts index 28d0b863..02c535df 100644 --- a/packages/cli/src/context/ingest/finalization-scope.test.ts +++ b/packages/cli/test/context/ingest/finalization-scope.test.ts @@ -3,7 +3,7 @@ import { compareFinalizationDeclarations, deriveFinalizationTouchedSources, deriveFinalizationWikiPageKeys, -} from './finalization-scope.js'; +} from '../../../src/context/ingest/finalization-scope.js'; describe('deriveFinalizationWikiPageKeys', () => { it('maps changed global wiki markdown paths to page keys', () => { diff --git a/packages/cli/test/context/ingest/historic-sql-probes.test.ts b/packages/cli/test/context/ingest/historic-sql-probes.test.ts new file mode 100644 index 00000000..542c4fde --- /dev/null +++ b/packages/cli/test/context/ingest/historic-sql-probes.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { HistoricSqlDialect } from '../../../src/context/ingest/adapters/historic-sql/types.js'; +import { + historicSqlProbeCatalogName, + runHistoricSqlReadinessProbe, + type HistoricSqlProbeRunner, + type HistoricSqlProbeRunnerFactoryEntry, +} from '../../../src/context/ingest/historic-sql-probes.js'; + +function fakeRunner( + dialect: HistoricSqlDialect, + catalogName: string, + options: { result?: unknown; error?: unknown } = {}, +): HistoricSqlProbeRunner & { runCalls: () => number } { + let calls = 0; + return { + dialect, + catalogName, + async run() { + calls += 1; + if (options.error) { + throw options.error; + } + return options.result ?? { warnings: [], info: [] }; + }, + formatSuccessDetail() { + return { detail: `${catalogName} ready`, warnings: [] }; + }, + fixAdvice(error) { + return { + failHeadline: error instanceof Error ? error.message : String(error), + remediation: 'Fix the test probe.', + }; + }, + runCalls: () => calls, + }; +} + +function factories( + overrides: Partial>, +): Record { + const postgres = overrides.postgres ?? fakeRunner('postgres', 'pg_stat_statements'); + const snowflake = + overrides.snowflake ?? + fakeRunner('snowflake', 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY'); + const bigquery = + overrides.bigquery ?? fakeRunner('bigquery', 'INFORMATION_SCHEMA.JOBS_BY_PROJECT'); + + return { + postgres: { + catalogName: 'pg_stat_statements', + load: vi.fn(async () => postgres), + }, + snowflake: { + catalogName: 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY', + load: vi.fn(async () => snowflake), + }, + bigquery: { + catalogName: 'INFORMATION_SCHEMA.JOBS_BY_PROJECT', + load: vi.fn(async () => bigquery), + }, + }; +} + +describe('historic-SQL probe registry', () => { + it('returns null when the connection has no query-history dialect', async () => { + const deps = { factories: factories({}), cache: new Map() }; + + await expect( + runHistoricSqlReadinessProbe( + { + projectDir: '/work/project', + connectionId: 'mysql', + connection: { + driver: 'mysql', + context: { queryHistory: { enabled: true } }, + }, + env: {}, + }, + deps, + ), + ).resolves.toBeNull(); + + expect(deps.factories.postgres.load).not.toHaveBeenCalled(); + expect(deps.factories.snowflake.load).not.toHaveBeenCalled(); + expect(deps.factories.bigquery.load).not.toHaveBeenCalled(); + }); + + it('dispatches to the dialect runner and caches the runner instance', async () => { + const runner = fakeRunner('postgres', 'pg_stat_statements', { + result: { pgServerVersion: 'PostgreSQL 16.4', warnings: [], info: [] }, + }); + const deps = { factories: factories({ postgres: runner }), cache: new Map() }; + const input = { + projectDir: '/work/project', + connectionId: 'warehouse', + connection: { + driver: 'postgres' as const, + url: 'env:DATABASE_URL', + context: { queryHistory: { enabled: true } }, + }, + env: {}, + }; + + const first = await runHistoricSqlReadinessProbe(input, deps); + const second = await runHistoricSqlReadinessProbe(input, deps); + + expect(first).toMatchObject({ ok: true, dialect: 'postgres', runner }); + expect(second).toMatchObject({ ok: true, dialect: 'postgres', runner }); + expect(deps.factories.postgres.load).toHaveBeenCalledTimes(1); + expect(runner.runCalls()).toBe(2); + }); + + it('normalizes runner errors into a failed outcome', async () => { + const error = new Error('missing grants'); + const runner = fakeRunner('bigquery', 'INFORMATION_SCHEMA.JOBS_BY_PROJECT', { + error, + }); + const deps = { factories: factories({ bigquery: runner }), cache: new Map() }; + + await expect( + runHistoricSqlReadinessProbe( + { + projectDir: '/work/project', + connectionId: 'bq', + connection: { + driver: 'bigquery', + credentials_json: '{"project_id":"project-1"}', + context: { queryHistory: { enabled: true } }, + }, + env: {}, + }, + deps, + ), + ).resolves.toEqual({ + ok: false, + dialect: 'bigquery', + runner, + error, + }); + }); + + it('returns catalog names without loading runner modules', () => { + const deps = { factories: factories({}), cache: new Map() }; + + expect(historicSqlProbeCatalogName('postgres', deps)).toBe('pg_stat_statements'); + expect(historicSqlProbeCatalogName('snowflake', deps)).toBe( + 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY', + ); + expect(historicSqlProbeCatalogName('bigquery', deps)).toBe( + 'INFORMATION_SCHEMA.JOBS_BY_PROJECT', + ); + expect(deps.factories.postgres.load).not.toHaveBeenCalled(); + expect(deps.factories.snowflake.load).not.toHaveBeenCalled(); + expect(deps.factories.bigquery.load).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli/test/context/ingest/historic-sql-probes/bigquery-runner.test.ts b/packages/cli/test/context/ingest/historic-sql-probes/bigquery-runner.test.ts new file mode 100644 index 00000000..d51aaf42 --- /dev/null +++ b/packages/cli/test/context/ingest/historic-sql-probes/bigquery-runner.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it, vi } from 'vitest'; +import { HistoricSqlGrantsMissingError } from '../../../../src/context/ingest/adapters/historic-sql/errors.js'; +import { BigQueryJobsByProjectProbeRunner } from '../../../../src/context/ingest/historic-sql-probes/bigquery-runner.js'; + +describe('BigQueryJobsByProjectProbeRunner', () => { + it('creates a region-scoped reader, runs it, and cleans up the connector', async () => { + const cleanup = vi.fn(async () => undefined); + const reader = { + probe: vi.fn(async () => ({ warnings: [], info: ['region: eu'] })), + }; + const createReader = vi.fn(() => reader); + const runner = new BigQueryJobsByProjectProbeRunner({ + createReader, + createClient: () => ({ client: { executeQuery: vi.fn() }, cleanup }), + resolveReference: () => '{"project_id":"project-1"}', + }); + + await expect( + runner.run({ + projectDir: '/work/project', + connectionId: 'bq', + connection: { + driver: 'bigquery', + credentials_json: 'env:BQ_CREDENTIALS_JSON', + location: 'EU', + }, + env: {}, + }), + ).resolves.toEqual({ warnings: [], info: ['region: eu'] }); + expect(createReader).toHaveBeenCalledWith({ projectId: 'project-1', region: 'EU' }); + expect(reader.probe).toHaveBeenCalledOnce(); + expect(cleanup).toHaveBeenCalledOnce(); + }); + + it('uses us as the default BigQuery region', async () => { + const createReader = vi.fn(() => ({ + probe: vi.fn(async () => ({ warnings: [], info: [] })), + })); + const runner = new BigQueryJobsByProjectProbeRunner({ + createReader, + createClient: () => ({ client: {}, cleanup: vi.fn(async () => undefined) }), + resolveReference: () => '{"project_id":"project-1"}', + }); + + await runner.run({ + projectDir: '/work/project', + connectionId: 'bq', + connection: { + driver: 'bigquery', + credentials_json: '{"project_id":"project-1"}', + }, + env: {}, + }); + + expect(createReader).toHaveBeenCalledWith({ projectId: 'project-1', region: 'us' }); + }); + + it('rejects missing BigQuery credentials_json.project_id', async () => { + const runner = new BigQueryJobsByProjectProbeRunner({ + createReader: vi.fn(), + createClient: () => ({ client: {}, cleanup: vi.fn() }), + resolveReference: () => '{"client_email":"svc@example.test"}', + }); + + await expect( + runner.run({ + projectDir: '/work/project', + connectionId: 'bq', + connection: { + driver: 'bigquery', + credentials_json: 'env:BQ_CREDENTIALS_JSON', + }, + env: {}, + }), + ).rejects.toThrow('Query history BigQuery connection bq requires credentials_json.project_id'); + }); + + it('formats successful BigQuery details', () => { + const runner = new BigQueryJobsByProjectProbeRunner(); + + expect( + runner.formatSuccessDetail({ + warnings: ['JOBS_BY_PROJECT is delayed'], + info: ['region: us'], + }), + ).toEqual({ + detail: 'INFORMATION_SCHEMA.JOBS_BY_PROJECT ready; region: us', + warnings: ['JOBS_BY_PROJECT is delayed'], + }); + }); + + it('maps BigQuery grant errors to runner advice', () => { + const runner = new BigQueryJobsByProjectProbeRunner(); + + expect( + runner.fixAdvice( + new HistoricSqlGrantsMissingError({ + dialect: 'bigquery', + message: 'principal cannot query JOBS_BY_PROJECT', + remediation: + 'Grant roles/bigquery.resourceViewer on the BigQuery project, or grant a custom role containing bigquery.jobs.listAll.', + }), + ), + ).toEqual({ + failHeadline: 'BigQuery principal cannot read INFORMATION_SCHEMA.JOBS_BY_PROJECT', + remediation: + 'Grant roles/bigquery.resourceViewer on the BigQuery project, or grant a custom role containing bigquery.jobs.listAll.', + }); + }); +}); diff --git a/packages/cli/test/context/ingest/historic-sql-probes/postgres-runner.test.ts b/packages/cli/test/context/ingest/historic-sql-probes/postgres-runner.test.ts new file mode 100644 index 00000000..c52443ee --- /dev/null +++ b/packages/cli/test/context/ingest/historic-sql-probes/postgres-runner.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + HistoricSqlExtensionMissingError, + HistoricSqlGrantsMissingError, + HistoricSqlVersionUnsupportedError, +} from '../../../../src/context/ingest/adapters/historic-sql/errors.js'; +import { PostgresPgssProbeRunner } from '../../../../src/context/ingest/historic-sql-probes/postgres-runner.js'; + +describe('PostgresPgssProbeRunner', () => { + it('runs the pg_stat_statements reader and cleans up the client', async () => { + const cleanup = vi.fn(async () => undefined); + const reader = { + probe: vi.fn(async () => ({ + pgServerVersion: 'PostgreSQL 16.4', + warnings: [], + info: ['tracked statements: 12'], + })), + }; + const runner = new PostgresPgssProbeRunner({ + reader, + createClient: () => ({ client: { executeQuery: vi.fn() }, cleanup }), + }); + + await expect( + runner.run({ + projectDir: '/work/project', + connectionId: 'warehouse', + connection: { driver: 'postgres', url: 'env:DATABASE_URL' }, + env: {}, + }), + ).resolves.toEqual({ + pgServerVersion: 'PostgreSQL 16.4', + warnings: [], + info: ['tracked statements: 12'], + }); + expect(reader.probe).toHaveBeenCalledOnce(); + expect(cleanup).toHaveBeenCalledOnce(); + }); + + it('rejects non-Postgres connections', async () => { + const runner = new PostgresPgssProbeRunner({ + reader: { probe: vi.fn() }, + createClient: () => ({ client: {}, cleanup: vi.fn() }), + }); + + await expect( + runner.run({ + projectDir: '/work/project', + connectionId: 'warehouse', + connection: { driver: 'snowflake' }, + env: {}, + }), + ).rejects.toThrow('Native PostgreSQL connector cannot run driver "snowflake"'); + }); + + it('formats successful Postgres details', () => { + const runner = new PostgresPgssProbeRunner(); + + expect( + runner.formatSuccessDetail({ + pgServerVersion: 'PostgreSQL 16.4', + warnings: ['pg_stat_statements.track is top'], + info: ['tracked statements: 12'], + }), + ).toEqual({ + detail: 'pg_stat_statements ready (PostgreSQL 16.4); tracked statements: 12', + warnings: ['pg_stat_statements.track is top'], + }); + }); + + it('maps Postgres probe errors to actionable advice', () => { + const runner = new PostgresPgssProbeRunner(); + + expect( + runner.fixAdvice( + new HistoricSqlExtensionMissingError({ + dialect: 'postgres', + message: 'pg_stat_statements missing', + remediation: 'CREATE EXTENSION pg_stat_statements;', + }), + ), + ).toEqual({ + failHeadline: 'pg_stat_statements extension is missing', + remediation: 'CREATE EXTENSION pg_stat_statements;', + }); + + expect( + runner.fixAdvice( + new HistoricSqlGrantsMissingError({ + dialect: 'postgres', + message: 'missing grants', + remediation: 'GRANT pg_read_all_stats TO ;', + }), + ), + ).toEqual({ + failHeadline: 'Postgres connection role lacks pg_read_all_stats', + remediation: 'GRANT pg_read_all_stats TO ;', + }); + + expect( + runner.fixAdvice( + new HistoricSqlVersionUnsupportedError({ + dialect: 'postgres', + detectedVersion: 'PostgreSQL 13.12', + minimumVersion: 'PostgreSQL 14', + }), + ), + ).toEqual({ + failHeadline: 'Postgres version too old', + remediation: 'Use PostgreSQL 14 or newer, or disable query history for this connection', + }); + }); +}); diff --git a/packages/cli/test/context/ingest/historic-sql-probes/snowflake-runner.test.ts b/packages/cli/test/context/ingest/historic-sql-probes/snowflake-runner.test.ts new file mode 100644 index 00000000..af3e73f3 --- /dev/null +++ b/packages/cli/test/context/ingest/historic-sql-probes/snowflake-runner.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it, vi } from 'vitest'; +import { HistoricSqlGrantsMissingError } from '../../../../src/context/ingest/adapters/historic-sql/errors.js'; +import { SnowflakeAccountUsageProbeRunner } from '../../../../src/context/ingest/historic-sql-probes/snowflake-runner.js'; + +describe('SnowflakeAccountUsageProbeRunner', () => { + it('runs the account usage reader and cleans up the client', async () => { + const cleanup = vi.fn(async () => undefined); + const reader = { + probe: vi.fn(async () => ({ warnings: [], info: ['query history available'] })), + }; + const runner = new SnowflakeAccountUsageProbeRunner({ + reader, + createClient: () => ({ client: { executeQuery: vi.fn() }, cleanup }), + }); + + await expect( + runner.run({ + projectDir: '/work/project', + connectionId: 'warehouse', + connection: { + driver: 'snowflake', + account: 'ACCT', + warehouse: 'WH', + database: 'ANALYTICS', + username: 'reader', + }, + env: {}, + }), + ).resolves.toEqual({ warnings: [], info: ['query history available'] }); + expect(reader.probe).toHaveBeenCalledOnce(); + expect(cleanup).toHaveBeenCalledOnce(); + }); + + it('rejects non-Snowflake connections', async () => { + const runner = new SnowflakeAccountUsageProbeRunner({ + reader: { probe: vi.fn() }, + createClient: () => ({ client: {}, cleanup: vi.fn() }), + }); + + await expect( + runner.run({ + projectDir: '/work/project', + connectionId: 'warehouse', + connection: { driver: 'postgres' }, + env: {}, + }), + ).rejects.toThrow('Native Snowflake connector cannot run driver "postgres"'); + }); + + it('formats successful Snowflake details', () => { + const runner = new SnowflakeAccountUsageProbeRunner(); + + expect( + runner.formatSuccessDetail({ + warnings: ['query history is delayed'], + info: ['warehouse: WH'], + }), + ).toEqual({ + detail: 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY ready; warehouse: WH', + warnings: ['query history is delayed'], + }); + }); + + it('maps Snowflake grant errors to runner advice', () => { + const runner = new SnowflakeAccountUsageProbeRunner(); + + expect( + runner.fixAdvice( + new HistoricSqlGrantsMissingError({ + dialect: 'snowflake', + message: 'role cannot read account usage', + remediation: + 'GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE ;', + }), + ), + ).toEqual({ + failHeadline: 'Snowflake role cannot read SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY', + remediation: + 'GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE ;', + }); + }); +}); diff --git a/packages/cli/src/context/ingest/ingest-bundle.runner.isolated-diff.test.ts b/packages/cli/test/context/ingest/ingest-bundle.runner.isolated-diff.test.ts similarity index 99% rename from packages/cli/src/context/ingest/ingest-bundle.runner.isolated-diff.test.ts rename to packages/cli/test/context/ingest/ingest-bundle.runner.isolated-diff.test.ts index 0201b881..bad40098 100644 --- a/packages/cli/src/context/ingest/ingest-bundle.runner.isolated-diff.test.ts +++ b/packages/cli/test/context/ingest/ingest-bundle.runner.isolated-diff.test.ts @@ -2,12 +2,12 @@ import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promis import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it, vi } from 'vitest'; -import { GitService } from '../../context/core/git.service.js'; -import { SessionWorktreeService } from '../../context/core/session-worktree.service.js'; -import { LocalGitFileStore } from '../project/local-git-file-store.js'; -import { addTouchedSlSource } from '../../context/tools/touched-sl-sources.js'; -import { IngestBundleRunner } from './ingest-bundle.runner.js'; -import type { IngestBundleRunnerDeps } from './ports.js'; +import { GitService } from '../../../src/context/core/git.service.js'; +import { SessionWorktreeService } from '../../../src/context/core/session-worktree.service.js'; +import { LocalGitFileStore } from '../../../src/context/project/local-git-file-store.js'; +import { addTouchedSlSource } from '../../../src/context/tools/touched-sl-sources.js'; +import { IngestBundleRunner } from '../../../src/context/ingest/ingest-bundle.runner.js'; +import type { IngestBundleRunnerDeps } from '../../../src/context/ingest/ports.js'; async function makeRealGitRuntime() { const homeDir = await mkdtemp(join(tmpdir(), 'ktx-isolated-runner-')); diff --git a/packages/cli/src/context/ingest/ingest-bundle.runner.test.ts b/packages/cli/test/context/ingest/ingest-bundle.runner.test.ts similarity index 92% rename from packages/cli/src/context/ingest/ingest-bundle.runner.test.ts rename to packages/cli/test/context/ingest/ingest-bundle.runner.test.ts index 85f45049..b491acf2 100644 --- a/packages/cli/src/context/ingest/ingest-bundle.runner.test.ts +++ b/packages/cli/test/context/ingest/ingest-bundle.runner.test.ts @@ -2,11 +2,11 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { addTouchedSlSource } from '../../context/tools/touched-sl-sources.js'; -import { IngestBundleRunner } from './ingest-bundle.runner.js'; -import { createMemoryFlowLiveBuffer } from './memory-flow/live-buffer.js'; -import type { MemoryFlowReplayInput } from './memory-flow/types.js'; -import type { IngestBundleRunnerDeps } from './ports.js'; +import { addTouchedSlSource } from '../../../src/context/tools/touched-sl-sources.js'; +import { IngestBundleRunner } from '../../../src/context/ingest/ingest-bundle.runner.js'; +import { createMemoryFlowLiveBuffer } from '../../../src/context/ingest/memory-flow/live-buffer.js'; +import type { MemoryFlowReplayInput } from '../../../src/context/ingest/memory-flow/types.js'; +import type { IngestBundleRunnerDeps } from '../../../src/context/ingest/ports.js'; class TestJobContext { private currentProgress = 0; @@ -426,6 +426,177 @@ describe('IngestBundleRunner — Stages 1 → 7', () => { ); }); + it('uses the rate-limit governor for work-unit start slots', async () => { + const deps = makeDeps(); + const acquireWorkSlot = vi.fn(async () => vi.fn()); + const runner = buildRunner(deps, { + settings: { + probeRowCount: 1, + memoryIngestionModel: 'test-model', + workUnitMaxConcurrency: 2, + rateLimitGovernor: { acquireWorkSlot, subscribe: vi.fn(() => vi.fn()) } as never, + }, + }); + deps.adapter.chunk.mockResolvedValue({ + workUnits: [ + { unitKey: 'u1', rawFiles: ['a.yml'], peerFileIndex: [], dependencyPaths: [] }, + { unitKey: 'u2', rawFiles: ['b.yml'], peerFileIndex: [], dependencyPaths: [] }, + ], + }); + (runner as any).stageRawFilesStage1 = vi.fn().mockResolvedValue({ + currentHashes: new Map([ + ['a.yml', 'h1'], + ['b.yml', 'h2'], + ]), + rawDirInWorktree: 'raw-sources/c1/fake/s', + }); + (runner as any).resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x'); + + await runner.run({ + jobId: 'j1', + connectionId: 'c1', + sourceKey: 'fake', + trigger: 'upload', + bundleRef: { kind: 'upload', uploadId: 'upload-x' }, + }); + + expect(acquireWorkSlot).toHaveBeenCalledTimes(2); + }); + + it('passes the job abort signal into rate-limit work-unit slots', async () => { + const deps = makeDeps(); + const controller = new AbortController(); + const acquireWorkSlot = vi.fn(async () => vi.fn()); + const runner = buildRunner(deps, { + settings: { + probeRowCount: 1, + memoryIngestionModel: 'test-model', + workUnitMaxConcurrency: 1, + rateLimitGovernor: { acquireWorkSlot, subscribe: vi.fn(() => vi.fn()) } as never, + }, + }); + (runner as any).stageRawFilesStage1 = vi.fn().mockResolvedValue({ + currentHashes: new Map([['a.yml', 'h1']]), + rawDirInWorktree: 'raw-sources/c1/fake/s', + }); + (runner as any).resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x'); + + await runner.run( + { + jobId: 'j1', + connectionId: 'c1', + sourceKey: 'fake', + trigger: 'upload', + bundleRef: { kind: 'upload', uploadId: 'upload-x' }, + }, + { jobId: 'j1', abortSignal: controller.signal, startPhase: () => new TestJobContext('j1', null, async () => undefined, async () => undefined) } as any, + ); + + expect(acquireWorkSlot).toHaveBeenCalledWith(controller.signal); + }); + + it('does not convert aborted work-unit agent loops into failed work units', async () => { + const deps = makeDeps(); + const controller = new AbortController(); + deps.agentRunner.runLoop.mockImplementation(async () => { + controller.abort(); + throw new DOMException('Aborted', 'AbortError'); + }); + const runner = buildRunner(deps, { + settings: { + probeRowCount: 1, + memoryIngestionModel: 'test-model', + workUnitMaxConcurrency: 1, + }, + }); + (runner as any).stageRawFilesStage1 = vi.fn().mockResolvedValue({ + currentHashes: new Map([['a.yml', 'h1']]), + rawDirInWorktree: 'raw-sources/c1/fake/s', + }); + (runner as any).resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x'); + + await expect( + runner.run( + { + jobId: 'j1', + connectionId: 'c1', + sourceKey: 'fake', + trigger: 'upload', + bundleRef: { kind: 'upload', uploadId: 'upload-x' }, + }, + { jobId: 'j1', abortSignal: controller.signal, startPhase: () => new TestJobContext('j1', null, async () => undefined, async () => undefined) } as any, + ), + ).rejects.toThrow(/Aborted/); + + expect(deps.runsRepo.markFailed).toHaveBeenCalledWith('run-1'); + expect(deps.reportsRepo.create).not.toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.objectContaining({ + failedWorkUnits: expect.arrayContaining(['u1']), + }), + }), + ); + }); + + it('emits trace and memory-flow status for rate-limit waits', async () => { + const deps = makeDeps(); + let subscriber: ((state: any) => void) | undefined; + const memoryFlow = createMemoryFlowLiveBuffer(bundleReplayInput()); + const runner = buildRunner(deps, { + settings: { + probeRowCount: 1, + memoryIngestionModel: 'test-model', + rateLimitGovernor: { + acquireWorkSlot: vi.fn(async () => vi.fn()), + subscribe: vi.fn((cb: (state: any) => void) => { + subscriber = cb; + return vi.fn(); + }), + } as never, + }, + }); + (runner as any).runInner = async (_job: any, ctx: any) => { + subscriber?.({ + kind: 'wait_tick', + provider: 'claude-subscription', + rateLimitType: 'five_hour', + resumeAtMs: 2_000, + remainingMs: 1_000, + }); + ctx.memoryFlow.emit({ type: 'report_created', runId: 'run-1' }); + return { + runId: 'run-1', + syncId: 'sync-1', + diffSummary: { added: 0, modified: 0, deleted: 0, unchanged: 0 }, + workUnitCount: 0, + failedWorkUnits: [], + artifactsWritten: 0, + commitSha: null, + }; + }; + + await runner.run( + { + jobId: 'j1', + connectionId: 'c1', + sourceKey: 'fake', + trigger: 'upload', + bundleRef: { kind: 'upload', uploadId: 'upload-x' }, + }, + { memoryFlow } as any, + ); + + expect(memoryFlow.snapshot().events).toContainEqual( + expect.objectContaining({ + type: 'rate_limit_wait', + provider: 'claude-subscription', + rateLimitType: 'five_hour', + resumeAtMs: 2_000, + remainingMs: 1_000, + }), + ); + }); + it('fails before squash when reconciliation leaves a touched wiki page with dangling refs', async () => { const deps = makeDeps(); let currentToolSession: any = null; diff --git a/packages/cli/test/context/ingest/ingest-profile.test.ts b/packages/cli/test/context/ingest/ingest-profile.test.ts new file mode 100644 index 00000000..1f3f6a58 --- /dev/null +++ b/packages/cli/test/context/ingest/ingest-profile.test.ts @@ -0,0 +1,247 @@ +import { mkdir, mkdtemp, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + aggregateIngestProfile, + formatIngestProfile, + formatIngestProfileJson, + type IngestProfilePaths, + parseTraceEvents, + readIngestProfile, + resolveIngestProfileMode, + type ProfiledTraceEvent, +} from '../../../src/context/ingest/ingest-profile.js'; +import { rm } from 'node:fs/promises'; + +function profilePaths(projectDir: string, jobId: string): IngestProfilePaths { + return { + tracePath: join(projectDir, '.ktx', 'ingest-traces', jobId, 'trace.jsonl'), + transcriptDir: join(projectDir, '.ktx', 'ingest-transcripts', jobId), + }; +} + +function traceLine(event: Partial & { phase: string; event: string }): string { + return JSON.stringify({ schemaVersion: 1, level: 'debug', ...event }); +} + +describe('parseTraceEvents', () => { + it('parses valid JSONL lines and skips blank and malformed ones', () => { + const text = [ + traceLine({ at: '2026-05-30T00:00:00.000Z', phase: 'fetch', event: 'fetch_finished', durationMs: 100 }), + '', + '{ not json', + traceLine({ phase: 'diff', event: 'compute_diff_set_finished', durationMs: 5 }), + ].join('\n'); + const events = parseTraceEvents(text); + expect(events).toHaveLength(2); + expect(events[0].phase).toBe('fetch'); + expect(events[1].event).toBe('compute_diff_set_finished'); + }); +}); + +describe('aggregateIngestProfile', () => { + it('sums durations per phase and sorts by total descending', () => { + const events = parseTraceEvents( + [ + traceLine({ phase: 'fetch', event: 'fetch_finished', durationMs: 1000 }), + traceLine({ phase: 'work_unit', event: 'work_unit_executed', durationMs: 5000, data: { unitKey: 'a' } }), + traceLine({ phase: 'work_unit', event: 'work_unit_executed', durationMs: 3000, data: { unitKey: 'b' } }), + traceLine({ phase: 'diff', event: 'compute_diff_set_finished', durationMs: 50 }), + ].join('\n'), + ); + const profile = aggregateIngestProfile({ jobId: 'job-1', events, toolMsByUnit: {} }); + expect(profile.phases.map((p) => p.phase)).toEqual(['work_unit', 'fetch', 'diff']); + expect(profile.phases[0]).toEqual({ phase: 'work_unit', totalMs: 8000, count: 2 }); + }); + + it('builds per-work-unit rows and derives model time from agent loop minus tool time', () => { + const events = parseTraceEvents( + [ + traceLine({ + phase: 'work_unit', + event: 'work_unit_child_created', + durationMs: 200, + data: { unitKey: 'cards/users' }, + }), + traceLine({ + phase: 'work_unit', + event: 'work_unit_executed', + durationMs: 12000, + data: { unitKey: 'cards/users', status: 'success', agentLoopMs: 10000, stepCount: 12, totalTokens: 48000 }, + }), + traceLine({ + phase: 'work_unit', + event: 'work_unit_child_cleanup', + durationMs: 80, + data: { unitKey: 'cards/users' }, + }), + ].join('\n'), + ); + const profile = aggregateIngestProfile({ jobId: 'job-1', events, toolMsByUnit: { 'cards/users': 2500 } }); + expect(profile.workUnitCount).toBe(1); + const wu = profile.workUnits[0]; + expect(wu).toMatchObject({ + unitKey: 'cards/users', + status: 'success', + totalMs: 12000, + agentLoopMs: 10000, + toolMs: 2500, + modelMs: 7500, + createMs: 200, + cleanupMs: 80, + stepCount: 12, + totalTokens: 48000, + }); + }); + + it('counts failed work units and tolerates missing tool transcripts', () => { + const events = parseTraceEvents( + [ + traceLine({ + phase: 'work_unit', + event: 'work_unit_executed', + durationMs: 4000, + data: { unitKey: 'wu-ok', status: 'success', agentLoopMs: 3800 }, + }), + traceLine({ + phase: 'work_unit', + event: 'work_unit_executed', + durationMs: 1000, + data: { unitKey: 'wu-bad', status: 'failed', agentLoopMs: 900 }, + }), + ].join('\n'), + ); + const profile = aggregateIngestProfile({ jobId: 'job-1', events, toolMsByUnit: {} }); + expect(profile.failedWorkUnitCount).toBe(1); + // No tool transcript → model time falls back to the full agent-loop time. + expect(profile.workUnits.find((w) => w.unitKey === 'wu-ok')?.modelMs).toBe(3800); + }); + + it('derives total wall time from the first and last event timestamps', () => { + const events = parseTraceEvents( + [ + traceLine({ at: '2026-05-30T00:00:00.000Z', phase: 'fetch', event: 'fetch_started' }), + traceLine({ at: '2026-05-30T00:01:30.000Z', phase: 'run', event: 'ingest_finished' }), + ].join('\n'), + ); + const profile = aggregateIngestProfile({ jobId: 'job-1', events, toolMsByUnit: {} }); + expect(profile.totalWallMs).toBe(90_000); + }); +}); + +describe('formatIngestProfile', () => { + it('renders phase breakdown and work-unit rows', () => { + const events = parseTraceEvents( + [ + traceLine({ at: '2026-05-30T00:00:00.000Z', phase: 'work_unit', event: 'work_unit_executed', durationMs: 8000, data: { unitKey: 'cards/users', status: 'success', agentLoopMs: 8000, stepCount: 10, totalTokens: 12000 } }), + traceLine({ at: '2026-05-30T00:00:10.000Z', phase: 'reconciliation', event: 'reconciliation_executed', durationMs: 2000 }), + ].join('\n'), + ); + const profile = aggregateIngestProfile({ jobId: 'job-xyz', events, toolMsByUnit: { 'cards/users': 1000 } }); + const text = formatIngestProfile(profile); + expect(text).toContain('job-xyz'); + expect(text).toContain('Phase breakdown'); + expect(text).toContain('work_unit'); + expect(text).toContain('reconciliation'); + expect(text).toContain('cards/users'); + expect(text).toContain('success'); + }); +}); + +describe('readIngestProfile', () => { + const created: string[] = []; + afterEach(async () => { + for (const dir of created.splice(0)) { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('joins nested tool transcripts to work units by wuKey', async () => { + const projectDir = await mkdtemp(join(tmpdir(), 'ktx-profile-')); + created.push(projectDir); + const jobId = 'job-nested'; + const paths = profilePaths(projectDir, jobId); + await mkdir(join(paths.transcriptDir, 'cards'), { recursive: true }); + await mkdir(join(paths.tracePath, '..'), { recursive: true }); + await writeFile( + paths.tracePath, + [ + JSON.stringify({ + phase: 'work_unit', + event: 'work_unit_executed', + durationMs: 10000, + data: { unitKey: 'cards/marketing', status: 'success', agentLoopMs: 9000, stepCount: 12 }, + }), + ].join('\n'), + 'utf-8', + ); + // Work-unit key has a slash → transcript lives at cards/marketing.jsonl. + await writeFile( + join(paths.transcriptDir, 'cards', 'marketing.jsonl'), + [ + JSON.stringify({ wuKey: 'cards/marketing', toolName: 'sl_write', durationMs: 2000, input: {} }), + JSON.stringify({ wuKey: 'cards/marketing', toolName: 'sl_validate', durationMs: 1000, input: {} }), + ].join('\n'), + 'utf-8', + ); + + const profile = await readIngestProfile(jobId, paths); + const wu = profile.workUnits.find((entry) => entry.unitKey === 'cards/marketing'); + expect(wu?.toolMs).toBe(3000); + expect(wu?.modelMs).toBe(6000); + }); +}); + +describe('resolveIngestProfileMode', () => { + it('reads the table/json/off mode from the env var', () => { + expect(resolveIngestProfileMode(undefined, { KTX_PROFILE_INGEST: '1' })).toBe('table'); + expect(resolveIngestProfileMode(undefined, { KTX_PROFILE_INGEST: 'true' })).toBe('table'); + expect(resolveIngestProfileMode(undefined, { KTX_PROFILE_INGEST: 'json' })).toBe('json'); + expect(resolveIngestProfileMode(undefined, { KTX_PROFILE_INGEST: '0' })).toBe('off'); + expect(resolveIngestProfileMode(undefined, {})).toBe('off'); + }); + + it('reads the mode from the config value', () => { + expect(resolveIngestProfileMode(true, {})).toBe('table'); + expect(resolveIngestProfileMode('json', {})).toBe('json'); + expect(resolveIngestProfileMode(false, {})).toBe('off'); + }); + + it('lets either source request json (json wins)', () => { + expect(resolveIngestProfileMode(true, { KTX_PROFILE_INGEST: 'json' })).toBe('json'); + expect(resolveIngestProfileMode('json', { KTX_PROFILE_INGEST: '1' })).toBe('json'); + }); +}); + +describe('summary and JSON output', () => { + function profileWithReconcileDominant() { + const events = parseTraceEvents( + [ + traceLine({ at: '2026-05-30T00:00:00.000Z', phase: 'work_unit', event: 'work_unit_executed', durationMs: 10000, data: { unitKey: 'a', status: 'success', agentLoopMs: 10000, stepCount: 12, totalTokens: 40000 } }), + traceLine({ at: '2026-05-30T00:01:40.000Z', phase: 'reconciliation', event: 'reconciliation_executed', durationMs: 90000 }), + ].join('\n'), + ); + return aggregateIngestProfile({ jobId: 'job-sum', events, toolMsByUnit: { a: 2000 } }); + } + + it('produces a headline naming the dominant phase and the model/tool split', () => { + const profile = profileWithReconcileDominant(); + expect(profile.summary.dominantPhase?.phase).toBe('reconciliation'); + expect(profile.summary.workUnits).toMatchObject({ count: 1, agentLoopMs: 10000, toolMs: 2000, modelMs: 8000, modelPct: 80 }); + expect(profile.summary.headline).toContain('reconciliation'); + expect(profile.summary.headline).toContain('80%'); + }); + + it('emits raw structured JSON with stable keys for agents', () => { + const profile = profileWithReconcileDominant(); + const text = formatIngestProfileJson(profile); + expect(text).toContain('ktx ingest profile (json)'); + const json = JSON.parse(text.slice(text.indexOf('{'))); + expect(json.jobId).toBe('job-sum'); + expect(json.summary.headline).toEqual(expect.any(String)); + // Raw milliseconds, not human-formatted strings. + expect(json.workUnits[0].agentLoopMs).toBe(10000); + expect(json.phases[0].totalMs).toBe(90000); + }); +}); diff --git a/packages/cli/src/context/ingest/ingest-prompts.test.ts b/packages/cli/test/context/ingest/ingest-prompts.test.ts similarity index 79% rename from packages/cli/src/context/ingest/ingest-prompts.test.ts rename to packages/cli/test/context/ingest/ingest-prompts.test.ts index 8adcbfef..54617eb2 100644 --- a/packages/cli/src/context/ingest/ingest-prompts.test.ts +++ b/packages/cli/test/context/ingest/ingest-prompts.test.ts @@ -8,7 +8,7 @@ function forbiddenProductPattern() { describe('ingest prompt assets', () => { it('teaches WorkUnit agents to apply canonical pins before writing contested artifacts', async () => { const prompt = await readFile( - new URL('../../prompts/memory_agent_bundle_ingest_work_unit.md', import.meta.url), + new URL('../../../src/prompts/memory_agent_bundle_ingest_work_unit.md', import.meta.url), 'utf-8', ); @@ -20,7 +20,7 @@ describe('ingest prompt assets', () => { it('uses product-neutral KTX runtime wording', async () => { const prompt = await readFile( - new URL('../../prompts/memory_agent_bundle_ingest_work_unit.md', import.meta.url), + new URL('../../../src/prompts/memory_agent_bundle_ingest_work_unit.md', import.meta.url), 'utf-8', ); @@ -31,7 +31,7 @@ describe('ingest prompt assets', () => { it('uses shipped warehouse verification tools in the WorkUnit prompt', async () => { const prompt = await readFile( - new URL('../../prompts/memory_agent_bundle_ingest_work_unit.md', import.meta.url), + new URL('../../../src/prompts/memory_agent_bundle_ingest_work_unit.md', import.meta.url), 'utf-8', ); @@ -42,7 +42,7 @@ describe('ingest prompt assets', () => { }); it('does not route historic-SQL through page-triage prompt examples', async () => { - const prompt = await readFile(new URL('../../prompts/skills/page_triage_classifier.md', import.meta.url), 'utf-8'); + const prompt = await readFile(new URL('../../../src/prompts/skills/page_triage_classifier.md', import.meta.url), 'utf-8'); expect(prompt).not.toContain(['historic_sql', 'template'].join('_')); expect(prompt).not.toContain('service_account_only=true AND below the frequency floor'); diff --git a/packages/cli/src/context/ingest/ingest-runtime-assets.test.ts b/packages/cli/test/context/ingest/ingest-runtime-assets.test.ts similarity index 92% rename from packages/cli/src/context/ingest/ingest-runtime-assets.test.ts rename to packages/cli/test/context/ingest/ingest-runtime-assets.test.ts index f6a46111..ad77e692 100644 --- a/packages/cli/src/context/ingest/ingest-runtime-assets.test.ts +++ b/packages/cli/test/context/ingest/ingest-runtime-assets.test.ts @@ -2,11 +2,11 @@ import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; -import { PromptService } from '../../context/prompts/prompt.service.js'; -import { SkillsRegistryService } from '../../context/skills/skills-registry.service.js'; +import { PromptService } from '../../../src/context/prompts/prompt.service.js'; +import { SkillsRegistryService } from '../../../src/context/skills/skills-registry.service.js'; -const promptsDir = fileURLToPath(new URL('../../prompts', import.meta.url)); -const skillsDir = fileURLToPath(new URL('../../skills', import.meta.url)); +const promptsDir = fileURLToPath(new URL('../../../src/prompts', import.meta.url)); +const skillsDir = fileURLToPath(new URL('../../../src/skills', import.meta.url)); const adapterSkillNames = [ 'live_database_ingest', diff --git a/packages/cli/src/context/ingest/ingest-trace.test.ts b/packages/cli/test/context/ingest/ingest-trace.test.ts similarity index 98% rename from packages/cli/src/context/ingest/ingest-trace.test.ts rename to packages/cli/test/context/ingest/ingest-trace.test.ts index 88b56a37..b10e2118 100644 --- a/packages/cli/src/context/ingest/ingest-trace.test.ts +++ b/packages/cli/test/context/ingest/ingest-trace.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, readFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it, vi } from 'vitest'; -import { FileIngestTraceWriter, ingestTracePathForJob, traceTimed } from './ingest-trace.js'; +import { FileIngestTraceWriter, ingestTracePathForJob, traceTimed } from '../../../src/context/ingest/ingest-trace.js'; describe('FileIngestTraceWriter', () => { it('persists structured trace events as JSONL', async () => { diff --git a/packages/cli/src/context/ingest/isolated-diff/git-patch.test.ts b/packages/cli/test/context/ingest/isolated-diff/git-patch.test.ts similarity index 97% rename from packages/cli/src/context/ingest/isolated-diff/git-patch.test.ts rename to packages/cli/test/context/ingest/isolated-diff/git-patch.test.ts index 2a48ce9b..f9925ebb 100644 --- a/packages/cli/src/context/ingest/isolated-diff/git-patch.test.ts +++ b/packages/cli/test/context/ingest/isolated-diff/git-patch.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { assertPatchAllowedForWorkUnit, parsePatchTouchedPaths, textArtifactRoots } from './git-patch.js'; +import { assertPatchAllowedForWorkUnit, parsePatchTouchedPaths, textArtifactRoots } from '../../../../src/context/ingest/isolated-diff/git-patch.js'; describe('isolated diff patch contract', () => { it('parses touched paths from no-rename git patches', () => { diff --git a/packages/cli/src/context/ingest/isolated-diff/patch-integrator.test.ts b/packages/cli/test/context/ingest/isolated-diff/patch-integrator.test.ts similarity index 82% rename from packages/cli/src/context/ingest/isolated-diff/patch-integrator.test.ts rename to packages/cli/test/context/ingest/isolated-diff/patch-integrator.test.ts index e547e22e..e822472d 100644 --- a/packages/cli/src/context/ingest/isolated-diff/patch-integrator.test.ts +++ b/packages/cli/test/context/ingest/isolated-diff/patch-integrator.test.ts @@ -2,9 +2,9 @@ import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it, vi } from 'vitest'; -import { GitService } from '../../../context/core/git.service.js'; -import { FileIngestTraceWriter } from '../ingest-trace.js'; -import { integrateWorkUnitPatch } from './patch-integrator.js'; +import { GitService } from '../../../../src/context/core/git.service.js'; +import { FileIngestTraceWriter } from '../../../../src/context/ingest/ingest-trace.js'; +import { integrateWorkUnitPatch } from '../../../../src/context/ingest/isolated-diff/patch-integrator.js'; async function makeRepo() { const homeDir = await mkdtemp(join(tmpdir(), 'ktx-integrate-')); @@ -401,4 +401,72 @@ describe('integrateWorkUnitPatch', () => { }); await expect(readFile(join(configDir, 'wiki/global/a.md'), 'utf-8')).resolves.toBe('old\n'); }); + + it('repairs a semantic gate failure after a textual conflict is resolved', async () => { + const { homeDir, configDir, git } = await makeRepo(); + await mkdir(join(configDir, 'wiki/global'), { recursive: true }); + await writeFile(join(configDir, 'wiki/global/a.md'), 'base\n', 'utf-8'); + await git.commitFiles(['wiki/global/a.md'], 'base page', 'System User', 'system@example.com'); + const conflictBase = await git.revParseHead(); + + await writeFile(join(configDir, 'wiki/global/a.md'), 'accepted\n', 'utf-8'); + await git.commitFiles(['wiki/global/a.md'], 'accepted edit', 'System User', 'system@example.com'); + + const childDir = join(homeDir, 'child-conflict-repair'); + await git.addWorktree(childDir, 'child-conflict-repair', conflictBase); + const childGit = git.forWorktree(childDir); + await writeFile(join(childDir, 'wiki/global/a.md'), 'proposal\n', 'utf-8'); + await childGit.commitFiles(['wiki/global/a.md'], 'proposal edit', 'System User', 'system@example.com'); + const patchPath = join(homeDir, 'proposal-repair.patch'); + await childGit.writeBinaryNoRenamePatch(conflictBase, 'HEAD', patchPath); + + const trace = new FileIngestTraceWriter({ + tracePath: join(homeDir, '.ktx/ingest-traces/job-resolver-repair/trace.jsonl'), + jobId: 'job-resolver-repair', + connectionId: 'warehouse', + sourceKey: 'metabase', + level: 'trace', + }); + + // Gate fails on the resolver's merged tree, then passes after the repair edit. + const validateAppliedTree = vi + .fn() + .mockRejectedValueOnce( + new Error('final artifact gates failed:\narr-definition: unknown sl_refs entity mart_arr_daily.arr_dollars'), + ) + .mockResolvedValueOnce(undefined); + + const repairGateFailure = vi.fn(async (context: { unitKey: string; touchedPaths: string[] }) => { + expect(context).toMatchObject({ unitKey: 'wu-conflict-repair', touchedPaths: ['wiki/global/a.md'] }); + await writeFile(join(configDir, 'wiki/global/a.md'), 'accepted\nproposal repaired\n', 'utf-8'); + return { status: 'repaired' as const, attempts: 1, changedPaths: ['wiki/global/a.md'] }; + }); + + const result = await integrateWorkUnitPatch({ + unitKey: 'wu-conflict-repair', + patchPath, + integrationGit: git, + trace, + author: { name: 'System User', email: 'system@example.com' }, + slDisallowed: false, + allowedTargetConnectionIds: new Set(['warehouse']), + validateAppliedTree, + resolveTextualConflict: vi.fn(async () => { + await writeFile(join(configDir, 'wiki/global/a.md'), 'accepted\nproposal\n', 'utf-8'); + return { status: 'repaired' as const, attempts: 1, changedPaths: ['wiki/global/a.md'] }; + }), + repairGateFailure, + }); + + expect(result).toMatchObject({ + status: 'accepted', + touchedPaths: ['wiki/global/a.md'], + textualResolution: { status: 'repaired' }, + gateRepair: { status: 'repaired', attempts: 1, changedPaths: ['wiki/global/a.md'] }, + }); + expect(validateAppliedTree).toHaveBeenCalledTimes(2); + expect(repairGateFailure).toHaveBeenCalledOnce(); + await expect(readFile(join(configDir, 'wiki/global/a.md'), 'utf-8')).resolves.toBe('accepted\nproposal repaired\n'); + await expect(readFile(trace.tracePath, 'utf-8')).resolves.toContain('patch_accepted_after_textual_resolution'); + }); }); diff --git a/packages/cli/src/context/ingest/isolated-diff/textual-conflict-resolver.test.ts b/packages/cli/test/context/ingest/isolated-diff/textual-conflict-resolver.test.ts similarity index 94% rename from packages/cli/src/context/ingest/isolated-diff/textual-conflict-resolver.test.ts rename to packages/cli/test/context/ingest/isolated-diff/textual-conflict-resolver.test.ts index ae5b4e21..a03eb66d 100644 --- a/packages/cli/src/context/ingest/isolated-diff/textual-conflict-resolver.test.ts +++ b/packages/cli/test/context/ingest/isolated-diff/textual-conflict-resolver.test.ts @@ -2,8 +2,8 @@ import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it, vi } from 'vitest'; -import { FileIngestTraceWriter } from '../ingest-trace.js'; -import { resolveTextualConflict } from './textual-conflict-resolver.js'; +import { FileIngestTraceWriter } from '../../../../src/context/ingest/ingest-trace.js'; +import { resolveTextualConflict } from '../../../../src/context/ingest/isolated-diff/textual-conflict-resolver.js'; async function makeHarness() { const root = await mkdtemp(join(tmpdir(), 'ktx-textual-resolver-')); diff --git a/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.test.ts b/packages/cli/test/context/ingest/isolated-diff/work-unit-executor.test.ts similarity index 95% rename from packages/cli/src/context/ingest/isolated-diff/work-unit-executor.test.ts rename to packages/cli/test/context/ingest/isolated-diff/work-unit-executor.test.ts index 5975dee8..cc06ba61 100644 --- a/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.test.ts +++ b/packages/cli/test/context/ingest/isolated-diff/work-unit-executor.test.ts @@ -2,9 +2,9 @@ import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it, vi } from 'vitest'; -import { GitService } from '../../../context/core/git.service.js'; -import { FileIngestTraceWriter } from '../ingest-trace.js'; -import { runIsolatedWorkUnit } from './work-unit-executor.js'; +import { GitService } from '../../../../src/context/core/git.service.js'; +import { FileIngestTraceWriter } from '../../../../src/context/ingest/ingest-trace.js'; +import { runIsolatedWorkUnit } from '../../../../src/context/ingest/isolated-diff/work-unit-executor.js'; async function makeGit() { const homeDir = await mkdtemp(join(tmpdir(), 'ktx-isolated-wu-')); diff --git a/packages/cli/src/context/ingest/local-adapters.test.ts b/packages/cli/test/context/ingest/local-adapters.test.ts similarity index 85% rename from packages/cli/src/context/ingest/local-adapters.test.ts rename to packages/cli/test/context/ingest/local-adapters.test.ts index 44fdc2cc..a8799cee 100644 --- a/packages/cli/src/context/ingest/local-adapters.test.ts +++ b/packages/cli/test/context/ingest/local-adapters.test.ts @@ -1,13 +1,13 @@ -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js'; -import type { SqlAnalysisPort } from '../../context/sql-analysis/ports.js'; -import type { HistoricSqlReader } from './adapters/historic-sql/types.js'; -import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js'; -import { LocalNotionRuntimeStore } from './adapters/notion/local-state-store.js'; -import { createDefaultLocalIngestAdapters, localPullConfigForAdapter } from './local-adapters.js'; +import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../../src/context/project/project.js'; +import type { SqlAnalysisPort } from '../../../src/context/sql-analysis/ports.js'; +import type { HistoricSqlReader } from '../../../src/context/ingest/adapters/historic-sql/types.js'; +import { LocalLookerRuntimeStore } from '../../../src/context/ingest/adapters/looker/local-runtime-store.js'; +import { LocalNotionRuntimeStore } from '../../../src/context/ingest/adapters/notion/local-state-store.js'; +import { createDefaultLocalIngestAdapters, localPullConfigForAdapter } from '../../../src/context/ingest/local-adapters.js'; describe('local ingest adapters', () => { let tempDir: string; @@ -34,6 +34,36 @@ describe('local ingest adapters', () => { }; } + async function seedLiveScanTable( + projectDir: string, + connectionId: string, + table: { catalog: string | null; db: string | null; name: string }, + ): Promise { + const rawRoot = join(projectDir, 'raw-sources', connectionId, 'live-database', 'sync-1'); + await mkdir(join(rawRoot, 'tables'), { recursive: true }); + await writeFile( + join(rawRoot, 'connection.json'), + `${JSON.stringify({ connectionId, driver: 'postgres' }, null, 2)}\n`, + 'utf-8', + ); + await writeFile( + join(rawRoot, 'tables', `${table.db ?? 'default'}-${table.name}.json`), + `${JSON.stringify( + { + ...table, + kind: 'table', + comment: null, + estimatedRows: null, + columns: [], + foreignKeys: [], + }, + null, + 2, + )}\n`, + 'utf-8', + ); + } + it('registers Metabase locally as a staged-bundle adapter', () => { const adapters = createDefaultLocalIngestAdapters(project); @@ -76,7 +106,7 @@ describe('local ingest adapters', () => { expect(looker?.fetch).toBeTypeOf('function'); }); - it('returns the explicit Metabase fan-out boundary before runner construction', async () => { + it('returns the explicit Metabase fanout boundary before runner construction', async () => { const metabase = createDefaultLocalIngestAdapters(project).find((adapter) => adapter.source === 'metabase'); await expect(localPullConfigForAdapter(project, metabase!, 'warehouse')).rejects.toThrow( @@ -205,11 +235,14 @@ describe('local ingest adapters', () => { dialect: 'postgres', minExecutions: 7, enabledTables: [], + enabledSchemas: [], + modeledTableCatalog: [], filters: { serviceAccounts: { patterns: ['^svc_'], mode: 'exclude' }, dropTrivialProbes: true, }, redactionPatterns: [], + scopeFloorWarnings: [], staleArchiveAfterDays: 90, }); }); @@ -237,6 +270,71 @@ describe('local ingest adapters', () => { }); }); + it('passes computed modeled scope to direct historic-sql adapter pull config', async () => { + await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true }); + await writeFile( + join(project.projectDir, 'semantic-layer/warehouse/revenue.yaml'), + [ + 'name: revenue', + 'table: orbit_analytics.mart_revenue', + 'grain: [id]', + 'columns:', + ' - name: id', + ' type: string', + '', + ].join('\n'), + 'utf-8', + ); + await seedLiveScanTable(project.projectDir, 'warehouse', { + catalog: null, + db: 'orbit_raw', + name: 'accounts', + }); + const projectWithQueryHistory = projectWithConnections({ + warehouse: { + driver: 'postgres', + schemas: ['orbit_raw'], + context: { + queryHistory: { + enabled: true, + minExecutions: 7, + filters: { dropTrivialProbes: true }, + }, + }, + }, + }); + const adapter = { source: 'historic-sql' } as never; + + await expect(localPullConfigForAdapter(projectWithQueryHistory, adapter, 'warehouse')).resolves.toMatchObject({ + dialect: 'postgres', + minExecutions: 7, + enabledSchemas: ['orbit_analytics', 'orbit_raw'], + modeledTableCatalog: [ + { catalog: null, db: 'orbit_analytics', name: 'mart_revenue' }, + { catalog: null, db: 'orbit_raw', name: 'accounts' }, + ], + }); + }); + + it('passes query-history scope fail-open warnings to direct historic-sql pull config', async () => { + const projectDir = await mkdtemp(join(tmpdir(), 'ktx-local-qh-scope-warning-')); + const project = await initKtxProject({ projectDir }); + project.config.connections.warehouse = { + driver: 'postgres', + schemas: ['orbit_raw'], + context: { queryHistory: { enabled: true } }, + } as never; + const adapter = { source: 'historic-sql' } as never; + + await expect(localPullConfigForAdapter(project, adapter, 'warehouse')).resolves.toMatchObject({ + dialect: 'postgres', + enabledSchemas: ['*'], + scopeFloorWarnings: ['query_history_scope_floor_disabled:catalog_unavailable'], + }); + + await rm(projectDir, { recursive: true, force: true }); + }); + it('rejects local historic-sql pulls when the connection has not enabled historic SQL', async () => { const historicSql = createDefaultLocalIngestAdapters(project, { historicSql: { diff --git a/packages/cli/src/context/ingest/local-bundle-ingest.test.ts b/packages/cli/test/context/ingest/local-bundle-ingest.test.ts similarity index 97% rename from packages/cli/src/context/ingest/local-bundle-ingest.test.ts rename to packages/cli/test/context/ingest/local-bundle-ingest.test.ts index 4b4b834c..6140dd10 100644 --- a/packages/cli/src/context/ingest/local-bundle-ingest.test.ts +++ b/packages/cli/test/context/ingest/local-bundle-ingest.test.ts @@ -3,22 +3,22 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import Database from 'better-sqlite3'; import YAML from 'yaml'; -import type { AgentRunnerPort, RunLoopParams } from '../../context/llm/runtime-port.js'; -import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js'; +import type { AgentRunnerPort, RunLoopParams } from '../../../src/context/llm/runtime-port.js'; +import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../../src/context/project/project.js'; import { makeLocalGitRepo } from '../test/make-local-git-repo.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js'; -import { projectHistoricSqlEvidence } from './adapters/historic-sql/projection.js'; -import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js'; -import { createDefaultLocalIngestAdapters, localPullConfigForAdapter } from './local-adapters.js'; -import { getLocalIngestStatus, runLocalIngest } from './local-ingest.js'; +import { FakeSourceAdapter } from '../../../src/context/ingest/adapters/fake/fake.adapter.js'; +import { projectHistoricSqlEvidence } from '../../../src/context/ingest/adapters/historic-sql/projection.js'; +import { LocalLookerRuntimeStore } from '../../../src/context/ingest/adapters/looker/local-runtime-store.js'; +import { createDefaultLocalIngestAdapters, localPullConfigForAdapter } from '../../../src/context/ingest/local-adapters.js'; +import { getLocalIngestStatus, runLocalIngest } from '../../../src/context/ingest/local-ingest.js'; import type { ChunkResult, DeterministicFinalizationContext, DiffSet, FinalizationResult, SourceAdapter, -} from './types.js'; +} from '../../../src/context/ingest/types.js'; class TestAgentRunner implements AgentRunnerPort { runLoop = vi.fn().mockResolvedValue({ stopReason: 'natural' as const }); @@ -139,7 +139,6 @@ class HistoricSqlEvidenceAgentRunner implements AgentRunnerPort { const result = await emitEvidence.execute({ kind: 'table_usage', table: 'public.orders', - rawPath: 'tables/public.orders.json', usage: { narrative: 'Orders are repeatedly queried by lifecycle status.', frequencyTier: 'high', diff --git a/packages/cli/src/context/ingest/local-bundle-runtime.test.ts b/packages/cli/test/context/ingest/local-bundle-runtime.test.ts similarity index 94% rename from packages/cli/src/context/ingest/local-bundle-runtime.test.ts rename to packages/cli/test/context/ingest/local-bundle-runtime.test.ts index 3c87c351..e3031cc5 100644 --- a/packages/cli/src/context/ingest/local-bundle-runtime.test.ts +++ b/packages/cli/test/context/ingest/local-bundle-runtime.test.ts @@ -1,11 +1,11 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import type { AgentRunnerPort } from '../../context/llm/runtime-port.js'; -import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js'; +import type { AgentRunnerPort } from '../../../src/context/llm/runtime-port.js'; +import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../../src/context/project/project.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js'; -import { createLocalBundleIngestRuntime } from './local-bundle-runtime.js'; +import { FakeSourceAdapter } from '../../../src/context/ingest/adapters/fake/fake.adapter.js'; +import { createLocalBundleIngestRuntime } from '../../../src/context/ingest/local-bundle-runtime.js'; type RuntimeWithConnectionDeps = { deps: { @@ -77,9 +77,10 @@ describe('createLocalBundleIngestRuntime', () => { }), ).toThrow( [ - 'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, or claude-code, or an injected agentRunner.', - 'Configure a local Claude Code session or API-backed LLM, then rerun ingest:', + 'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, claude-code, or codex, or an injected agentRunner.', + 'Configure a local Claude Code/Codex session or API-backed LLM, then rerun ingest:', ` ktx setup --project-dir ${project.projectDir} --llm-backend claude-code --no-input`, + ` ktx setup --project-dir ${project.projectDir} --llm-backend codex --llm-model gpt-5.5 --no-input`, ` ktx setup --project-dir ${project.projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --llm-model claude-sonnet-4-6 --no-input`, ].join('\n'), ); @@ -299,6 +300,8 @@ describe('createLocalBundleIngestRuntime', () => { 'ingestTraceLevel', 'memoryIngestionModel', 'probeRowCount', + 'profileIngest', + 'rateLimitGovernor', 'workUnitFailureMode', 'workUnitMaxConcurrency', 'workUnitStepBudget', diff --git a/packages/cli/src/context/ingest/local-embedding-provider.integration.test.ts b/packages/cli/test/context/ingest/local-embedding-provider.integration.test.ts similarity index 90% rename from packages/cli/src/context/ingest/local-embedding-provider.integration.test.ts rename to packages/cli/test/context/ingest/local-embedding-provider.integration.test.ts index 34114e88..f0214957 100644 --- a/packages/cli/src/context/ingest/local-embedding-provider.integration.test.ts +++ b/packages/cli/test/context/ingest/local-embedding-provider.integration.test.ts @@ -2,11 +2,11 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import type { KtxEmbeddingPort } from '../core/embedding.js'; -import { CandidateDedupService } from './context-candidates/candidate-dedup.service.js'; -import { ContextEvidenceIndexService } from './context-evidence/context-evidence-index.service.js'; -import { SqliteContextEvidenceStore } from './context-evidence/sqlite-context-evidence-store.js'; -import type { DiffSet } from './types.js'; +import type { KtxEmbeddingPort } from '../../../src/context/core/embedding.js'; +import { CandidateDedupService } from '../../../src/context/ingest/context-candidates/candidate-dedup.service.js'; +import { ContextEvidenceIndexService } from '../../../src/context/ingest/context-evidence/context-evidence-index.service.js'; +import { SqliteContextEvidenceStore } from '../../../src/context/ingest/context-evidence/sqlite-context-evidence-store.js'; +import type { DiffSet } from '../../../src/context/ingest/types.js'; describe('local ingest embedding providers with SQLite ingest stores', () => { let tempDir: string; diff --git a/packages/cli/src/context/ingest/local-mapping-reconcile.test.ts b/packages/cli/test/context/ingest/local-mapping-reconcile.test.ts similarity index 87% rename from packages/cli/src/context/ingest/local-mapping-reconcile.test.ts rename to packages/cli/test/context/ingest/local-mapping-reconcile.test.ts index 3eed9d53..8f7080c6 100644 --- a/packages/cli/src/context/ingest/local-mapping-reconcile.test.ts +++ b/packages/cli/test/context/ingest/local-mapping-reconcile.test.ts @@ -2,10 +2,10 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; -import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js'; -import type { KtxLocalProject } from '../../context/project/project.js'; -import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js'; -import { seedLocalMappingStateFromKtxYaml } from './local-mapping-reconcile.js'; +import { ktxLocalStateDbPath } from '../../../src/context/project/local-state-db.js'; +import type { KtxLocalProject } from '../../../src/context/project/project.js'; +import { LocalLookerRuntimeStore } from '../../../src/context/ingest/adapters/looker/local-runtime-store.js'; +import { seedLocalMappingStateFromKtxYaml } from '../../../src/context/ingest/local-mapping-reconcile.js'; describe('local mapping yaml reconciliation bridge', () => { let tempDir: string; diff --git a/packages/cli/src/context/ingest/local-metabase-ingest.test.ts b/packages/cli/test/context/ingest/local-metabase-ingest.test.ts similarity index 87% rename from packages/cli/src/context/ingest/local-metabase-ingest.test.ts rename to packages/cli/test/context/ingest/local-metabase-ingest.test.ts index ff91d827..8fb89bd0 100644 --- a/packages/cli/src/context/ingest/local-metabase-ingest.test.ts +++ b/packages/cli/test/context/ingest/local-metabase-ingest.test.ts @@ -1,12 +1,13 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import type { AgentRunnerPort, RunLoopParams } from '../../context/llm/runtime-port.js'; +import type { AgentRunnerPort, RunLoopParams } from '../../../src/context/llm/runtime-port.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; -import { LocalMetabaseDiscoveryCache } from './adapters/metabase/local-source-state-store.js'; -import { getLocalIngestStatus, runLocalMetabaseIngest } from './local-ingest.js'; -import type { ChunkResult, FetchContext, SourceAdapter } from './types.js'; +import { initKtxProject, type KtxLocalProject } from '../../../src/context/project/project.js'; +import { LocalMetabaseDiscoveryCache } from '../../../src/context/ingest/adapters/metabase/local-source-state-store.js'; +import { getLocalIngestStatus, runLocalMetabaseIngest } from '../../../src/context/ingest/local-ingest.js'; +import { ingestReportOutcome } from '../../../src/context/ingest/reports.js'; +import type { ChunkResult, FetchContext, SourceAdapter } from '../../../src/context/ingest/types.js'; class TestAgentRunner implements AgentRunnerPort { runLoop = vi.fn(async (params: RunLoopParams) => { @@ -148,7 +149,7 @@ describe('runLocalMetabaseIngest', () => { ).rejects.toThrow('no sync-enabled mappings with a target connection'); }); - it('seeds yaml-only Metabase mappings before the unhydrated fan-out preflight', async () => { + it('seeds yaml-only Metabase mappings before the unhydrated fanout preflight', async () => { project.config.connections['prod-metabase'].mappings = { databaseMappings: { '1': 'warehouse_a' }, syncEnabled: { '1': true }, @@ -172,7 +173,7 @@ describe('runLocalMetabaseIngest', () => { ]); }); - it('rejects source-dir uploads through the Metabase fan-out runner', async () => { + it('rejects source-dir uploads through the Metabase fanout runner', async () => { await expect( runLocalMetabaseIngest({ project, @@ -181,7 +182,7 @@ describe('runLocalMetabaseIngest', () => { agentRunner: new TestAgentRunner(), sourceDir: tempDir, } as Parameters[0] & { sourceDir: string }), - ).rejects.toThrow('source-dir uploads are not supported for the Metabase fan-out adapter'); + ).rejects.toThrow('source-dir uploads are not supported for the Metabase fanout adapter'); }); it('reports partial failure when a child job fails', async () => { @@ -202,6 +203,24 @@ describe('runLocalMetabaseIngest', () => { expect(result.children[1]?.report.body.failedWorkUnits).toEqual(['metabase-db-2']); }); + it('keeps a child that saved memory out of all_failed when another child fails', async () => { + await seedMetabaseState(); + const agentRunner = new TestAgentRunner(); + const ids = ['metabase-child-1', 'metabase-child-2']; + + const result = await runLocalMetabaseIngest({ + project, + adapters: [new FakeMetabaseSourceAdapter()], + metabaseConnectionId: 'prod-metabase', + agentRunner, + jobIdFactory: () => ids.shift() ?? 'metabase-child-extra', + }); + + expect(result.status).toBe('partial_failure'); + expect(ingestReportOutcome(result.children[0].report)).toBe('done'); + expect(ingestReportOutcome(result.children[1].report)).toBe('error'); + }); + it('captures fetch-time child failures and continues later mappings', async () => { await seedMetabaseState(); project.config.connections.warehouse_c = { driver: 'postgres', url: 'postgres://localhost/c' }; diff --git a/packages/cli/src/context/ingest/local-stage-ingest.test.ts b/packages/cli/test/context/ingest/local-stage-ingest.test.ts similarity index 97% rename from packages/cli/src/context/ingest/local-stage-ingest.test.ts rename to packages/cli/test/context/ingest/local-stage-ingest.test.ts index 7a2c5a6a..c57d18b4 100644 --- a/packages/cli/src/context/ingest/local-stage-ingest.test.ts +++ b/packages/cli/test/context/ingest/local-stage-ingest.test.ts @@ -2,16 +2,16 @@ import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promise import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js'; -import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js'; -import { createDefaultLocalIngestAdapters } from './local-adapters.js'; +import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../../src/context/project/project.js'; +import { FakeSourceAdapter } from '../../../src/context/ingest/adapters/fake/fake.adapter.js'; +import { createDefaultLocalIngestAdapters } from '../../../src/context/ingest/local-adapters.js'; import { getLocalStageOnlyIngestStatus, runLocalStageOnlyIngest, -} from './local-stage-ingest.js'; -import { createMemoryFlowLiveBuffer } from './memory-flow/live-buffer.js'; -import type { MemoryFlowReplayInput } from './memory-flow/types.js'; -import type { SourceAdapter } from './types.js'; +} from '../../../src/context/ingest/local-stage-ingest.js'; +import { createMemoryFlowLiveBuffer } from '../../../src/context/ingest/memory-flow/live-buffer.js'; +import type { MemoryFlowReplayInput } from '../../../src/context/ingest/memory-flow/types.js'; +import type { SourceAdapter } from '../../../src/context/ingest/types.js'; async function writeWarehouseConfig(projectDir: string): Promise { await writeFile( @@ -591,7 +591,7 @@ describe('local ingest', () => { status: 'done', adapter: 'live-database', connectionId: 'warehouse', - rawFileCount: 3, + rawFileCount: 4, workUnitCount: 1, }); }); diff --git a/packages/cli/src/context/ingest/memory-flow/acceptance-fixtures.ts b/packages/cli/test/context/ingest/memory-flow/acceptance-fixtures.ts similarity index 98% rename from packages/cli/src/context/ingest/memory-flow/acceptance-fixtures.ts rename to packages/cli/test/context/ingest/memory-flow/acceptance-fixtures.ts index 66f1afb8..1d57aaa8 100644 --- a/packages/cli/src/context/ingest/memory-flow/acceptance-fixtures.ts +++ b/packages/cli/test/context/ingest/memory-flow/acceptance-fixtures.ts @@ -1,4 +1,4 @@ -import type { MemoryFlowReplayInput } from './types.js'; +import type { MemoryFlowReplayInput } from '../../../../src/context/ingest/memory-flow/types.js'; function baseScenario(overrides: Partial = {}): MemoryFlowReplayInput { return { diff --git a/packages/cli/src/context/ingest/memory-flow/acceptance.test.ts b/packages/cli/test/context/ingest/memory-flow/acceptance.test.ts similarity index 92% rename from packages/cli/src/context/ingest/memory-flow/acceptance.test.ts rename to packages/cli/test/context/ingest/memory-flow/acceptance.test.ts index 42bff8a0..04e7fa46 100644 --- a/packages/cli/src/context/ingest/memory-flow/acceptance.test.ts +++ b/packages/cli/test/context/ingest/memory-flow/acceptance.test.ts @@ -6,8 +6,8 @@ import { successfulReplayScenario, validationRevertScenario, } from './acceptance-fixtures.js'; -import { renderMemoryFlowReplay } from './render.js'; -import { buildMemoryFlowViewModel } from './view-model.js'; +import { renderMemoryFlowReplay } from '../../../../src/context/ingest/memory-flow/render.js'; +import { buildMemoryFlowViewModel } from '../../../../src/context/ingest/memory-flow/view-model.js'; function renderScenario(input = successfulReplayScenario(), terminalWidth = 140): string { return renderMemoryFlowReplay(buildMemoryFlowViewModel(input), { terminalWidth }); diff --git a/packages/cli/src/context/ingest/memory-flow/events.test.ts b/packages/cli/test/context/ingest/memory-flow/events.test.ts similarity index 97% rename from packages/cli/src/context/ingest/memory-flow/events.test.ts rename to packages/cli/test/context/ingest/memory-flow/events.test.ts index be97342b..cb0e72c8 100644 --- a/packages/cli/src/context/ingest/memory-flow/events.test.ts +++ b/packages/cli/test/context/ingest/memory-flow/events.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; -import type { LocalIngestRunRecord } from '../local-stage-ingest.js'; -import type { IngestReportSnapshot } from '../reports.js'; -import { ingestReportToMemoryFlowReplay, localIngestRunToMemoryFlowReplay } from './events.js'; +import type { LocalIngestRunRecord } from '../../../../src/context/ingest/local-stage-ingest.js'; +import type { IngestReportSnapshot } from '../../../../src/context/ingest/reports.js'; +import { ingestReportToMemoryFlowReplay, localIngestRunToMemoryFlowReplay } from '../../../../src/context/ingest/memory-flow/events.js'; function localRecord(): LocalIngestRunRecord { return { @@ -166,7 +166,7 @@ describe('memory-flow event mapping', () => { runId: 'run-1', connectionId: 'warehouse', adapter: 'lookml', - status: 'error', + status: 'done', sourceDir: null, syncId: 'sync-2', reportId: 'report-1', @@ -308,7 +308,7 @@ describe('memory-flow event mapping', () => { sourceReportPath: 'report-1', fallbackReason: null, }); - expect(replay.status).toBe('error'); + expect(replay.status).toBe('done'); expect(replay.reportId).toBe('report-1'); expect(replay.reportPath).toBe('report-1'); expect(replay.events[0]).toMatchObject({ type: 'source_acquired', emittedAt: '2026-05-01T10:00:00.000Z' }); diff --git a/packages/cli/src/context/ingest/memory-flow/interaction.test.ts b/packages/cli/test/context/ingest/memory-flow/interaction.test.ts similarity index 98% rename from packages/cli/src/context/ingest/memory-flow/interaction.test.ts rename to packages/cli/test/context/ingest/memory-flow/interaction.test.ts index 290180df..012373f7 100644 --- a/packages/cli/src/context/ingest/memory-flow/interaction.test.ts +++ b/packages/cli/test/context/ingest/memory-flow/interaction.test.ts @@ -8,8 +8,8 @@ import { selectedMemoryFlowColumn, selectedMemoryFlowDetails, visibleMemoryFlowChips, -} from './interaction.js'; -import type { MemoryFlowInteractionState, MemoryFlowViewModel } from './types.js'; +} from '../../../../src/context/ingest/memory-flow/interaction.js'; +import type { MemoryFlowInteractionState, MemoryFlowViewModel } from '../../../../src/context/ingest/memory-flow/types.js'; function view(): MemoryFlowViewModel { return { diff --git a/packages/cli/src/context/ingest/memory-flow/interactive-render.test.ts b/packages/cli/test/context/ingest/memory-flow/interactive-render.test.ts similarity index 95% rename from packages/cli/src/context/ingest/memory-flow/interactive-render.test.ts rename to packages/cli/test/context/ingest/memory-flow/interactive-render.test.ts index 6b703a2a..8c9a6a52 100644 --- a/packages/cli/src/context/ingest/memory-flow/interactive-render.test.ts +++ b/packages/cli/test/context/ingest/memory-flow/interactive-render.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; -import { createInitialMemoryFlowInteractionState, reduceMemoryFlowInteractionState } from './interaction.js'; -import { renderMemoryFlowInteractive } from './interactive-render.js'; -import type { MemoryFlowViewModel } from './types.js'; +import { createInitialMemoryFlowInteractionState, reduceMemoryFlowInteractionState } from '../../../../src/context/ingest/memory-flow/interaction.js'; +import { renderMemoryFlowInteractive } from '../../../../src/context/ingest/memory-flow/interactive-render.js'; +import type { MemoryFlowViewModel } from '../../../../src/context/ingest/memory-flow/types.js'; function view(): MemoryFlowViewModel { return { diff --git a/packages/cli/src/context/ingest/memory-flow/live-buffer.test.ts b/packages/cli/test/context/ingest/memory-flow/live-buffer.test.ts similarity index 95% rename from packages/cli/src/context/ingest/memory-flow/live-buffer.test.ts rename to packages/cli/test/context/ingest/memory-flow/live-buffer.test.ts index fc1962a3..a9c1210b 100644 --- a/packages/cli/src/context/ingest/memory-flow/live-buffer.test.ts +++ b/packages/cli/test/context/ingest/memory-flow/live-buffer.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { createMemoryFlowLiveBuffer, sanitizeMemoryFlowError } from './live-buffer.js'; -import type { MemoryFlowReplayInput } from './types.js'; +import { createMemoryFlowLiveBuffer, sanitizeMemoryFlowError } from '../../../../src/context/ingest/memory-flow/live-buffer.js'; +import type { MemoryFlowReplayInput } from '../../../../src/context/ingest/memory-flow/types.js'; function initialReplay(): MemoryFlowReplayInput { return { diff --git a/packages/cli/src/context/ingest/memory-flow/render.test.ts b/packages/cli/test/context/ingest/memory-flow/render.test.ts similarity index 95% rename from packages/cli/src/context/ingest/memory-flow/render.test.ts rename to packages/cli/test/context/ingest/memory-flow/render.test.ts index 0053eefd..adcc89f0 100644 --- a/packages/cli/src/context/ingest/memory-flow/render.test.ts +++ b/packages/cli/test/context/ingest/memory-flow/render.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import type { MemoryFlowViewModel } from './types.js'; -import { renderMemoryFlowReplay } from './render.js'; +import type { MemoryFlowViewModel } from '../../../../src/context/ingest/memory-flow/types.js'; +import { renderMemoryFlowReplay } from '../../../../src/context/ingest/memory-flow/render.js'; function view(): MemoryFlowViewModel { return { diff --git a/packages/cli/src/context/ingest/memory-flow/schema.test.ts b/packages/cli/test/context/ingest/memory-flow/schema.test.ts similarity index 88% rename from packages/cli/src/context/ingest/memory-flow/schema.test.ts rename to packages/cli/test/context/ingest/memory-flow/schema.test.ts index b8c70856..ee8f3bb9 100644 --- a/packages/cli/src/context/ingest/memory-flow/schema.test.ts +++ b/packages/cli/test/context/ingest/memory-flow/schema.test.ts @@ -3,8 +3,8 @@ import { memoryFlowReplayInputSchema, memoryFlowStreamEventSchema, parseMemoryFlowReplayInput, -} from './schema.js'; -import type { MemoryFlowReplayInput } from './types.js'; +} from '../../../../src/context/ingest/memory-flow/schema.js'; +import type { MemoryFlowReplayInput } from '../../../../src/context/ingest/memory-flow/types.js'; function snapshot(overrides: Partial = {}): MemoryFlowReplayInput { return { @@ -146,6 +146,29 @@ describe('memory-flow schemas', () => { expect(parsed.events).toContainEqual({ type: 'stage_skipped', stage: 'actions', reason: 'requires LLM' }); }); + it('accepts rate-limit wait replay events', () => { + expect( + memoryFlowReplayInputSchema.parse({ + ...snapshot(), + events: [ + { + type: 'rate_limit_wait', + provider: 'claude-subscription', + rateLimitType: 'five_hour', + resumeAtMs: 2_000, + remainingMs: 1_000, + }, + ], + }).events[0], + ).toEqual({ + type: 'rate_limit_wait', + provider: 'claude-subscription', + rateLimitType: 'five_hour', + resumeAtMs: 2_000, + remainingMs: 1_000, + }); + }); + it('parses snapshot and closed stream events', () => { expect(memoryFlowStreamEventSchema.parse({ type: 'snapshot', snapshot: snapshot({ status: 'done' }) })).toEqual({ type: 'snapshot', diff --git a/packages/cli/src/context/ingest/memory-flow/summary.test.ts b/packages/cli/test/context/ingest/memory-flow/summary.test.ts similarity index 96% rename from packages/cli/src/context/ingest/memory-flow/summary.test.ts rename to packages/cli/test/context/ingest/memory-flow/summary.test.ts index a22ca1ff..967acf46 100644 --- a/packages/cli/src/context/ingest/memory-flow/summary.test.ts +++ b/packages/cli/test/context/ingest/memory-flow/summary.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import type { MemoryFlowReplayInput } from './types.js'; -import { formatMemoryFlowFinalSummary } from './summary.js'; +import type { MemoryFlowReplayInput } from '../../../../src/context/ingest/memory-flow/types.js'; +import { formatMemoryFlowFinalSummary } from '../../../../src/context/ingest/memory-flow/summary.js'; function input(overrides: Partial = {}): MemoryFlowReplayInput { return { diff --git a/packages/cli/src/context/ingest/memory-flow/view-model.test.ts b/packages/cli/test/context/ingest/memory-flow/view-model.test.ts similarity index 98% rename from packages/cli/src/context/ingest/memory-flow/view-model.test.ts rename to packages/cli/test/context/ingest/memory-flow/view-model.test.ts index 4e6edae3..6bd64943 100644 --- a/packages/cli/src/context/ingest/memory-flow/view-model.test.ts +++ b/packages/cli/test/context/ingest/memory-flow/view-model.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import type { MemoryFlowReplayInput } from './types.js'; -import { buildMemoryFlowViewModel } from './view-model.js'; +import type { MemoryFlowReplayInput } from '../../../../src/context/ingest/memory-flow/types.js'; +import { buildMemoryFlowViewModel } from '../../../../src/context/ingest/memory-flow/view-model.js'; function replayInput(): MemoryFlowReplayInput { return { diff --git a/packages/cli/src/context/ingest/memory-flow/visuals.test.ts b/packages/cli/test/context/ingest/memory-flow/visuals.test.ts similarity index 94% rename from packages/cli/src/context/ingest/memory-flow/visuals.test.ts rename to packages/cli/test/context/ingest/memory-flow/visuals.test.ts index 7144c897..da271248 100644 --- a/packages/cli/src/context/ingest/memory-flow/visuals.test.ts +++ b/packages/cli/test/context/ingest/memory-flow/visuals.test.ts @@ -3,8 +3,8 @@ import { buildMemoryFlowVisualModel, memoryFlowStatusBadge, renderMemoryFlowConnectorLine, -} from './visuals.js'; -import type { MemoryFlowViewModel } from './types.js'; +} from '../../../../src/context/ingest/memory-flow/visuals.js'; +import type { MemoryFlowViewModel } from '../../../../src/context/ingest/memory-flow/types.js'; function viewWithStatuses(statuses: Array<'waiting' | 'active' | 'complete' | 'warning' | 'failed'>): MemoryFlowViewModel { const titles = ['SOURCE', 'CHUNKS', 'WORKUNITS', 'ACTIONS', 'GATES', 'SAVED']; diff --git a/packages/cli/src/context/ingest/page-triage/page-triage.service.test.ts b/packages/cli/test/context/ingest/page-triage/page-triage.service.test.ts similarity index 99% rename from packages/cli/src/context/ingest/page-triage/page-triage.service.test.ts rename to packages/cli/test/context/ingest/page-triage/page-triage.service.test.ts index 6432347d..33aa2979 100644 --- a/packages/cli/src/context/ingest/page-triage/page-triage.service.test.ts +++ b/packages/cli/test/context/ingest/page-triage/page-triage.service.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { PageTriageService } from './page-triage.service.js'; +import { PageTriageService } from '../../../../src/context/ingest/page-triage/page-triage.service.js'; describe('PageTriageService', () => { let stagedDir: string; diff --git a/packages/cli/src/context/ingest/raw-sources-paths.test.ts b/packages/cli/test/context/ingest/raw-sources-paths.test.ts similarity index 92% rename from packages/cli/src/context/ingest/raw-sources-paths.test.ts rename to packages/cli/test/context/ingest/raw-sources-paths.test.ts index dcc17ddc..045f479f 100644 --- a/packages/cli/src/context/ingest/raw-sources-paths.test.ts +++ b/packages/cli/test/context/ingest/raw-sources-paths.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { buildSyncId, provenanceMarker, rawSourcesDirForSync, rawSourcesRoot } from './raw-sources-paths.js'; +import { buildSyncId, provenanceMarker, rawSourcesDirForSync, rawSourcesRoot } from '../../../src/context/ingest/raw-sources-paths.js'; describe('raw-sources paths', () => { it('buildSyncId uses timestamp + jobId', () => { diff --git a/packages/cli/src/context/ingest/repo-fetch.test.ts b/packages/cli/test/context/ingest/repo-fetch.test.ts similarity index 95% rename from packages/cli/src/context/ingest/repo-fetch.test.ts rename to packages/cli/test/context/ingest/repo-fetch.test.ts index dcefd6ca..d9e66e33 100644 --- a/packages/cli/src/context/ingest/repo-fetch.test.ts +++ b/packages/cli/test/context/ingest/repo-fetch.test.ts @@ -4,10 +4,10 @@ import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { makeLocalGitRepo } from '../test/make-local-git-repo.js'; -const FIXTURE_ROOT = join(__dirname, '../../test/fixtures/lookml/single-model'); +const FIXTURE_ROOT = join(__dirname, '../../fixtures/lookml/single-model'); async function loadRepoFetch() { - return await import('./repo-fetch.js'); + return await import('../../../src/context/ingest/repo-fetch.js'); } describe('repo-fetch', () => { @@ -16,13 +16,13 @@ describe('repo-fetch', () => { beforeEach(async () => { tmpRoot = await mkdtemp(join(tmpdir(), 'repo-fetch-')); vi.resetModules(); - vi.doUnmock('./git-env.js'); + vi.doUnmock('../../../src/context/ingest/git-env.js'); }); afterEach(async () => { vi.restoreAllMocks(); vi.resetModules(); - vi.doUnmock('./git-env.js'); + vi.doUnmock('../../../src/context/ingest/git-env.js'); await rm(tmpRoot, { recursive: true, force: true }); }); @@ -98,7 +98,7 @@ describe('repo-fetch', () => { it('falls back to a fresh clone when the existing cache diverges locally', async () => { const { cloneOrPull } = await loadRepoFetch(); - const { createSimpleGit } = await import('./git-env.js'); + const { createSimpleGit } = await import('../../../src/context/ingest/git-env.js'); const repo = await makeLocalGitRepo(FIXTURE_ROOT, join(tmpRoot, 'origin')); const cacheDir = join(tmpRoot, 'cache', 'conn-diverged'); @@ -197,7 +197,7 @@ describe('repo-fetch', () => { clone: vi.fn(async () => undefined), }; - vi.doMock('./git-env.js', () => ({ + vi.doMock('../../../src/context/ingest/git-env.js', () => ({ createSimpleGit: vi.fn((baseDir?: string) => (baseDir ? worktreeGit : rootGit)), })); diff --git a/packages/cli/src/context/ingest/report-snapshot.test.ts b/packages/cli/test/context/ingest/report-snapshot.test.ts similarity index 99% rename from packages/cli/src/context/ingest/report-snapshot.test.ts rename to packages/cli/test/context/ingest/report-snapshot.test.ts index 028c222c..36f822e1 100644 --- a/packages/cli/src/context/ingest/report-snapshot.test.ts +++ b/packages/cli/test/context/ingest/report-snapshot.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { parseIngestReportSnapshot } from './report-snapshot.js'; +import { parseIngestReportSnapshot } from '../../../src/context/ingest/report-snapshot.js'; function validReportSnapshot() { return { diff --git a/packages/cli/test/context/ingest/reports.test.ts b/packages/cli/test/context/ingest/reports.test.ts new file mode 100644 index 00000000..5fc24f6d --- /dev/null +++ b/packages/cli/test/context/ingest/reports.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from 'vitest'; +import { ingestReportOutcome } from '../../../src/context/ingest/reports.js'; +import type { IngestReportSnapshot } from '../../../src/context/ingest/reports.js'; + +function report(body: Partial): IngestReportSnapshot { + return { + id: 'r', + runId: 'run', + jobId: 'job', + connectionId: 'warehouse', + sourceKey: 'metabase', + createdAt: '2026-05-29T00:00:00.000Z', + body: { + syncId: 'sync', + diffSummary: { added: 0, modified: 0, deleted: 0, unchanged: 0 }, + commitSha: null, + workUnits: [], + failedWorkUnits: [], + reconciliationSkipped: false, + conflictsResolved: [], + evictionsApplied: [], + unmappedFallbacks: [], + evictionInputs: [], + unresolvedCards: [], + supersededBy: null, + overrideOf: null, + provenanceRows: [], + toolTranscripts: [], + ...body, + }, + }; +} + +const savingWorkUnit = { + unitKey: 'ok', + rawFiles: ['cards/1.json'], + status: 'success' as const, + actions: [{ target: 'sl' as const, type: 'updated' as const, key: 'warehouse.orders', detail: 'measure' }], + touchedSlSources: [], +}; + +const failedWorkUnit = { + unitKey: 'bad', + rawFiles: ['cards/2.json'], + status: 'failed' as const, + reason: 'tool write failed', + actions: [], + touchedSlSources: [], +}; + +describe('ingestReportOutcome', () => { + it('returns done when there are no failed work units', () => { + expect(ingestReportOutcome(report({ workUnits: [savingWorkUnit] }))).toBe('done'); + }); + + it('returns partial when failed work units coexist with saved memory', () => { + expect( + ingestReportOutcome(report({ workUnits: [savingWorkUnit, failedWorkUnit], failedWorkUnits: ['bad'] })), + ).toBe('partial'); + }); + + it('returns error when failed work units produced no saved memory', () => { + expect(ingestReportOutcome(report({ workUnits: [failedWorkUnit], failedWorkUnits: ['bad'] }))).toBe('error'); + }); + + it('returns error for a stage-level failure even if artifacts were recorded', () => { + expect(ingestReportOutcome(report({ status: 'failed', workUnits: [savingWorkUnit], failedWorkUnits: [] }))).toBe( + 'error', + ); + }); +}); diff --git a/packages/cli/src/context/ingest/semantic-layer-target-policy.test.ts b/packages/cli/test/context/ingest/semantic-layer-target-policy.test.ts similarity index 95% rename from packages/cli/src/context/ingest/semantic-layer-target-policy.test.ts rename to packages/cli/test/context/ingest/semantic-layer-target-policy.test.ts index 73d09dc0..5f0980f6 100644 --- a/packages/cli/src/context/ingest/semantic-layer-target-policy.test.ts +++ b/packages/cli/test/context/ingest/semantic-layer-target-policy.test.ts @@ -3,7 +3,7 @@ import { assertSemanticLayerTargetPathsAllowed, findDisallowedSemanticLayerTargetPaths, semanticLayerConnectionIdFromPath, -} from './semantic-layer-target-policy.js'; +} from '../../../src/context/ingest/semantic-layer-target-policy.js'; describe('semantic-layer target policy', () => { it('extracts connection ids from semantic-layer paths', () => { diff --git a/packages/cli/src/context/ingest/source-adapter-registry.test.ts b/packages/cli/test/context/ingest/source-adapter-registry.test.ts similarity index 87% rename from packages/cli/src/context/ingest/source-adapter-registry.test.ts rename to packages/cli/test/context/ingest/source-adapter-registry.test.ts index 9a74c597..abd34f51 100644 --- a/packages/cli/src/context/ingest/source-adapter-registry.test.ts +++ b/packages/cli/test/context/ingest/source-adapter-registry.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { SourceAdapterRegistry } from './source-adapter-registry.js'; -import type { SourceAdapter } from './types.js'; +import { SourceAdapterRegistry } from '../../../src/context/ingest/source-adapter-registry.js'; +import type { SourceAdapter } from '../../../src/context/ingest/types.js'; const makeAdapter = (source: string): SourceAdapter => ({ source, diff --git a/packages/cli/src/context/ingest/sqlite-bundle-ingest-store.test.ts b/packages/cli/test/context/ingest/sqlite-bundle-ingest-store.test.ts similarity index 98% rename from packages/cli/src/context/ingest/sqlite-bundle-ingest-store.test.ts rename to packages/cli/test/context/ingest/sqlite-bundle-ingest-store.test.ts index 0cee47d0..9f6e9f9f 100644 --- a/packages/cli/src/context/ingest/sqlite-bundle-ingest-store.test.ts +++ b/packages/cli/test/context/ingest/sqlite-bundle-ingest-store.test.ts @@ -2,10 +2,10 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { DiffSetService } from './diff-set.service.js'; -import type { IngestDiffSummary, IngestTrigger } from '../../context/ingest/types.js'; -import type { IngestReportBody } from '../../context/ingest/reports.js'; -import { SqliteBundleIngestStore } from './sqlite-bundle-ingest-store.js'; +import { DiffSetService } from '../../../src/context/ingest/diff-set.service.js'; +import type { IngestDiffSummary, IngestTrigger } from '../../../src/context/ingest/types.js'; +import type { IngestReportBody } from '../../../src/context/ingest/reports.js'; +import { SqliteBundleIngestStore } from '../../../src/context/ingest/sqlite-bundle-ingest-store.js'; function idFactory(ids: string[]): () => string { let index = 0; diff --git a/packages/cli/src/context/ingest/sqlite-local-ingest-store.test.ts b/packages/cli/test/context/ingest/sqlite-local-ingest-store.test.ts similarity index 95% rename from packages/cli/src/context/ingest/sqlite-local-ingest-store.test.ts rename to packages/cli/test/context/ingest/sqlite-local-ingest-store.test.ts index 67fad006..3d38fe58 100644 --- a/packages/cli/src/context/ingest/sqlite-local-ingest-store.test.ts +++ b/packages/cli/test/context/ingest/sqlite-local-ingest-store.test.ts @@ -2,8 +2,8 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { SqliteLocalIngestStore } from './sqlite-local-ingest-store.js'; -import type { LocalIngestRunRecord } from './local-stage-ingest.js'; +import { SqliteLocalIngestStore } from '../../../src/context/ingest/sqlite-local-ingest-store.js'; +import type { LocalIngestRunRecord } from '../../../src/context/ingest/local-stage-ingest.js'; function runRecord(overrides: Partial = {}): LocalIngestRunRecord { return { diff --git a/packages/cli/src/context/ingest/stages/build-reconcile-context.context-candidates.test.ts b/packages/cli/test/context/ingest/stages/build-reconcile-context.context-candidates.test.ts similarity index 97% rename from packages/cli/src/context/ingest/stages/build-reconcile-context.context-candidates.test.ts rename to packages/cli/test/context/ingest/stages/build-reconcile-context.context-candidates.test.ts index 22427ddd..09a1c610 100644 --- a/packages/cli/src/context/ingest/stages/build-reconcile-context.context-candidates.test.ts +++ b/packages/cli/test/context/ingest/stages/build-reconcile-context.context-candidates.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { buildReconcileUserPrompt } from './build-reconcile-context.js'; +import { buildReconcileUserPrompt } from '../../../../src/context/ingest/stages/build-reconcile-context.js'; const emptyStageIndex = { jobId: 'job-1', diff --git a/packages/cli/src/context/ingest/stages/build-reconcile-context.test.ts b/packages/cli/test/context/ingest/stages/build-reconcile-context.test.ts similarity index 98% rename from packages/cli/src/context/ingest/stages/build-reconcile-context.test.ts rename to packages/cli/test/context/ingest/stages/build-reconcile-context.test.ts index 8de7611a..2aaeb061 100644 --- a/packages/cli/src/context/ingest/stages/build-reconcile-context.test.ts +++ b/packages/cli/test/context/ingest/stages/build-reconcile-context.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { buildReconcileSystemPrompt, buildReconcileToolSet, buildReconcileUserPrompt } from './build-reconcile-context.js'; +import { buildReconcileSystemPrompt, buildReconcileToolSet, buildReconcileUserPrompt } from '../../../../src/context/ingest/stages/build-reconcile-context.js'; describe('buildReconcileSystemPrompt', () => { it('appends canonical pins when relevant pins are supplied', () => { diff --git a/packages/cli/src/context/ingest/stages/build-wu-context.test.ts b/packages/cli/test/context/ingest/stages/build-wu-context.test.ts similarity index 99% rename from packages/cli/src/context/ingest/stages/build-wu-context.test.ts rename to packages/cli/test/context/ingest/stages/build-wu-context.test.ts index 81c0c923..cdbb22ba 100644 --- a/packages/cli/src/context/ingest/stages/build-wu-context.test.ts +++ b/packages/cli/test/context/ingest/stages/build-wu-context.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { buildWuSystemPrompt, buildWuToolSet, buildWuUserPrompt } from './build-wu-context.js'; +import { buildWuSystemPrompt, buildWuToolSet, buildWuUserPrompt } from '../../../../src/context/ingest/stages/build-wu-context.js'; describe('buildWuUserPrompt', () => { it('includes rawFiles, dependencyPaths, peerFileIndex, and priorProvenance when present', () => { diff --git a/packages/cli/src/context/ingest/stages/stage-1-stage-raw-files.test.ts b/packages/cli/test/context/ingest/stages/stage-1-stage-raw-files.test.ts similarity index 95% rename from packages/cli/src/context/ingest/stages/stage-1-stage-raw-files.test.ts rename to packages/cli/test/context/ingest/stages/stage-1-stage-raw-files.test.ts index 3cd5cde5..d943d2d0 100644 --- a/packages/cli/src/context/ingest/stages/stage-1-stage-raw-files.test.ts +++ b/packages/cli/test/context/ingest/stages/stage-1-stage-raw-files.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { stageRawFilesStage1 } from './stage-1-stage-raw-files.js'; +import { stageRawFilesStage1 } from '../../../../src/context/ingest/stages/stage-1-stage-raw-files.js'; describe('Stage 1 — stageRawFiles', () => { let stagedDir: string; diff --git a/packages/cli/src/context/ingest/stages/stage-3-work-units.test.ts b/packages/cli/test/context/ingest/stages/stage-3-work-units.test.ts similarity index 96% rename from packages/cli/src/context/ingest/stages/stage-3-work-units.test.ts rename to packages/cli/test/context/ingest/stages/stage-3-work-units.test.ts index fc39fd9b..6d6deccd 100644 --- a/packages/cli/src/context/ingest/stages/stage-3-work-units.test.ts +++ b/packages/cli/test/context/ingest/stages/stage-3-work-units.test.ts @@ -1,8 +1,8 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import type { CaptureSession, MemoryAction } from '../../../context/memory/types.js'; -import { addTouchedSlSource, createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; -import type { WorkUnit } from '../types.js'; -import { executeWorkUnit, type WorkUnitExecutionDeps } from './stage-3-work-units.js'; +import type { CaptureSession, MemoryAction } from '../../../../src/context/memory/types.js'; +import { addTouchedSlSource, createTouchedSlSources } from '../../../../src/context/tools/touched-sl-sources.js'; +import type { WorkUnit } from '../../../../src/context/ingest/types.js'; +import { executeWorkUnit, type WorkUnitExecutionDeps } from '../../../../src/context/ingest/stages/stage-3-work-units.js'; const makeWu = (overrides: Partial = {}): WorkUnit => ({ unitKey: 'u1', diff --git a/packages/cli/src/context/ingest/stages/stage-4-reconciliation.test.ts b/packages/cli/test/context/ingest/stages/stage-4-reconciliation.test.ts similarity index 97% rename from packages/cli/src/context/ingest/stages/stage-4-reconciliation.test.ts rename to packages/cli/test/context/ingest/stages/stage-4-reconciliation.test.ts index 4244ab12..d66533a9 100644 --- a/packages/cli/src/context/ingest/stages/stage-4-reconciliation.test.ts +++ b/packages/cli/test/context/ingest/stages/stage-4-reconciliation.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { runReconciliationStage4 } from './stage-4-reconciliation.js'; +import { runReconciliationStage4 } from '../../../../src/context/ingest/stages/stage-4-reconciliation.js'; describe('Stage 4 — runReconciliationStage4', () => { it('short-circuits when stage index is empty and eviction is empty', async () => { diff --git a/packages/cli/src/context/ingest/stages/validate-wu-sources.test.ts b/packages/cli/test/context/ingest/stages/validate-wu-sources.test.ts similarity index 93% rename from packages/cli/src/context/ingest/stages/validate-wu-sources.test.ts rename to packages/cli/test/context/ingest/stages/validate-wu-sources.test.ts index 668062e9..807a8b10 100644 --- a/packages/cli/src/context/ingest/stages/validate-wu-sources.test.ts +++ b/packages/cli/test/context/ingest/stages/validate-wu-sources.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { validateWuTouchedSources } from './validate-wu-sources.js'; +import { validateWuTouchedSources } from '../../../../src/context/ingest/stages/validate-wu-sources.js'; describe('validateWuTouchedSources', () => { it('validates each touched source against its own connection', async () => { diff --git a/packages/cli/src/context/ingest/tools/emit-reconciliation-records.tool.test.ts b/packages/cli/test/context/ingest/tools/emit-reconciliation-records.tool.test.ts similarity index 93% rename from packages/cli/src/context/ingest/tools/emit-reconciliation-records.tool.test.ts rename to packages/cli/test/context/ingest/tools/emit-reconciliation-records.tool.test.ts index 1cd77514..7801038b 100644 --- a/packages/cli/src/context/ingest/tools/emit-reconciliation-records.tool.test.ts +++ b/packages/cli/test/context/ingest/tools/emit-reconciliation-records.tool.test.ts @@ -1,10 +1,10 @@ import type { Tool } from 'ai'; import { describe, expect, it } from 'vitest'; -import type { StageIndex } from '../stages/stage-index.types.js'; -import { createEmitArtifactResolutionTool } from './emit-artifact-resolution.tool.js'; -import { createEmitConflictResolutionTool } from './emit-conflict-resolution.tool.js'; -import { createEmitEvictionDecisionTool } from './emit-eviction-decision.tool.js'; -import { createEmitUnmappedFallbackTool } from './emit-unmapped-fallback.tool.js'; +import type { StageIndex } from '../../../../src/context/ingest/stages/stage-index.types.js'; +import { createEmitArtifactResolutionTool } from '../../../../src/context/ingest/tools/emit-artifact-resolution.tool.js'; +import { createEmitConflictResolutionTool } from '../../../../src/context/ingest/tools/emit-conflict-resolution.tool.js'; +import { createEmitEvictionDecisionTool } from '../../../../src/context/ingest/tools/emit-eviction-decision.tool.js'; +import { createEmitUnmappedFallbackTool } from '../../../../src/context/ingest/tools/emit-unmapped-fallback.tool.js'; function makeStageIndex(): StageIndex { return { diff --git a/packages/cli/src/context/ingest/tools/eviction-list.tool.test.ts b/packages/cli/test/context/ingest/tools/eviction-list.tool.test.ts similarity index 94% rename from packages/cli/src/context/ingest/tools/eviction-list.tool.test.ts rename to packages/cli/test/context/ingest/tools/eviction-list.tool.test.ts index 96fb7f65..3dcfd005 100644 --- a/packages/cli/src/context/ingest/tools/eviction-list.tool.test.ts +++ b/packages/cli/test/context/ingest/tools/eviction-list.tool.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { createEvictionListTool } from './eviction-list.tool.js'; +import { createEvictionListTool } from '../../../../src/context/ingest/tools/eviction-list.tool.js'; describe('eviction_list tool', () => { it('returns artifacts produced for each deleted raw path', async () => { diff --git a/packages/cli/src/context/ingest/tools/read-raw-file.tool.test.ts b/packages/cli/test/context/ingest/tools/read-raw-file.tool.test.ts similarity index 96% rename from packages/cli/src/context/ingest/tools/read-raw-file.tool.test.ts rename to packages/cli/test/context/ingest/tools/read-raw-file.tool.test.ts index db4aef42..041ea188 100644 --- a/packages/cli/src/context/ingest/tools/read-raw-file.tool.test.ts +++ b/packages/cli/test/context/ingest/tools/read-raw-file.tool.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { createReadRawFileTool } from './read-raw-file.tool.js'; +import { createReadRawFileTool } from '../../../../src/context/ingest/tools/read-raw-file.tool.js'; describe('read_raw_file tool', () => { let stagedDir: string; diff --git a/packages/cli/src/context/ingest/tools/read-raw-span.tool.test.ts b/packages/cli/test/context/ingest/tools/read-raw-span.tool.test.ts similarity index 95% rename from packages/cli/src/context/ingest/tools/read-raw-span.tool.test.ts rename to packages/cli/test/context/ingest/tools/read-raw-span.tool.test.ts index 30696046..cd4e8f2a 100644 --- a/packages/cli/src/context/ingest/tools/read-raw-span.tool.test.ts +++ b/packages/cli/test/context/ingest/tools/read-raw-span.tool.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { createReadRawSpanTool } from './read-raw-span.tool.js'; +import { createReadRawSpanTool } from '../../../../src/context/ingest/tools/read-raw-span.tool.js'; describe('read_raw_span tool', () => { let stagedDir: string; diff --git a/packages/cli/src/context/ingest/tools/stage-diff.tool.test.ts b/packages/cli/test/context/ingest/tools/stage-diff.tool.test.ts similarity index 97% rename from packages/cli/src/context/ingest/tools/stage-diff.tool.test.ts rename to packages/cli/test/context/ingest/tools/stage-diff.tool.test.ts index 0dae87ab..fd756099 100644 --- a/packages/cli/src/context/ingest/tools/stage-diff.tool.test.ts +++ b/packages/cli/test/context/ingest/tools/stage-diff.tool.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { createStageDiffTool } from './stage-diff.tool.js'; +import { createStageDiffTool } from '../../../../src/context/ingest/tools/stage-diff.tool.js'; describe('stage_diff tool', () => { const stageIndex = { diff --git a/packages/cli/src/context/ingest/tools/stage-list.tool.test.ts b/packages/cli/test/context/ingest/tools/stage-list.tool.test.ts similarity index 95% rename from packages/cli/src/context/ingest/tools/stage-list.tool.test.ts rename to packages/cli/test/context/ingest/tools/stage-list.tool.test.ts index d14acb2a..3ed975bc 100644 --- a/packages/cli/src/context/ingest/tools/stage-list.tool.test.ts +++ b/packages/cli/test/context/ingest/tools/stage-list.tool.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { createStageListTool } from './stage-list.tool.js'; +import { createStageListTool } from '../../../../src/context/ingest/tools/stage-list.tool.js'; describe('stage_list tool', () => { it('returns a compact summary of the stage index', async () => { diff --git a/packages/cli/test/context/ingest/tools/tool-call-logger.test.ts b/packages/cli/test/context/ingest/tools/tool-call-logger.test.ts new file mode 100644 index 00000000..be534de6 --- /dev/null +++ b/packages/cli/test/context/ingest/tools/tool-call-logger.test.ts @@ -0,0 +1,54 @@ +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { flushToolCallLogs, wrapToolsWithLogger } from '../../../../src/context/ingest/tools/tool-call-logger.js'; + +describe('wrapToolsWithLogger + flushToolCallLogs', () => { + const dirs: string[] = []; + afterEach(async () => { + for (const dir of dirs.splice(0)) { + await rm(dir, { recursive: true, force: true }); + } + }); + + function toolset() { + return { + my_tool: { + name: 'my_tool', + description: 'test tool', + inputSchema: z.object({}), + execute: async (_input: unknown) => ({ markdown: 'ok' }), + }, + }; + } + + it('makes the fire-and-forget transcript write observable after a flush', async () => { + const dir = await mkdtemp(join(tmpdir(), 'ktx-toollog-')); + dirs.push(dir); + const logPath = join(dir, 'wu.jsonl'); + const wrapped = wrapToolsWithLogger(toolset(), logPath, 'cards/users'); + + await wrapped.my_tool.execute({}); + // The append is fire-and-forget; flushing must guarantee it has landed. + await flushToolCallLogs(); + + const entry = JSON.parse((await readFile(logPath, 'utf-8')).trim()); + expect(entry.wuKey).toBe('cards/users'); + expect(entry.toolName).toBe('my_tool'); + expect(typeof entry.durationMs).toBe('number'); + }); + + it('resolves immediately when there is nothing to flush', async () => { + await expect(flushToolCallLogs()).resolves.toBeUndefined(); + }); + + it('is bounded by its timeout and never rejects', async () => { + const dir = await mkdtemp(join(tmpdir(), 'ktx-toollog-')); + dirs.push(dir); + const wrapped = wrapToolsWithLogger(toolset(), join(dir, 'wu.jsonl'), 'wu/1'); + await wrapped.my_tool.execute({}); + await expect(flushToolCallLogs(0)).resolves.toBeUndefined(); + }); +}); diff --git a/packages/cli/src/context/ingest/tools/tool-transcript-summary.test.ts b/packages/cli/test/context/ingest/tools/tool-transcript-summary.test.ts similarity index 97% rename from packages/cli/src/context/ingest/tools/tool-transcript-summary.test.ts rename to packages/cli/test/context/ingest/tools/tool-transcript-summary.test.ts index 9e110789..ddc8d256 100644 --- a/packages/cli/src/context/ingest/tools/tool-transcript-summary.test.ts +++ b/packages/cli/test/context/ingest/tools/tool-transcript-summary.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import type { ToolCallLogEntry } from './tool-call-logger.js'; -import { createMutableToolTranscriptSummary, recordToolTranscriptEntry } from './tool-transcript-summary.js'; +import type { ToolCallLogEntry } from '../../../../src/context/ingest/tools/tool-call-logger.js'; +import { createMutableToolTranscriptSummary, recordToolTranscriptEntry } from '../../../../src/context/ingest/tools/tool-transcript-summary.js'; function entry(overrides: Partial): ToolCallLogEntry { return { diff --git a/packages/cli/src/context/ingest/tools/warehouse-verification/discover-data.tool.test.ts b/packages/cli/test/context/ingest/tools/warehouse-verification/discover-data.tool.test.ts similarity index 94% rename from packages/cli/src/context/ingest/tools/warehouse-verification/discover-data.tool.test.ts rename to packages/cli/test/context/ingest/tools/warehouse-verification/discover-data.tool.test.ts index 7aebc101..09abbc6a 100644 --- a/packages/cli/src/context/ingest/tools/warehouse-verification/discover-data.tool.test.ts +++ b/packages/cli/test/context/ingest/tools/warehouse-verification/discover-data.tool.test.ts @@ -1,7 +1,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import type { WarehouseCatalogService } from '../../../scan/warehouse-catalog.js'; -import type { BaseTool, ToolContext } from '../../../../context/tools/base-tool.js'; -import { DiscoverDataTool } from './discover-data.tool.js'; +import type { WarehouseCatalogService } from '../../../../../src/context/scan/warehouse-catalog.js'; +import type { BaseTool, ToolContext } from '../../../../../src/context/tools/base-tool.js'; +import { DiscoverDataTool } from '../../../../../src/context/ingest/tools/warehouse-verification/discover-data.tool.js'; describe('DiscoverDataTool', () => { const wikiSearchTool = { call: vi.fn() } as unknown as BaseTool & { call: ReturnType }; diff --git a/packages/cli/src/context/ingest/tools/warehouse-verification/entity-details.tool.test.ts b/packages/cli/test/context/ingest/tools/warehouse-verification/entity-details.tool.test.ts similarity index 95% rename from packages/cli/src/context/ingest/tools/warehouse-verification/entity-details.tool.test.ts rename to packages/cli/test/context/ingest/tools/warehouse-verification/entity-details.tool.test.ts index fcef38df..0107951d 100644 --- a/packages/cli/src/context/ingest/tools/warehouse-verification/entity-details.tool.test.ts +++ b/packages/cli/test/context/ingest/tools/warehouse-verification/entity-details.tool.test.ts @@ -2,10 +2,10 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../../../../context/project/project.js'; -import { WarehouseCatalogService } from '../../../scan/warehouse-catalog.js'; -import type { ToolContext } from '../../../../context/tools/base-tool.js'; -import { EntityDetailsTool } from './entity-details.tool.js'; +import { initKtxProject, type KtxLocalProject } from '../../../../../src/context/project/project.js'; +import { WarehouseCatalogService } from '../../../../../src/context/scan/warehouse-catalog.js'; +import type { ToolContext } from '../../../../../src/context/tools/base-tool.js'; +import { EntityDetailsTool } from '../../../../../src/context/ingest/tools/warehouse-verification/entity-details.tool.js'; describe('EntityDetailsTool', () => { let tempDir: string; diff --git a/packages/cli/src/context/ingest/tools/warehouse-verification/sql-execution.tool.test.ts b/packages/cli/test/context/ingest/tools/warehouse-verification/sql-execution.tool.test.ts similarity index 89% rename from packages/cli/src/context/ingest/tools/warehouse-verification/sql-execution.tool.test.ts rename to packages/cli/test/context/ingest/tools/warehouse-verification/sql-execution.tool.test.ts index 4458471a..0fb87655 100644 --- a/packages/cli/src/context/ingest/tools/warehouse-verification/sql-execution.tool.test.ts +++ b/packages/cli/test/context/ingest/tools/warehouse-verification/sql-execution.tool.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { SlConnectionCatalogPort } from '../../../../context/sl/ports.js'; -import type { ToolContext } from '../../../../context/tools/base-tool.js'; -import { SqlExecutionTool } from './sql-execution.tool.js'; +import type { SlConnectionCatalogPort } from '../../../../../src/context/sl/ports.js'; +import type { ToolContext } from '../../../../../src/context/tools/base-tool.js'; +import { SqlExecutionTool } from '../../../../../src/context/ingest/tools/warehouse-verification/sql-execution.tool.js'; describe('SqlExecutionTool', () => { const connections = { diff --git a/packages/cli/src/context/ingest/wiki-body-refs.test.ts b/packages/cli/test/context/ingest/wiki-body-refs.test.ts similarity index 98% rename from packages/cli/src/context/ingest/wiki-body-refs.test.ts rename to packages/cli/test/context/ingest/wiki-body-refs.test.ts index 2af8935f..578dc600 100644 --- a/packages/cli/src/context/ingest/wiki-body-refs.test.ts +++ b/packages/cli/test/context/ingest/wiki-body-refs.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { findInvalidWikiBodyRefs, parseWikiBodyRefs } from './wiki-body-refs.js'; +import { findInvalidWikiBodyRefs, parseWikiBodyRefs } from '../../../src/context/ingest/wiki-body-refs.js'; const sources = [ { diff --git a/packages/cli/src/context/ingest/wiki-sl-ref-repair.test.ts b/packages/cli/test/context/ingest/wiki-sl-ref-repair.test.ts similarity index 97% rename from packages/cli/src/context/ingest/wiki-sl-ref-repair.test.ts rename to packages/cli/test/context/ingest/wiki-sl-ref-repair.test.ts index bcf4a993..16f0acb5 100644 --- a/packages/cli/src/context/ingest/wiki-sl-ref-repair.test.ts +++ b/packages/cli/test/context/ingest/wiki-sl-ref-repair.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { repairWikiSlRefs } from './wiki-sl-ref-repair.js'; +import { repairWikiSlRefs } from '../../../src/context/ingest/wiki-sl-ref-repair.js'; describe('repairWikiSlRefs', () => { it('removes missing measure refs while keeping source, measure, segment, and manifest-backed refs', async () => { diff --git a/packages/cli/src/context/llm/ai-sdk-runtime.test.ts b/packages/cli/test/context/llm/ai-sdk-runtime.test.ts similarity index 57% rename from packages/cli/src/context/llm/ai-sdk-runtime.test.ts rename to packages/cli/test/context/llm/ai-sdk-runtime.test.ts index ba5e286d..bab7d1d7 100644 --- a/packages/cli/src/context/llm/ai-sdk-runtime.test.ts +++ b/packages/cli/test/context/llm/ai-sdk-runtime.test.ts @@ -7,8 +7,8 @@ vi.mock('ai', () => ({ })); import { generateText } from 'ai'; -import { AiSdkKtxLlmRuntime } from './ai-sdk-runtime.js'; -import type { RunLoopStepInfo } from './runtime-port.js'; +import { AiSdkKtxLlmRuntime } from '../../../src/context/llm/ai-sdk-runtime.js'; +import type { RunLoopStepInfo } from '../../../src/context/llm/runtime-port.js'; describe('AiSdkKtxLlmRuntime.runAgentLoop', () => { let runtime: AiSdkKtxLlmRuntime; @@ -107,6 +107,266 @@ describe('AiSdkKtxLlmRuntime.runAgentLoop', () => { expect(result.error).toBe(err); }); + it('reports AI SDK retry-after rate limits and retries through the governor', async () => { + const waitForReady = vi.fn().mockResolvedValue(undefined); + const report = vi.fn(); + const rateLimitError = Object.assign(new Error('too many requests'), { + name: 'TooManyRequestsError', + retryAfter: 2, + statusCode: 429, + }); + (generateText as any).mockRejectedValueOnce(rateLimitError).mockResolvedValueOnce({ + text: 'done', + toolCalls: [], + steps: [], + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + }); + const runtime = new AiSdkKtxLlmRuntime({ + llmProvider: llmProvider as any, + rateLimitGovernor: { waitForReady, report, maxRetryAttempts: () => 6 } as never, + }); + + const result = await runtime.runAgentLoop({ + modelRole: 'candidateExtraction', + systemPrompt: '', + userPrompt: '', + toolSet: {}, + stepBudget: 10, + telemetryTags: {}, + }); + + expect(result.stopReason).toBe('natural'); + expect(report).toHaveBeenCalledWith({ + provider: 'anthropic-api', + status: 'rejected', + retryAfterMs: 2_000, + rateLimitType: 'http_429', + }); + expect(waitForReady).toHaveBeenCalledTimes(2); + expect(generateText).toHaveBeenCalledTimes(2); + }); + + it('does not retry AI SDK rate limits without a governor', async () => { + const rateLimitError = Object.assign(new Error('too many requests'), { + name: 'TooManyRequestsError', + statusCode: 429, + }); + (generateText as any).mockRejectedValue(rateLimitError); + // The beforeEach runtime is constructed without a rateLimitGovernor. + + const result = await runtime.runAgentLoop({ + modelRole: 'candidateExtraction', + systemPrompt: '', + userPrompt: '', + toolSet: {}, + stepBudget: 10, + telemetryTags: {}, + }); + + expect(result.stopReason).toBe('error'); + expect(generateText).toHaveBeenCalledTimes(1); + }); + + it('honors a governor retry budget of one attempt without retrying', async () => { + const waitForReady = vi.fn().mockResolvedValue(undefined); + const report = vi.fn(); + const rateLimitError = Object.assign(new Error('too many requests'), { + name: 'TooManyRequestsError', + statusCode: 429, + }); + (generateText as any).mockRejectedValue(rateLimitError); + const runtime = new AiSdkKtxLlmRuntime({ + llmProvider: llmProvider as any, + rateLimitGovernor: { waitForReady, report, maxRetryAttempts: () => 1 } as never, + }); + + const result = await runtime.runAgentLoop({ + modelRole: 'candidateExtraction', + systemPrompt: '', + userPrompt: '', + toolSet: {}, + stepBudget: 10, + telemetryTags: {}, + }); + + expect(result.stopReason).toBe('error'); + expect(generateText).toHaveBeenCalledTimes(1); + expect(report).not.toHaveBeenCalled(); + }); + + it('reports Anthropic API response-header utilization to the governor', async () => { + const waitForReady = vi.fn().mockResolvedValue(undefined); + const report = vi.fn(); + (generateText as any).mockResolvedValue({ + text: 'done', + toolCalls: [], + steps: [], + response: { + headers: { + 'anthropic-ratelimit-requests-limit': '100', + 'anthropic-ratelimit-requests-remaining': '8', + 'anthropic-ratelimit-input-tokens-limit': '10000', + 'anthropic-ratelimit-input-tokens-remaining': '9000', + }, + }, + }); + const runtime = new AiSdkKtxLlmRuntime({ + llmProvider: llmProvider as any, + rateLimitGovernor: { waitForReady, report, maxRetryAttempts: () => 6 } as never, + }); + + const result = await runtime.runAgentLoop({ + modelRole: 'candidateExtraction', + systemPrompt: '', + userPrompt: '', + toolSet: {}, + stepBudget: 10, + telemetryTags: {}, + }); + + expect(result.stopReason).toBe('natural'); + expect(report).toHaveBeenCalledWith({ + provider: 'anthropic-api', + status: 'allowed', + rateLimitType: 'rpm', + utilization: 0.92, + }); + }); + + it('reports generic x-ratelimit response-header utilization for Vertex providers', async () => { + const waitForReady = vi.fn().mockResolvedValue(undefined); + const report = vi.fn(); + const vertexProvider = { + ...llmProvider, + getModel: vi.fn().mockReturnValue({ modelId: 'gemini-3-pro', provider: 'google-vertex' }), + }; + (generateText as any).mockResolvedValue({ + text: 'done', + toolCalls: [], + steps: [], + response: { + headers: { + 'x-ratelimit-limit-requests': '200', + 'x-ratelimit-remaining-requests': '30', + 'x-ratelimit-limit-tokens': '100000', + 'x-ratelimit-remaining-tokens': '4000', + }, + }, + }); + const runtime = new AiSdkKtxLlmRuntime({ + llmProvider: vertexProvider as any, + rateLimitGovernor: { waitForReady, report, maxRetryAttempts: () => 6 } as never, + }); + + const result = await runtime.runAgentLoop({ + modelRole: 'candidateExtraction', + systemPrompt: '', + userPrompt: '', + toolSet: {}, + stepBudget: 10, + telemetryTags: {}, + }); + + expect(result.stopReason).toBe('natural'); + expect(report).toHaveBeenCalledWith({ + provider: 'vertex', + status: 'allowed', + rateLimitType: 'tpm', + utilization: 0.96, + }); + }); + + it('passes abort signals into governor waits and AI SDK generateText calls', async () => { + const controller = new AbortController(); + const waitForReady = vi.fn().mockResolvedValue(undefined); + (generateText as any).mockResolvedValue({ text: 'done', toolCalls: [], steps: [] }); + const runtime = new AiSdkKtxLlmRuntime({ + llmProvider: llmProvider as any, + rateLimitGovernor: { waitForReady, report: vi.fn(), maxRetryAttempts: () => 6 } as never, + }); + + const result = await runtime.runAgentLoop({ + modelRole: 'candidateExtraction', + systemPrompt: '', + userPrompt: '', + toolSet: {}, + stepBudget: 10, + telemetryTags: {}, + abortSignal: controller.signal, + }); + + expect(result.stopReason).toBe('natural'); + expect(waitForReady).toHaveBeenCalledWith(controller.signal); + expect((generateText as any).mock.calls[0][0].abortSignal).toBe(controller.signal); + }); + + it('returns metrics with stepCount, per-step boundaries, and aggregate token usage', async () => { + (generateText as any).mockImplementation(async (opts: any) => { + await opts.onStepFinish({}); + await opts.onStepFinish({}); + return { + text: 'ok', + toolCalls: [], + steps: [], + totalUsage: { inputTokens: 100, outputTokens: 20, totalTokens: 120 }, + }; + }); + + const result = await runtime.runAgentLoop({ + modelRole: 'candidateExtraction', + systemPrompt: '', + userPrompt: '', + toolSet: {}, + stepBudget: 10, + telemetryTags: {}, + }); + + expect(result.metrics).toBeDefined(); + expect(result.metrics?.stepCount).toBe(2); + expect(result.metrics?.stepBoundariesMs).toHaveLength(2); + expect(result.metrics?.totalMs).toBeGreaterThanOrEqual(0); + expect(result.metrics?.usage).toEqual({ inputTokens: 100, outputTokens: 20, totalTokens: 120 }); + }); + + it('falls back to result.usage when totalUsage is absent', async () => { + (generateText as any).mockResolvedValue({ + text: 'ok', + toolCalls: [], + steps: [], + usage: { inputTokens: 7, outputTokens: 3, totalTokens: 10 }, + }); + + const result = await runtime.runAgentLoop({ + modelRole: 'candidateExtraction', + systemPrompt: '', + userPrompt: '', + toolSet: {}, + stepBudget: 10, + telemetryTags: {}, + }); + + expect(result.metrics?.usage).toEqual({ inputTokens: 7, outputTokens: 3, totalTokens: 10 }); + expect(result.metrics?.stepCount).toBe(0); + }); + + it('returns partial metrics even when the loop errors', async () => { + (generateText as any).mockRejectedValue(new Error('boom')); + + const result = await runtime.runAgentLoop({ + modelRole: 'candidateExtraction', + systemPrompt: '', + userPrompt: '', + toolSet: {}, + stepBudget: 10, + telemetryTags: {}, + }); + + expect(result.stopReason).toBe('error'); + expect(result.metrics).toBeDefined(); + expect(result.metrics?.stepCount).toBe(0); + expect(result.metrics?.usage).toEqual({}); + }); + it('invokes caller onStepFinish with incrementing stepIndex and total budget', async () => { const calls: RunLoopStepInfo[] = []; (generateText as any).mockImplementation(async (opts: any) => { diff --git a/packages/cli/src/context/llm/claude-code-env.test.ts b/packages/cli/test/context/llm/claude-code-env.test.ts similarity index 92% rename from packages/cli/src/context/llm/claude-code-env.test.ts rename to packages/cli/test/context/llm/claude-code-env.test.ts index 19cbd1ff..9f015563 100644 --- a/packages/cli/src/context/llm/claude-code-env.test.ts +++ b/packages/cli/test/context/llm/claude-code-env.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { CLAUDE_CODE_PROVIDER_ENV_DENYLIST, createKtxClaudeCodeEnv } from './claude-code-env.js'; +import { CLAUDE_CODE_PROVIDER_ENV_DENYLIST, createKtxClaudeCodeEnv } from '../../../src/context/llm/claude-code-env.js'; describe('createKtxClaudeCodeEnv', () => { it('strips provider-routing credentials from the Claude Code child environment', () => { diff --git a/packages/cli/src/context/llm/claude-code-models.test.ts b/packages/cli/test/context/llm/claude-code-models.test.ts similarity index 85% rename from packages/cli/src/context/llm/claude-code-models.test.ts rename to packages/cli/test/context/llm/claude-code-models.test.ts index 482e6af8..34de9233 100644 --- a/packages/cli/src/context/llm/claude-code-models.test.ts +++ b/packages/cli/test/context/llm/claude-code-models.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { resolveClaudeCodeModel } from './claude-code-models.js'; +import { resolveClaudeCodeModel } from '../../../src/context/llm/claude-code-models.js'; describe('resolveClaudeCodeModel', () => { it.each([ diff --git a/packages/cli/src/context/llm/claude-code-runtime.test.ts b/packages/cli/test/context/llm/claude-code-runtime.test.ts similarity index 59% rename from packages/cli/src/context/llm/claude-code-runtime.test.ts rename to packages/cli/test/context/llm/claude-code-runtime.test.ts index b1003b78..ba83cde6 100644 --- a/packages/cli/src/context/llm/claude-code-runtime.test.ts +++ b/packages/cli/test/context/llm/claude-code-runtime.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; import { z } from 'zod'; import type { SDKMessage } from '@anthropic-ai/claude-agent-sdk'; -import { ClaudeCodeKtxLlmRuntime, mapClaudeCodeStopReason, runClaudeCodeAuthProbe } from './claude-code-runtime.js'; +import { ClaudeCodeKtxLlmRuntime, mapClaudeCodeStopReason, runClaudeCodeAuthProbe } from '../../../src/context/llm/claude-code-runtime.js'; async function* stream(messages: SDKMessage[]): AsyncGenerator { for (const message of messages) { @@ -9,6 +9,14 @@ async function* stream(messages: SDKMessage[]): AsyncGenerator } } +function deferred() { + let resolve!: (value: T | PromiseLike) => void; + const promise = new Promise((innerResolve) => { + resolve = innerResolve; + }); + return { promise, resolve }; +} + function initMessage(overrides: Partial> = {}): Extract< SDKMessage, { type: 'system'; subtype: 'init' } @@ -91,6 +99,247 @@ describe('ClaudeCodeKtxLlmRuntime', () => { }); }); + it('waits before Claude Code text generation and reports rate-limit events', async () => { + const waitForReady = vi.fn().mockResolvedValue(undefined); + const report = vi.fn(); + const query = vi.fn((_input: any) => + stream([ + { + type: 'rate_limit_event', + rate_limit_info: { + status: 'allowed_warning', + resetsAt: new Date(2_000).toISOString(), + rateLimitType: 'five_hour', + utilization: 0.91, + }, + } as unknown as SDKMessage, + resultMessage({ result: 'ok' }), + ]), + ); + const runtime = new ClaudeCodeKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'sonnet' }, + query, + env: {}, + rateLimitGovernor: { waitForReady, report, maxRetryAttempts: () => 6 } as never, + }); + + await expect(runtime.generateText({ role: 'default', prompt: 'hello' })).resolves.toBe('ok'); + expect(waitForReady).toHaveBeenCalledTimes(1); + expect(report).toHaveBeenCalledWith({ + provider: 'claude-subscription', + status: 'warning', + resetAtMs: 2_000, + rateLimitType: 'five_hour', + utilization: 0.91, + }); + }); + + it('maps numeric Claude Code reset times from SDK rate-limit events', async () => { + const report = vi.fn(); + const resetAtMs = 1_700_000_000_000; + const query = vi.fn((_input: any) => + stream([ + { + type: 'rate_limit_event', + rate_limit_info: { + status: 'rejected', + resetsAt: resetAtMs, + rateLimitType: 'five_hour', + utilization: 1, + }, + } as unknown as SDKMessage, + resultMessage({ result: 'ok' }), + ]), + ); + const runtime = new ClaudeCodeKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'sonnet' }, + query, + env: {}, + rateLimitGovernor: { waitForReady: vi.fn().mockResolvedValue(undefined), report, maxRetryAttempts: () => 6 } as never, + }); + + await expect(runtime.generateText({ role: 'default', prompt: 'hello' })).resolves.toBe('ok'); + + expect(report).toHaveBeenCalledWith({ + provider: 'claude-subscription', + status: 'rejected', + resetAtMs, + rateLimitType: 'five_hour', + utilization: 1, + }); + }); + + it('retries a Claude Code query after an SDK rate-limit result error', async () => { + const waitForReady = vi.fn().mockResolvedValue(undefined); + const report = vi.fn(); + const resetAtMs = 1_700_000_000_000; + const query = vi + .fn() + .mockReturnValueOnce( + stream([ + { + type: 'rate_limit_event', + rate_limit_info: { + status: 'rejected', + resetsAt: resetAtMs, + rateLimitType: 'five_hour', + utilization: 1, + }, + } as unknown as SDKMessage, + resultMessage({ + subtype: 'error_during_execution', + is_error: true, + result: '', + errors: ['rate limit retry budget exhausted'], + terminal_reason: 'model_error', + } as never), + ]), + ) + .mockReturnValueOnce(stream([resultMessage({ result: 'ok' })])); + const runtime = new ClaudeCodeKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'sonnet' }, + query, + env: {}, + rateLimitGovernor: { waitForReady, report, maxRetryAttempts: () => 6 } as never, + }); + + await expect(runtime.generateText({ role: 'default', prompt: 'hello' })).resolves.toBe('ok'); + + expect(query).toHaveBeenCalledTimes(2); + expect(waitForReady).toHaveBeenCalledTimes(2); + expect(report).toHaveBeenCalledWith({ + provider: 'claude-subscription', + status: 'rejected', + resetAtMs, + rateLimitType: 'five_hour', + utilization: 1, + }); + }); + + it('reports Claude Code api retry messages as warning signals', async () => { + const report = vi.fn(); + const query = vi.fn((_input: any) => + stream([ + { + type: 'system', + subtype: 'api_retry', + retry_delay_ms: 12_000, + } as unknown as SDKMessage, + resultMessage({ result: 'ok' }), + ]), + ); + const runtime = new ClaudeCodeKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'sonnet' }, + query, + env: {}, + rateLimitGovernor: { waitForReady: vi.fn().mockResolvedValue(undefined), report, maxRetryAttempts: () => 6 } as never, + }); + + await runtime.generateText({ role: 'default', prompt: 'hello' }); + expect(report).toHaveBeenCalledWith({ + provider: 'claude-subscription', + status: 'warning', + retryAfterMs: 12_000, + rateLimitType: 'api_retry', + }); + }); + + it('passes abort signals into Claude Code governor waits', async () => { + const controller = new AbortController(); + const waitForReady = vi.fn().mockResolvedValue(undefined); + const query = vi.fn((_input: any) => stream([resultMessage({ result: 'ok' })])); + const runtime = new ClaudeCodeKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'sonnet' }, + query, + env: {}, + rateLimitGovernor: { waitForReady, report: vi.fn(), maxRetryAttempts: () => 6 } as never, + }); + + await expect(runtime.generateText({ role: 'default', prompt: 'hello', abortSignal: controller.signal })).resolves.toBe('ok'); + + expect(waitForReady).toHaveBeenCalledWith(controller.signal); + }); + + it('interrupts an active Claude Code query when the abort signal fires', async () => { + const controller = new AbortController(); + const streamStarted = deferred(); + const releaseStream = deferred(); + const interrupt = vi.fn(() => releaseStream.resolve()); + const queryResult = { + async *[Symbol.asyncIterator]() { + streamStarted.resolve(); + await releaseStream.promise; + yield resultMessage({ result: 'ok' }); + }, + interrupt, + }; + const query = vi.fn(() => queryResult as never); + const runtime = new ClaudeCodeKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'sonnet' }, + query, + env: {}, + rateLimitGovernor: { waitForReady: vi.fn().mockResolvedValue(undefined), report: vi.fn(), maxRetryAttempts: () => 6 } as never, + }); + + const pending = runtime.generateText({ role: 'default', prompt: 'hello', abortSignal: controller.signal }); + await streamStarted.promise; + controller.abort(); + + await expect(pending).rejects.toThrow(/Aborted/); + expect(interrupt).toHaveBeenCalledTimes(1); + }); + + it('throws abort before starting Claude Code query when the signal is already aborted', async () => { + const controller = new AbortController(); + controller.abort(); + const query = vi.fn((_input: any) => stream([resultMessage({ result: 'ok' })])); + const runtime = new ClaudeCodeKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'sonnet' }, + query, + env: {}, + rateLimitGovernor: { waitForReady: vi.fn().mockResolvedValue(undefined), report: vi.fn(), maxRetryAttempts: () => 6 } as never, + }); + + await expect(runtime.generateText({ role: 'default', prompt: 'hello', abortSignal: controller.signal })).rejects.toThrow(/Aborted/); + expect(query).not.toHaveBeenCalled(); + }); + + it('treats an interrupted Claude Code stream with no result as abort', async () => { + const controller = new AbortController(); + const streamStarted = deferred(); + const releaseStream = deferred(); + const interrupt = vi.fn(() => releaseStream.resolve()); + const queryResult = { + async *[Symbol.asyncIterator]() { + streamStarted.resolve(); + await releaseStream.promise; + }, + interrupt, + }; + const query = vi.fn(() => queryResult as never); + const runtime = new ClaudeCodeKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'sonnet' }, + query, + env: {}, + rateLimitGovernor: { waitForReady: vi.fn().mockResolvedValue(undefined), report: vi.fn(), maxRetryAttempts: () => 6 } as never, + }); + + const pending = runtime.generateText({ role: 'default', prompt: 'hello', abortSignal: controller.signal }); + await streamStarted.promise; + controller.abort(); + + await expect(pending).rejects.toThrow(/Aborted/); + expect(interrupt).toHaveBeenCalledTimes(1); + }); + it('validates structured output with the caller schema and whitelists the SDK StructuredOutput tool', async () => { const schema = z.object({ answer: z.string() }); const query = vi.fn((_input: any) => @@ -284,7 +533,7 @@ describe('ClaudeCodeKtxLlmRuntime', () => { stepBudget: 1, telemetryTags: { operationName: 'test' }, }), - ).resolves.toEqual({ stopReason: 'budget' }); + ).resolves.toMatchObject({ stopReason: 'budget' }); const options = query.mock.calls[0][0].options; expect(options.allowedTools).toEqual(['mcp__ktx__load_skill']); @@ -415,6 +664,64 @@ describe('ClaudeCodeKtxLlmRuntime', () => { ); }); + it('counts only assistant turns the SDK counts toward num_turns', async () => { + const assistantMessage = ( + overrides: Partial> & { uuid: string }, + ): SDKMessage => + ({ + type: 'assistant', + message: { role: 'assistant', content: [], stop_reason: 'end_turn' }, + parent_tool_use_id: null, + session_id: 'session-id', + ...overrides, + }) as unknown as SDKMessage; + + const query = vi.fn((_input: any) => + stream([ + initMessage(), + assistantMessage({ + uuid: '00000000-0000-4000-8000-0000000000a1', + error: 'max_output_tokens', + }), + assistantMessage({ + uuid: '00000000-0000-4000-8000-0000000000a2', + message: { role: 'assistant', content: [], stop_reason: 'pause_turn' } as never, + }), + assistantMessage({ uuid: '00000000-0000-4000-8000-0000000000a3' }), + { + type: 'assistant', + message: { role: 'assistant', content: [], stop_reason: 'end_turn' }, + parent_tool_use_id: 'tool-use-1', + uuid: '00000000-0000-4000-8000-0000000000a4', + session_id: 'session-id', + } as unknown as SDKMessage, + resultMessage({ subtype: 'success', terminal_reason: 'completed' }), + ]), + ); + const runtime = new ClaudeCodeKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'sonnet' }, + query, + env: {}, + }); + const onStepFinish = vi.fn(); + + await expect( + runtime.runAgentLoop({ + modelRole: 'default', + systemPrompt: 'system', + userPrompt: 'user', + toolSet: {}, + stepBudget: 40, + telemetryTags: { operationName: 'test' }, + onStepFinish, + }), + ).resolves.toMatchObject({ stopReason: 'natural' }); + + expect(onStepFinish).toHaveBeenCalledTimes(1); + expect(onStepFinish).toHaveBeenCalledWith({ stepIndex: 1, stepBudget: 40 }); + }); + it('logs and ignores onStepFinish callback errors', async () => { const query = vi.fn((_input: any) => stream([ @@ -455,7 +762,7 @@ describe('ClaudeCodeKtxLlmRuntime', () => { throw new Error('callback exploded'); }, }), - ).resolves.toEqual({ stopReason: 'natural' }); + ).resolves.toMatchObject({ stopReason: 'natural' }); expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('callback exploded')); }); @@ -467,6 +774,45 @@ describe('ClaudeCodeKtxLlmRuntime', () => { expect(mapClaudeCodeStopReason(resultMessage({ subtype: 'error_during_execution' }))).toBe('error'); }); + it('returns loop metrics including step count and mapped token usage', async () => { + const query = vi.fn((_input: any) => + stream([ + initMessage(), + { + type: 'assistant', + message: { role: 'assistant', content: [] }, + parent_tool_use_id: null, + uuid: '00000000-0000-4000-8000-000000000006', + session_id: 'session-id', + } as unknown as SDKMessage, + resultMessage({ + subtype: 'success', + terminal_reason: 'completed', + usage: { input_tokens: 50, output_tokens: 10 } as never, + }), + ]), + ); + const runtime = new ClaudeCodeKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'sonnet' }, + query, + env: {}, + }); + + const result = await runtime.runAgentLoop({ + modelRole: 'default', + systemPrompt: 'system', + userPrompt: 'user', + toolSet: {}, + stepBudget: 40, + telemetryTags: { operationName: 'test' }, + }); + + expect(result.metrics?.stepCount).toBe(1); + expect(result.metrics?.stepBoundariesMs).toHaveLength(1); + expect(result.metrics?.usage).toEqual({ inputTokens: 50, outputTokens: 10, totalTokens: 60 }); + }); + it('auth probe uses isolation options and a scrubbed env', async () => { const query = vi.fn((_input: any) => stream([initMessage(), resultMessage({ result: 'ok' })])); diff --git a/packages/cli/test/context/llm/codex-exec-events.test.ts b/packages/cli/test/context/llm/codex-exec-events.test.ts new file mode 100644 index 00000000..5edcfed8 --- /dev/null +++ b/packages/cli/test/context/llm/codex-exec-events.test.ts @@ -0,0 +1,188 @@ +import { describe, expect, it } from 'vitest'; +import { + parseCodexExecEventLine, + summarizeCodexExecEvents, +} from '../../../src/context/llm/codex-exec-events.js'; + +describe('Codex exec event parsing', () => { + it('uses the completed turn as one step when no MCP tools run', () => { + const summary = summarizeCodexExecEvents( + [ + { type: 'thread.started', thread_id: 'thr_1' }, + { type: 'turn.started' }, + { type: 'item.completed', item: { id: 'item_1', type: 'agent_message', text: 'hello from codex' } }, + { + type: 'turn.completed', + usage: { + input_tokens: 12, + cached_input_tokens: 4, + output_tokens: 5, + reasoning_output_tokens: 2, + }, + }, + ], + { startedAt: 100, now: () => 125 }, + ); + + expect(summary).toEqual({ + finalText: 'hello from codex', + stopReason: 'natural', + usage: { inputTokens: 12, outputTokens: 5, totalTokens: 17 }, + stepCount: 1, + stepBoundariesMs: [25], + toolCallCount: 0, + toolFailures: [], + }); + }); + + it('uses completed MCP tool calls as loop steps', () => { + const offsets = [115, 140, 175]; + const summary = summarizeCodexExecEvents( + [ + { type: 'turn.started' }, + { + type: 'item.started', + item: { id: 'call_1', type: 'mcp_tool_call', server: 'ktx', tool: 'search', arguments: {}, status: 'in_progress' }, + }, + { + type: 'item.completed', + item: { id: 'call_1', type: 'mcp_tool_call', server: 'ktx', tool: 'search', arguments: {}, status: 'completed' }, + }, + { + type: 'item.started', + item: { id: 'call_2', type: 'mcp_tool_call', server: 'ktx', tool: 'lookup', arguments: {}, status: 'in_progress' }, + }, + { + type: 'item.completed', + item: { + id: 'call_2', + type: 'mcp_tool_call', + server: 'ktx', + tool: 'lookup', + arguments: {}, + status: 'failed', + error: { message: 'denied' }, + }, + }, + { type: 'item.completed', item: { id: 'item_1', type: 'agent_message', text: 'done' } }, + { type: 'turn.completed', usage: { input_tokens: 1, output_tokens: 1, cached_input_tokens: 0, reasoning_output_tokens: 0 } }, + ], + { startedAt: 100, now: () => offsets.shift() ?? 175 }, + ); + + expect(summary).toEqual({ + finalText: 'done', + stopReason: 'natural', + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + stepCount: 2, + stepBoundariesMs: [15, 40], + toolCallCount: 2, + toolFailures: ['lookup: denied'], + }); + }); + + it('does not treat a completed MCP tool call as failed when Codex sends error: null', () => { + // Captured verbatim from a real @openai/codex-sdk run: successful tool calls + // carry `error: null` and `result` alongside `status: "completed"`. + const summary = summarizeCodexExecEvents([ + { type: 'turn.started' }, + { + type: 'item.started', + item: { + id: 'item_1', + type: 'mcp_tool_call', + server: 'ktx', + tool: 'echo_value', + arguments: { value: 'ktx_codex_tool_ok' }, + result: null, + error: null, + status: 'in_progress', + }, + }, + { + type: 'item.completed', + item: { + id: 'item_1', + type: 'mcp_tool_call', + server: 'ktx', + tool: 'echo_value', + arguments: { value: 'ktx_codex_tool_ok' }, + result: { content: [{ type: 'text', text: 'echo:ktx_codex_tool_ok' }], structured_content: null }, + error: null, + status: 'completed', + }, + }, + { type: 'item.completed', item: { id: 'm1', type: 'agent_message', text: 'done' } }, + { type: 'turn.completed', usage: { input_tokens: 1, output_tokens: 1 } }, + ]); + + expect(summary.toolFailures).toEqual([]); + expect(summary.toolCallCount).toBe(1); + }); + + it('counts built-in command executions as loop steps without failing the loop', () => { + const offsets = [110, 130]; + const summary = summarizeCodexExecEvents( + [ + { type: 'turn.started' }, + { type: 'item.completed', item: { id: 'c1', type: 'command_execution', command: 'ls', status: 'completed', exit_code: 0 } }, + { type: 'item.completed', item: { id: 'c2', type: 'command_execution', command: 'cat missing', status: 'failed', exit_code: 1 } }, + { type: 'item.completed', item: { id: 'm1', type: 'agent_message', text: 'done' } }, + { type: 'turn.completed', usage: { input_tokens: 2, output_tokens: 1 } }, + ], + { startedAt: 100, now: () => offsets.shift() ?? 130 }, + ); + + expect(summary.stepCount).toBe(2); + expect(summary.stepBoundariesMs).toEqual([10, 30]); + // A non-zero command exit is normal agent exploration, not a runtime tool failure. + expect(summary.toolFailures).toEqual([]); + expect(summary.toolCallCount).toBe(0); + }); + + it('maps turn failures into error stop reason', () => { + const summary = summarizeCodexExecEvents([ + { type: 'turn.started' }, + { type: 'turn.failed', error: { message: 'Codex could not connect to required MCP server' } }, + ]); + + expect(summary.stopReason).toBe('error'); + expect(summary.error?.message).toContain('Codex could not connect to required MCP server'); + }); + + it('unwraps the Codex API error envelope into its human-readable message', () => { + // Codex serializes API errors as a JSON envelope inside the event message. + const apiError = JSON.stringify({ + type: 'error', + status: 400, + error: { + type: 'invalid_request_error', + message: "The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.", + }, + }); + const summary = summarizeCodexExecEvents([ + { type: 'thread.started', thread_id: 'thr_1' }, + { type: 'turn.started' }, + { type: 'error', message: apiError }, + { type: 'turn.failed', error: { message: apiError } }, + ]); + + expect(summary.stopReason).toBe('error'); + expect(summary.error?.message).toBe( + "The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.", + ); + }); + + it('maps max-turns terminal reasons into budget stop reason when Codex emits one', () => { + const summary = summarizeCodexExecEvents([ + { type: 'turn.started' }, + { type: 'turn.completed', reason: 'max_turns', usage: { input_tokens: 1, output_tokens: 1 } }, + ]); + + expect(summary.stopReason).toBe('budget'); + }); + + it('throws a clear error for malformed JSONL lines', () => { + expect(() => parseCodexExecEventLine('{not-json')).toThrow('Codex JSONL event stream was malformed'); + }); +}); diff --git a/packages/cli/test/context/llm/codex-isolation.test.ts b/packages/cli/test/context/llm/codex-isolation.test.ts new file mode 100644 index 00000000..0ef39ee3 --- /dev/null +++ b/packages/cli/test/context/llm/codex-isolation.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest'; +import { + CODEX_ISOLATION_WARNING, + CODEX_ISOLATION_WARNING_FIX, + formatCodexIsolationWarning, +} from '../../../src/context/llm/codex-isolation.js'; + +describe('Codex isolation warning', () => { + it('documents the enforced and unenforced Codex isolation boundaries', () => { + expect(CODEX_ISOLATION_WARNING).toContain('runtime MCP server to the current ktx tool set'); + expect(CODEX_ISOLATION_WARNING).toContain('disables Codex web search'); + expect(CODEX_ISOLATION_WARNING).toContain('may still load user Codex config'); + expect(CODEX_ISOLATION_WARNING).toContain('built-in command execution'); + expect(CODEX_ISOLATION_WARNING_FIX).toContain('claude-code'); + expect(formatCodexIsolationWarning()).toBe( + `${CODEX_ISOLATION_WARNING} ${CODEX_ISOLATION_WARNING_FIX}`, + ); + }); +}); diff --git a/packages/cli/test/context/llm/codex-mcp-runtime-server.test.ts b/packages/cli/test/context/llm/codex-mcp-runtime-server.test.ts new file mode 100644 index 00000000..c793afb7 --- /dev/null +++ b/packages/cli/test/context/llm/codex-mcp-runtime-server.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it, vi } from 'vitest'; +import { z } from 'zod'; +import { + createCodexRuntimeMcpServer, + startCodexRuntimeMcpServer, +} from '../../../src/context/llm/codex-mcp-runtime-server.js'; + +describe('Codex runtime MCP server', () => { + it('registers runtime tools with markdown output', async () => { + const registered = new Map< + string, + { + config: { description?: string; inputSchema: unknown }; + handler: (input: Record) => Promise; + } + >(); + const server = createCodexRuntimeMcpServer({ + server: { + registerTool(name, config, handler) { + registered.set(name, { config, handler }); + }, + }, + toolSet: { + wiki_search: { + name: 'wiki_search', + description: 'Search the wiki', + inputSchema: z.object({ query: z.string() }), + execute: vi.fn(async () => ({ markdown: 'result markdown', structured: { matches: 1 } })), + }, + }, + }); + + expect(server).toBeDefined(); + expect([...registered.keys()]).toEqual(['wiki_search']); + expect(registered.get('wiki_search')?.config).toMatchObject({ + description: 'Search the wiki', + }); + await expect(registered.get('wiki_search')?.handler({ query: 'revenue' })).resolves.toEqual({ + content: [{ type: 'text', text: 'result markdown' }], + structuredContent: { matches: 1 }, + }); + }); + + it('starts loopback HTTP MCP with a bearer token and reports the runtime URL', async () => { + const close = vi.fn(async () => undefined); + const runServer = vi.fn(async () => ({ + server: { address: () => ({ port: 4321 }) }, + close, + })); + + const handle = await startCodexRuntimeMcpServer({ + projectDir: '/tmp/ktx-project', + toolSet: {}, + runServer: runServer as never, + }); + + expect(handle.url).toBe('http://127.0.0.1:4321/mcp'); + expect(handle.bearerTokenEnvVar).toBe('KTX_CODEX_RUNTIME_MCP_TOKEN'); + expect(handle.bearerToken).toMatch(/^[a-f0-9]{64}$/); + expect(runServer).toHaveBeenCalledWith( + expect.objectContaining({ + projectDir: '/tmp/ktx-project', + host: '127.0.0.1', + port: 0, + token: handle.bearerToken, + allowedHosts: ['127.0.0.1', 'localhost'], + allowedOrigins: [], + }), + ); + await handle.close(); + expect(close).toHaveBeenCalled(); + }); +}); diff --git a/packages/cli/test/context/llm/codex-models.test.ts b/packages/cli/test/context/llm/codex-models.test.ts new file mode 100644 index 00000000..83a1e2c8 --- /dev/null +++ b/packages/cli/test/context/llm/codex-models.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest'; +import { resolveCodexModel } from '../../../src/context/llm/codex-models.js'; + +describe('resolveCodexModel', () => { + it.each([ + ['codex', 'gpt-5.5'], + ['default', 'gpt-5.5'], + ['gpt-5.3-codex-spark', 'gpt-5.3-codex-spark'], + ['gpt-5.4', 'gpt-5.4'], + ])('maps %s to %s', (input, expected) => { + expect(resolveCodexModel(input)).toBe(expected); + }); + + it.each(['', ' ', 'sonnet', 'claude-sonnet-4-6'])('rejects %s', (input) => { + expect(() => resolveCodexModel(input)).toThrow('Unsupported Codex model'); + }); +}); diff --git a/packages/cli/test/context/llm/codex-runtime-config.test.ts b/packages/cli/test/context/llm/codex-runtime-config.test.ts new file mode 100644 index 00000000..97c80446 --- /dev/null +++ b/packages/cli/test/context/llm/codex-runtime-config.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest'; +import { buildCodexRuntimeConfig } from '../../../src/context/llm/codex-runtime-config.js'; + +describe('buildCodexRuntimeConfig', () => { + it('builds generic config without SDK thread-option fields', () => { + expect(buildCodexRuntimeConfig({ model: 'gpt-5.3-codex' })).toEqual({ + configOverrides: { + history: { persistence: 'none' }, + }, + env: {}, + }); + }); + + it('adds only the temporary ktx MCP server and exact enabled tools', () => { + expect( + buildCodexRuntimeConfig({ + model: 'gpt-5.3-codex', + mcp: { + url: 'http://127.0.0.1:4567/mcp', + bearerTokenEnvVar: 'KTX_CODEX_RUNTIME_MCP_TOKEN', + bearerToken: 'secret-token', + toolNames: ['sl_read_source', 'wiki_search'], + }, + }), + ).toEqual({ + configOverrides: { + history: { persistence: 'none' }, + mcp_servers: { + ktx: { + url: 'http://127.0.0.1:4567/mcp', + bearer_token_env_var: 'KTX_CODEX_RUNTIME_MCP_TOKEN', + enabled_tools: ['sl_read_source', 'wiki_search'], + default_tools_approval_mode: 'approve', + required: true, + }, + }, + }, + env: { + KTX_CODEX_RUNTIME_MCP_TOKEN: 'secret-token', + }, + }); + }); +}); diff --git a/packages/cli/test/context/llm/codex-runtime.test.ts b/packages/cli/test/context/llm/codex-runtime.test.ts new file mode 100644 index 00000000..4c3fcdfd --- /dev/null +++ b/packages/cli/test/context/llm/codex-runtime.test.ts @@ -0,0 +1,604 @@ +import { describe, expect, it, vi } from 'vitest'; +import { z } from 'zod'; +import { + CodexKtxLlmRuntime, + runCodexAuthProbe, +} from '../../../src/context/llm/codex-runtime.js'; + +async function* events(items: unknown[]) { + for (const item of items) { + yield item; + } +} + +function runner(items: unknown[]) { + return { + runStreamed: vi.fn(async () => events(items)), + }; +} + +/** Yields the given events, then throws — mirroring the SDK throwing on a non-zero codex exec exit. */ +function throwingRunner(items: unknown[], error: Error) { + return { + runStreamed: vi.fn(async () => + (async function* () { + for (const item of items) { + yield item; + } + throw error; + })(), + ), + }; +} + +const MODEL_UNSUPPORTED_API_ERROR = JSON.stringify({ + type: 'error', + status: 400, + error: { + type: 'invalid_request_error', + message: "The 'gpt-5.3-codex' model is not supported when using Codex with a ChatGPT account.", + }, +}); + +function budgetRunner() { + let observedSignal: AbortSignal | undefined; + return { + observedSignal: () => observedSignal, + runStreamed: vi.fn(async (input: { signal?: AbortSignal }) => { + observedSignal = input.signal; + return events([ + { type: 'turn.started' }, + { type: 'item.started', item: { type: 'mcp_tool_call', server: 'ktx', tool: 'first', status: 'in_progress' } }, + { type: 'item.completed', item: { type: 'mcp_tool_call', server: 'ktx', tool: 'first', status: 'completed' } }, + { type: 'item.started', item: { type: 'mcp_tool_call', server: 'ktx', tool: 'second', status: 'in_progress' } }, + { type: 'item.completed', item: { type: 'mcp_tool_call', server: 'ktx', tool: 'second', status: 'completed' } }, + { type: 'turn.completed', usage: { input_tokens: 1, output_tokens: 1 } }, + ]); + }), + }; +} + +describe('CodexKtxLlmRuntime', () => { + it('generates text with the role-selected model and metrics', async () => { + const onMetrics = vi.fn(); + const fakeRunner = runner([ + { type: 'turn.started' }, + { type: 'item.completed', item: { type: 'agent_message', text: 'hello' } }, + { type: 'turn.completed', usage: { input_tokens: 3, output_tokens: 4, total_tokens: 7 } }, + ]); + const runtime = new CodexKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'codex', triage: 'gpt-5.4' }, + runner: fakeRunner, + }); + + await expect(runtime.generateText({ role: 'triage', system: 'system', prompt: 'prompt', onMetrics })).resolves.toBe('hello'); + expect(fakeRunner.runStreamed).toHaveBeenCalledWith( + expect.objectContaining({ + projectDir: '/tmp/project', + model: 'gpt-5.4', + prompt: 'system\n\nprompt', + }), + ); + expect(onMetrics).toHaveBeenCalledWith(expect.objectContaining({ usage: { inputTokens: 3, outputTokens: 4, totalTokens: 7 } })); + }); + + it('generates and validates structured output', async () => { + const fakeRunner = runner([ + { type: 'turn.started' }, + { type: 'item.completed', item: { type: 'agent_message', text: '{"answer":"yes"}' } }, + { type: 'turn.completed' }, + ]); + const runtime = new CodexKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'codex' }, + runner: fakeRunner, + }); + + await expect( + runtime.generateObject({ + role: 'default', + prompt: 'json', + schema: z.object({ answer: z.string() }), + }), + ).resolves.toEqual({ answer: 'yes' }); + expect(fakeRunner.runStreamed).toHaveBeenCalledWith( + expect.objectContaining({ + outputSchema: expect.objectContaining({ type: 'object' }), + }), + ); + }); + + it('returns a structured-output error when Codex final text is invalid JSON', async () => { + const fakeRunner = runner([ + { type: 'turn.started' }, + { type: 'item.completed', item: { type: 'agent_message', text: 'not json' } }, + { type: 'turn.completed' }, + ]); + const runtime = new CodexKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'codex' }, + runner: fakeRunner, + }); + + await expect( + runtime.generateObject({ + role: 'default', + prompt: 'json', + schema: z.object({ answer: z.string() }), + }), + ).rejects.toThrow('Codex structured output failed validation'); + }); + + it('reports Codex rate-limit failures and retries with opaque backoff', async () => { + const waitForReady = vi.fn().mockResolvedValue(undefined); + const report = vi.fn(); + const fakeRunner = { + runStreamed: vi + .fn() + .mockResolvedValueOnce(events([{ type: 'turn.failed', error: { message: '429 rate limit exceeded' } }])) + .mockResolvedValueOnce( + events([ + { type: 'turn.started' }, + { type: 'item.completed', item: { type: 'agent_message', text: 'ok' } }, + { type: 'turn.completed' }, + ]), + ), + }; + const runtime = new CodexKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'codex' }, + runner: fakeRunner, + rateLimitGovernor: { waitForReady, report, maxRetryAttempts: () => 6 } as never, + }); + + await expect(runtime.generateText({ role: 'default', prompt: 'hello' })).resolves.toBe('ok'); + expect(report).toHaveBeenCalledWith({ provider: 'codex', status: 'rejected', rateLimitType: 'opaque' }); + expect(waitForReady).toHaveBeenCalledTimes(2); + expect(fakeRunner.runStreamed).toHaveBeenCalledTimes(2); + }); + + it('reports thrown Codex rate-limit failures and retries with opaque backoff', async () => { + const waitForReady = vi.fn().mockResolvedValue(undefined); + const report = vi.fn(); + const fakeRunner = { + runStreamed: vi + .fn() + .mockRejectedValueOnce(new Error('ThreadError: 429 rate limit exceeded')) + .mockResolvedValueOnce( + events([ + { type: 'turn.started' }, + { type: 'item.completed', item: { type: 'agent_message', text: 'ok' } }, + { type: 'turn.completed' }, + ]), + ), + }; + const runtime = new CodexKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'codex' }, + runner: fakeRunner, + rateLimitGovernor: { waitForReady, report, maxRetryAttempts: () => 6 } as never, + }); + + await expect(runtime.generateText({ role: 'default', prompt: 'hello' })).resolves.toBe('ok'); + + expect(report).toHaveBeenCalledWith({ provider: 'codex', status: 'rejected', rateLimitType: 'opaque' }); + expect(waitForReady).toHaveBeenCalledTimes(2); + expect(fakeRunner.runStreamed).toHaveBeenCalledTimes(2); + }); + + it('surfaces Codex rate-limit failures without retrying when no governor is present', async () => { + const fakeRunner = runner([{ type: 'turn.failed', error: { message: '429 rate limit exceeded' } }]); + const runtime = new CodexKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'codex' }, + runner: fakeRunner, + }); + + await expect(runtime.generateText({ role: 'default', prompt: 'hello' })).rejects.toThrow(/rate limit/i); + expect(fakeRunner.runStreamed).toHaveBeenCalledTimes(1); + }); + + it('passes abort signals into Codex text generation and governor waits', async () => { + const controller = new AbortController(); + const waitForReady = vi.fn().mockResolvedValue(undefined); + let observedSignal: AbortSignal | undefined; + const fakeRunner = { + runStreamed: vi.fn(async (input: { signal?: AbortSignal }) => { + observedSignal = input.signal; + return events([ + { type: 'turn.started' }, + { type: 'item.completed', item: { type: 'agent_message', text: 'ok' } }, + { type: 'turn.completed' }, + ]); + }), + }; + const runtime = new CodexKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'codex' }, + runner: fakeRunner, + rateLimitGovernor: { waitForReady, report: vi.fn(), maxRetryAttempts: () => 6 } as never, + }); + + await expect(runtime.generateText({ role: 'default', prompt: 'hello', abortSignal: controller.signal })).resolves.toBe('ok'); + + expect(waitForReady).toHaveBeenCalledWith(controller.signal); + expect(observedSignal).toBe(controller.signal); + }); + + it('links the parent abort signal into Codex agent-loop streamed runs', async () => { + const controller = new AbortController(); + let releaseStream!: () => void; + const streamRelease = new Promise((resolve) => { + releaseStream = resolve; + }); + let markRunnerCalled!: () => void; + const runnerCalled = new Promise((resolve) => { + markRunnerCalled = resolve; + }); + let observedSignal: AbortSignal | undefined; + const fakeRunner = { + runStreamed: vi.fn(async (input: { signal?: AbortSignal }) => { + observedSignal = input.signal; + markRunnerCalled(); + return (async function* () { + await streamRelease; + yield { type: 'turn.started' }; + yield { type: 'item.completed', item: { type: 'agent_message', text: 'ok' } }; + yield { type: 'turn.completed' }; + })(); + }), + }; + const runtime = new CodexKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'codex' }, + runner: fakeRunner, + }); + + const pending = runtime.runAgentLoop({ + modelRole: 'default', + systemPrompt: '', + userPrompt: '', + toolSet: {}, + stepBudget: 10, + telemetryTags: {}, + abortSignal: controller.signal, + }); + + await runnerCalled; + expect(observedSignal).toBeDefined(); + expect(observedSignal).not.toBe(controller.signal); + controller.abort(); + expect(observedSignal?.aborted).toBe(true); + releaseStream(); + await expect(pending).resolves.toMatchObject({ stopReason: 'natural' }); + }); + + it('starts and closes a temporary MCP server for tool-backed agent loops', async () => { + const close = vi.fn(async () => undefined); + const startMcpServer = vi.fn(async () => ({ + url: 'http://127.0.0.1:4321/mcp', + bearerTokenEnvVar: 'KTX_CODEX_RUNTIME_MCP_TOKEN' as const, + bearerToken: 'token', + close, + })); + const fakeRunner = runner([ + { type: 'turn.started' }, + { type: 'item.started', item: { type: 'mcp_tool_call', name: 'wiki_search' } }, + { type: 'item.completed', item: { type: 'agent_message', text: 'done' } }, + { type: 'turn.completed', usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } }, + ]); + const runtime = new CodexKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'codex' }, + runner: fakeRunner, + startMcpServer, + }); + const onStepFinish = vi.fn(); + + const result = await runtime.runAgentLoop({ + modelRole: 'default', + systemPrompt: 'system', + userPrompt: 'user', + stepBudget: 5, + telemetryTags: {}, + onStepFinish, + toolSet: { + aliased_wiki_tool: { + name: 'wiki_search', + description: 'Search wiki', + inputSchema: z.object({ query: z.string() }), + execute: vi.fn(), + }, + }, + }); + + expect(result.stopReason).toBe('natural'); + expect(result.metrics).toMatchObject({ stepCount: 1, usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 } }); + expect(onStepFinish).toHaveBeenCalledWith({ stepIndex: 1, stepBudget: 5 }); + expect(startMcpServer).toHaveBeenCalledWith({ projectDir: '/tmp/project', toolSet: expect.any(Object) }); + expect(fakeRunner.runStreamed).toHaveBeenCalledWith( + expect.objectContaining({ + env: { KTX_CODEX_RUNTIME_MCP_TOKEN: 'token' }, + configOverrides: expect.objectContaining({ + mcp_servers: expect.objectContaining({ + ktx: expect.objectContaining({ + url: 'http://127.0.0.1:4321/mcp', + enabled_tools: ['wiki_search'], + required: true, + }), + }), + }), + }), + ); + expect(close).toHaveBeenCalled(); + }); + + it('returns error stop reason on turn failure', async () => { + const runtime = new CodexKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'codex' }, + runner: runner([{ type: 'turn.failed', error: { message: 'boom' } }]), + }); + + const result = await runtime.runAgentLoop({ + modelRole: 'default', + systemPrompt: 'system', + userPrompt: 'user', + stepBudget: 5, + telemetryTags: {}, + toolSet: {}, + }); + + expect(result.stopReason).toBe('error'); + expect(result.error?.message).toBe('boom'); + }); + + it('surfaces failed MCP tool calls as agent-loop errors', async () => { + const runtime = new CodexKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'codex' }, + runner: runner([ + { type: 'turn.started' }, + { type: 'item.started', item: { type: 'mcp_tool_call', server: 'ktx', tool: 'search', status: 'in_progress' } }, + { + type: 'item.completed', + item: { + type: 'mcp_tool_call', + server: 'ktx', + tool: 'search', + status: 'failed', + error: { message: 'denied' }, + }, + }, + { type: 'turn.completed', usage: { input_tokens: 1, output_tokens: 1 } }, + ]), + }); + + const result = await runtime.runAgentLoop({ + modelRole: 'default', + systemPrompt: 'system', + userPrompt: 'user', + stepBudget: 5, + telemetryTags: {}, + toolSet: {}, + }); + + expect(result.stopReason).toBe('error'); + expect(result.error?.message).toBe('Codex runtime tool call failed: search: denied'); + expect(result.metrics).toMatchObject({ + stepCount: 1, + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + }); + }); + + it('returns budget and aborts the Codex stream when local MCP step budget is reached', async () => { + const fakeRunner = budgetRunner(); + const runtime = new CodexKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'codex' }, + runner: fakeRunner, + }); + const onStepFinish = vi.fn(); + + const result = await runtime.runAgentLoop({ + modelRole: 'default', + systemPrompt: 'system', + userPrompt: 'user', + stepBudget: 1, + telemetryTags: {}, + onStepFinish, + toolSet: { + first: { + name: 'first', + description: 'First tool', + inputSchema: z.object({}), + execute: vi.fn(), + }, + }, + }); + + expect(result.stopReason).toBe('budget'); + expect(result.error).toBeUndefined(); + expect(result.metrics).toMatchObject({ stepCount: 1 }); + expect(onStepFinish).toHaveBeenCalledTimes(1); + expect(onStepFinish).toHaveBeenCalledWith({ stepIndex: 1, stepBudget: 1 }); + expect(fakeRunner.observedSignal()?.aborted).toBe(true); + }); + + it('counts built-in command_execution steps against the budget and aborts the stream', async () => { + let observedSignal: AbortSignal | undefined; + const fakeRunner = { + observedSignal: () => observedSignal, + runStreamed: vi.fn(async (input: { signal?: AbortSignal }) => { + observedSignal = input.signal; + return events([ + { type: 'turn.started' }, + { type: 'item.started', item: { type: 'command_execution', command: 'ls', status: 'in_progress' } }, + { type: 'item.completed', item: { type: 'command_execution', command: 'ls', status: 'completed', exit_code: 0 } }, + { type: 'item.started', item: { type: 'command_execution', command: 'cat a', status: 'in_progress' } }, + { type: 'item.completed', item: { type: 'command_execution', command: 'cat a', status: 'completed', exit_code: 0 } }, + { type: 'item.completed', item: { type: 'command_execution', command: 'cat b', status: 'completed', exit_code: 0 } }, + { type: 'turn.completed', usage: { input_tokens: 1, output_tokens: 1 } }, + ]); + }), + }; + const runtime = new CodexKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'codex' }, + runner: fakeRunner, + }); + const onStepFinish = vi.fn(); + + const result = await runtime.runAgentLoop({ + modelRole: 'default', + systemPrompt: 'system', + userPrompt: 'user', + stepBudget: 2, + telemetryTags: {}, + onStepFinish, + toolSet: {}, + }); + + expect(result.stopReason).toBe('budget'); + expect(result.error).toBeUndefined(); + expect(result.metrics).toMatchObject({ stepCount: 2 }); + expect(onStepFinish).toHaveBeenCalledTimes(2); + expect(onStepFinish).toHaveBeenLastCalledWith({ stepIndex: 2, stepBudget: 2 }); + expect(fakeRunner.observedSignal()?.aborted).toBe(true); + }); + + it('fires onStepFinish live as each step completes, before the stream drains', async () => { + const order: string[] = []; + async function* liveEvents() { + yield { type: 'turn.started' }; + yield { type: 'item.completed', item: { type: 'mcp_tool_call', server: 'ktx', tool: 'a', status: 'completed' } }; + order.push('yielded-after-step-1'); + yield { type: 'item.completed', item: { type: 'mcp_tool_call', server: 'ktx', tool: 'b', status: 'completed' } }; + order.push('yielded-after-step-2'); + yield { type: 'item.completed', item: { type: 'agent_message', text: 'done' } }; + yield { type: 'turn.completed', usage: { input_tokens: 1, output_tokens: 1 } }; + } + const fakeRunner = { runStreamed: vi.fn(async () => liveEvents()) }; + const runtime = new CodexKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'codex' }, + runner: fakeRunner, + }); + + const result = await runtime.runAgentLoop({ + modelRole: 'default', + systemPrompt: 'system', + userPrompt: 'user', + stepBudget: 10, + telemetryTags: {}, + onStepFinish: ({ stepIndex }) => { + order.push(`step-${stepIndex}`); + }, + toolSet: {}, + }); + + expect(result.stopReason).toBe('natural'); + expect(result.metrics).toMatchObject({ stepCount: 2 }); + expect(order).toEqual(['step-1', 'yielded-after-step-1', 'step-2', 'yielded-after-step-2']); + }); + + it('surfaces the real Codex error event even when the SDK stream throws afterward', async () => { + // The SDK yields the error/turn.failed events on stdout, then throws on the + // non-zero exit. The masked exit message must not hide the real API error. + const fakeRunner = throwingRunner( + [ + { type: 'thread.started', thread_id: 't' }, + { type: 'turn.started' }, + { type: 'error', message: MODEL_UNSUPPORTED_API_ERROR }, + { type: 'turn.failed', error: { message: MODEL_UNSUPPORTED_API_ERROR } }, + ], + new Error('Codex Exec exited with code 1: Reading prompt from stdin...'), + ); + const runtime = new CodexKtxLlmRuntime({ + projectDir: '/tmp/project', + modelSlots: { default: 'codex' }, + runner: fakeRunner, + }); + + await expect(runtime.generateText({ role: 'default', prompt: 'hi' })).rejects.toThrow( + 'not supported when using Codex with a ChatGPT account', + ); + }); + + it('probes Codex authentication through a minimal non-interactive turn', async () => { + const fakeRunner = runner([ + { type: 'turn.started' }, + { type: 'item.completed', item: { type: 'agent_message', text: 'ok' } }, + { type: 'turn.completed' }, + ]); + + await expect( + runCodexAuthProbe({ + projectDir: '/tmp/project', + model: 'codex', + runner: fakeRunner, + }), + ).resolves.toEqual({ ok: true }); + }); + + it('reports an unavailable model without blaming auth when Codex rejects the model', async () => { + const fakeRunner = throwingRunner( + [ + { type: 'turn.started' }, + { type: 'turn.failed', error: { message: MODEL_UNSUPPORTED_API_ERROR } }, + ], + new Error('Codex Exec exited with code 1: Reading prompt from stdin...'), + ); + + const result = await runCodexAuthProbe({ + projectDir: '/tmp/project', + model: 'gpt-5.3-codex', + runner: fakeRunner, + }); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.message).not.toContain('authentication is not usable'); + expect(result.message).toContain('not available'); + expect(result.message).toContain('gpt-5.3-codex'); + expect(result.message).toContain('not supported when using Codex with a ChatGPT account'); + // A model-access failure must steer the user at the model config, not auth. + expect(result.fix).toContain('llm.models.default'); + expect(result.fix).not.toContain('Authenticate Codex'); + } + }); + + it('reports an auth failure when Codex exits without an error event', async () => { + const fakeRunner = throwingRunner( + [], + new Error('Codex Exec exited with code 1: Not logged in. Run `codex login`.'), + ); + + const result = await runCodexAuthProbe({ + projectDir: '/tmp/project', + model: 'gpt-5.5', + runner: fakeRunner, + }); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.message).toContain('authentication is not usable'); + expect(result.message).toContain('Not logged in'); + expect(result.fix).toContain('Authenticate Codex'); + } + }); + + it('rejects an unsupported model id before probing, steering at llm.models.default', async () => { + const result = await runCodexAuthProbe({ + projectDir: '/tmp/project', + model: 'not-a-real-model', + }); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.message).toContain('Unsupported Codex model'); + expect(result.fix).toContain('llm.models.default'); + } + }); +}); diff --git a/packages/cli/test/context/llm/codex-sdk-runner.test.ts b/packages/cli/test/context/llm/codex-sdk-runner.test.ts new file mode 100644 index 00000000..fdafc666 --- /dev/null +++ b/packages/cli/test/context/llm/codex-sdk-runner.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it, vi } from 'vitest'; + +const sdkMock = vi.hoisted(() => { + const events = (async function* () { + yield { type: 'turn.completed', usage: { input_tokens: 1, output_tokens: 2 } }; + })(); + const runStreamed = vi.fn(async () => ({ events })); + const startThread = vi.fn(() => ({ runStreamed })); + const Codex = vi.fn(function Codex(this: { startThread: typeof startThread }, options?: unknown) { + Object.assign(this, { options, startThread }); + }); + return { Codex, startThread, runStreamed }; +}); + +vi.mock('@openai/codex-sdk', () => ({ Codex: sdkMock.Codex })); + +import { CodexSdkCliRunner } from '../../../src/context/llm/codex-sdk-runner.js'; + +async function collectAsync(items: AsyncIterable): Promise { + const collected: T[] = []; + for await (const item of items) { + collected.push(item); + } + return collected; +} + +describe('CodexSdkCliRunner', () => { + it('passes isolated env through the SDK and runtime controls through thread options', async () => { + const runner = new CodexSdkCliRunner({ + envBase: { + HOME: '/home/ktx-user', + PATH: '/usr/local/bin:/usr/bin', + CODEX_HOME: '/home/ktx-user/.codex', + HTTPS_PROXY: 'http://proxy.example', + KTX_UNRELATED_SECRET: 'must-not-copy', // pragma: allowlist secret + }, + }); + const previousToken = process.env.KTX_CODEX_RUNTIME_MCP_TOKEN; + process.env.KTX_CODEX_RUNTIME_MCP_TOKEN = 'outer-token'; + const outputSchema = { + type: 'object', + properties: { answer: { type: 'string' } }, + required: ['answer'], + additionalProperties: false, + }; + const controller = new AbortController(); + + try { + const events = await runner.runStreamed({ + projectDir: '/tmp/ktx-project', + model: 'gpt-5.3-codex', + prompt: 'Return JSON.', + configOverrides: { + history: { persistence: 'none' }, + }, + env: { KTX_CODEX_RUNTIME_MCP_TOKEN: 'run-token' }, + outputSchema, + signal: controller.signal, + }); + + expect(sdkMock.Codex).toHaveBeenCalledWith({ + config: { + history: { persistence: 'none' }, + }, + env: { + HOME: '/home/ktx-user', + PATH: '/usr/local/bin:/usr/bin', + CODEX_HOME: '/home/ktx-user/.codex', + HTTPS_PROXY: 'http://proxy.example', + KTX_CODEX_RUNTIME_MCP_TOKEN: 'run-token', + }, + }); + expect(process.env.KTX_CODEX_RUNTIME_MCP_TOKEN).toBe('outer-token'); + expect(sdkMock.startThread).toHaveBeenCalledWith({ + workingDirectory: '/tmp/ktx-project', + skipGitRepoCheck: true, + model: 'gpt-5.3-codex', + sandboxMode: 'read-only', + webSearchMode: 'disabled', + approvalPolicy: 'never', + }); + expect(sdkMock.runStreamed).toHaveBeenCalledWith('Return JSON.', { + outputSchema, + signal: controller.signal, + }); + await expect(collectAsync(events)).resolves.toEqual([ + { type: 'turn.completed', usage: { input_tokens: 1, output_tokens: 2 } }, + ]); + } finally { + if (previousToken === undefined) { + delete process.env.KTX_CODEX_RUNTIME_MCP_TOKEN; + } else { + process.env.KTX_CODEX_RUNTIME_MCP_TOKEN = previousToken; + } + } + }); +}); diff --git a/packages/cli/src/context/llm/debug-request-recorder.test.ts b/packages/cli/test/context/llm/debug-request-recorder.test.ts similarity index 98% rename from packages/cli/src/context/llm/debug-request-recorder.test.ts rename to packages/cli/test/context/llm/debug-request-recorder.test.ts index 4a00400f..e7a15b88 100644 --- a/packages/cli/src/context/llm/debug-request-recorder.test.ts +++ b/packages/cli/test/context/llm/debug-request-recorder.test.ts @@ -5,7 +5,7 @@ import { afterEach, describe, expect, it } from 'vitest'; import { createJsonlKtxLlmDebugRequestRecorder, summarizeKtxLlmDebugRequest, -} from './debug-request-recorder.js'; +} from '../../../src/context/llm/debug-request-recorder.js'; describe('summarizeKtxLlmDebugRequest', () => { it('records providerOptions positions without message text or tool schemas', () => { diff --git a/packages/cli/src/context/llm/embedding-port.test.ts b/packages/cli/test/context/llm/embedding-port.test.ts similarity index 95% rename from packages/cli/src/context/llm/embedding-port.test.ts rename to packages/cli/test/context/llm/embedding-port.test.ts index 323e7c5b..6bde1f2a 100644 --- a/packages/cli/src/context/llm/embedding-port.test.ts +++ b/packages/cli/test/context/llm/embedding-port.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { KtxIngestEmbeddingPortAdapter, KtxScanEmbeddingPortAdapter } from './embedding-port.js'; +import { KtxIngestEmbeddingPortAdapter, KtxScanEmbeddingPortAdapter } from '../../../src/context/llm/embedding-port.js'; describe('KTX embedding port adapters', () => { it('adapts LLM modules embeddings to ingest embedding port shape', async () => { diff --git a/packages/cli/src/context/llm/local-config.test.ts b/packages/cli/test/context/llm/local-config.test.ts similarity index 76% rename from packages/cli/src/context/llm/local-config.test.ts rename to packages/cli/test/context/llm/local-config.test.ts index 930ee8a5..eed66261 100644 --- a/packages/cli/src/context/llm/local-config.test.ts +++ b/packages/cli/test/context/llm/local-config.test.ts @@ -3,13 +3,14 @@ import { buildDefaultKtxProjectConfig, type KtxProjectEmbeddingConfig, type KtxProjectLlmConfig, -} from '../project/config.js'; +} from '../../../src/context/project/config.js'; import { createLocalKtxEmbeddingProviderFromConfig, createLocalKtxLlmProviderFromConfig, + createLocalKtxLlmRuntimeFromConfig, resolveLocalKtxEmbeddingConfig, resolveLocalKtxLlmConfig, -} from './local-config.js'; +} from '../../../src/context/llm/local-config.js'; describe('local KTX LLM config', () => { it('resolves env and file references into a KtxLlmConfig', () => { @@ -129,6 +130,64 @@ describe('local KTX LLM config', () => { vertexFallbackTo5m: false, }); }); + + it('passes the rate-limit governor into created runtimes', () => { + const rateLimitGovernor = {} as never; + const createClaudeCodeRuntime = vi.fn(() => ({ + generateText: vi.fn(), + generateObject: vi.fn(), + runAgentLoop: vi.fn(), + })); + const createCodexRuntime = vi.fn(() => ({ + generateText: vi.fn(), + generateObject: vi.fn(), + runAgentLoop: vi.fn(), + })); + const createAiSdkRuntime = vi.fn(() => ({ + generateText: vi.fn(), + generateObject: vi.fn(), + runAgentLoop: vi.fn(), + })); + const createKtxLlmProvider = vi.fn(() => ({ + getModel: vi.fn(), + getModelByName: vi.fn(), + cacheMarker: vi.fn(), + repairToolCallHandler: vi.fn(), + thinkingProviderOptions: vi.fn(), + telemetryConfig: vi.fn(), + promptCachingConfig: vi.fn(), + activeBackend: vi.fn(() => 'anthropic'), + })); + + createLocalKtxLlmRuntimeFromConfig( + { + provider: { backend: 'claude-code' }, + models: { default: 'sonnet' }, + promptCaching: undefined, + }, + { projectDir: '/tmp/project', env: {}, rateLimitGovernor, createClaudeCodeRuntime }, + ); + createLocalKtxLlmRuntimeFromConfig( + { + provider: { backend: 'codex' }, + models: { default: 'codex' }, + promptCaching: undefined, + }, + { projectDir: '/tmp/project', env: {}, rateLimitGovernor, createCodexRuntime }, + ); + createLocalKtxLlmRuntimeFromConfig( + { + provider: { backend: 'anthropic' }, + models: { default: 'claude-sonnet-4-6' }, + promptCaching: undefined, + }, + { env: {}, rateLimitGovernor, createAiSdkRuntime, createKtxLlmProvider: createKtxLlmProvider as never }, + ); + + expect(createClaudeCodeRuntime).toHaveBeenCalledWith(expect.objectContaining({ rateLimitGovernor })); + expect(createCodexRuntime).toHaveBeenCalledWith(expect.objectContaining({ rateLimitGovernor })); + expect(createAiSdkRuntime).toHaveBeenCalledWith(expect.objectContaining({ rateLimitGovernor })); + }); }); describe('local KTX embedding config', () => { diff --git a/packages/cli/test/context/llm/rate-limit-governor.test.ts b/packages/cli/test/context/llm/rate-limit-governor.test.ts new file mode 100644 index 00000000..51fcba84 --- /dev/null +++ b/packages/cli/test/context/llm/rate-limit-governor.test.ts @@ -0,0 +1,278 @@ +import { describe, expect, it } from 'vitest'; +import { + createRateLimitGovernorConfig, + RateLimitGovernor, + type RateLimitWaitState, +} from '../../../src/context/llm/rate-limit-governor.js'; + +function testClock(startMs = 1_000) { + let nowMs = startMs; + return { + now: () => nowMs, + advance: (ms: number) => { + nowMs += ms; + }, + }; +} + +async function flushMicrotasks(turns = 10): Promise { + for (let i = 0; i < turns; i += 1) { + await Promise.resolve(); + } +} + +describe('RateLimitGovernor', () => { + it('drops and restores the effective work-unit limit from warning signals', () => { + const clock = testClock(); + const states: RateLimitWaitState[] = []; + const governor = new RateLimitGovernor( + createRateLimitGovernorConfig({ maxConcurrency: 6, minConcurrencyUnderPressure: 1 }), + { now: clock.now, sleep: async () => undefined, random: () => 0 }, + ); + governor.subscribe((state) => states.push(state)); + + expect(governor.currentLimit()).toBe(6); + governor.report({ + provider: 'claude-subscription', + status: 'warning', + utilization: 0.91, + rateLimitType: 'five_hour', + }); + expect(governor.currentLimit()).toBe(1); + governor.report({ + provider: 'claude-subscription', + status: 'allowed', + utilization: 0.2, + rateLimitType: 'five_hour', + }); + expect(governor.currentLimit()).toBe(6); + expect(states.map((state) => state.kind)).toContain('concurrency_adjusted'); + }); + + it('blocks work slots during a rejected reset window and emits wait states', async () => { + const clock = testClock(); + const states: RateLimitWaitState[] = []; + const sleeps: number[] = []; + const governor = new RateLimitGovernor( + createRateLimitGovernorConfig({ maxConcurrency: 2, waitStateTickMs: 100 }), + { + now: clock.now, + random: () => 0, + sleep: async (ms) => { + sleeps.push(ms); + clock.advance(ms); + }, + }, + ); + governor.subscribe((state) => states.push(state)); + + governor.report({ provider: 'anthropic-api', status: 'rejected', retryAfterMs: 250, rateLimitType: 'rpm' }); + const release = await governor.acquireWorkSlot(); + release(); + + expect(sleeps).toEqual([100, 100, 50]); + expect(states.some((state) => state.kind === 'wait_started' && state.provider === 'anthropic-api')).toBe(true); + expect(states.some((state) => state.kind === 'wait_finished' && state.provider === 'anthropic-api')).toBe(true); + }); + + it('rejects an interrupted wait without consuming a work slot', async () => { + const clock = testClock(); + let abortListener: (() => void) | undefined; + const governor = new RateLimitGovernor( + createRateLimitGovernorConfig({ maxConcurrency: 1, waitStateTickMs: 100 }), + { + now: clock.now, + random: () => 0, + sleep: async (_ms, signal) => + new Promise((_resolve, reject) => { + abortListener = () => reject(new DOMException('Aborted', 'AbortError')); + signal?.addEventListener('abort', abortListener, { once: true }); + }), + }, + ); + const controller = new AbortController(); + + governor.report({ + provider: 'claude-subscription', + status: 'rejected', + resetAtMs: 2_000, + rateLimitType: 'five_hour', + }); + const pending = governor.acquireWorkSlot(controller.signal); + controller.abort(); + abortListener?.(); + + await expect(pending).rejects.toThrow(/Aborted/); + expect(governor.activeSlots()).toBe(0); + }); + + it('rejects an already-aborted ready wait', async () => { + const governor = new RateLimitGovernor( + createRateLimitGovernorConfig({ maxConcurrency: 1 }), + { sleep: async () => undefined, random: () => 0 }, + ); + const controller = new AbortController(); + controller.abort(); + + await expect(governor.waitForReady(controller.signal)).rejects.toThrow(/Aborted/); + }); + + it('rejects an already-aborted work slot without consuming capacity', async () => { + const governor = new RateLimitGovernor( + createRateLimitGovernorConfig({ maxConcurrency: 1 }), + { sleep: async () => undefined, random: () => 0 }, + ); + const controller = new AbortController(); + controller.abort(); + + await expect(governor.acquireWorkSlot(controller.signal)).rejects.toThrow(/Aborted/); + expect(governor.activeSlots()).toBe(0); + }); + + it('uses bounded opaque backoff for rejected signals without reset hints', async () => { + const clock = testClock(); + const sleeps: number[] = []; + const governor = new RateLimitGovernor( + createRateLimitGovernorConfig({ + maxConcurrency: 1, + retry: { maxAttempts: 3, baseDelayMs: 1_000, maxDelayMs: 60_000, jitter: false }, + }), + { + now: clock.now, + random: () => 0, + sleep: async (ms) => { + sleeps.push(ms); + clock.advance(ms); + }, + }, + ); + + governor.report({ provider: 'codex', status: 'rejected', rateLimitType: 'opaque' }); + const release1 = await governor.acquireWorkSlot(); + release1(); + governor.report({ provider: 'codex', status: 'rejected', rateLimitType: 'opaque' }); + const release2 = await governor.acquireWorkSlot(); + release2(); + + expect(sleeps).toEqual([1_000, 2_000]); + }); + + it('exposes the configured retry budget and disables outer retries when pacing is off', () => { + const retry = { maxAttempts: 3, baseDelayMs: 1_000, maxDelayMs: 60_000, jitter: false }; + const enabled = new RateLimitGovernor(createRateLimitGovernorConfig({ retry })); + expect(enabled.maxRetryAttempts()).toBe(3); + + const disabled = new RateLimitGovernor(createRateLimitGovernorConfig({ enabled: false, retry })); + expect(disabled.maxRetryAttempts()).toBe(1); + }); + + it('emits visible wait ticks after a rejected report without a waiting caller', async () => { + const clock = testClock(); + const states: RateLimitWaitState[] = []; + const sleeps: number[] = []; + const governor = new RateLimitGovernor( + createRateLimitGovernorConfig({ maxConcurrency: 4, minConcurrencyUnderPressure: 1, waitStateTickMs: 100 }), + { + now: clock.now, + random: () => 0, + sleep: async (ms, signal) => { + if (signal?.aborted) { + throw new DOMException('Aborted', 'AbortError'); + } + sleeps.push(ms); + clock.advance(ms); + }, + }, + ); + governor.subscribe((state) => states.push(state)); + + governor.report({ + provider: 'claude-subscription', + status: 'rejected', + resetAtMs: 1_250, + rateLimitType: 'five_hour', + }); + await flushMicrotasks(); + + expect(sleeps).toEqual([100, 100, 50]); + expect(states).toContainEqual( + expect.objectContaining({ + kind: 'wait_started', + provider: 'claude-subscription', + rateLimitType: 'five_hour', + remainingMs: 250, + }), + ); + expect(states.filter((state) => state.kind === 'wait_tick')).toHaveLength(3); + expect(states).toContainEqual( + expect.objectContaining({ + kind: 'wait_finished', + provider: 'claude-subscription', + rateLimitType: 'five_hour', + remainingMs: 0, + }), + ); + }); + + it('does not duplicate countdown sleeps when a work slot waits during the same pause', async () => { + const clock = testClock(); + const states: RateLimitWaitState[] = []; + const sleeps: number[] = []; + const governor = new RateLimitGovernor( + createRateLimitGovernorConfig({ maxConcurrency: 2, waitStateTickMs: 100 }), + { + now: clock.now, + random: () => 0, + sleep: async (ms, signal) => { + if (signal?.aborted) { + throw new DOMException('Aborted', 'AbortError'); + } + sleeps.push(ms); + clock.advance(ms); + }, + }, + ); + governor.subscribe((state) => states.push(state)); + + governor.report({ provider: 'anthropic-api', status: 'rejected', retryAfterMs: 250, rateLimitType: 'rpm' }); + const pendingRelease = governor.acquireWorkSlot(); + await flushMicrotasks(); + const release = await pendingRelease; + release(); + + expect(sleeps).toEqual([100, 100, 50]); + expect(states.filter((state) => state.kind === 'wait_tick')).toHaveLength(3); + expect(governor.activeSlots()).toBe(0); + }); + + it('stops the visible wait ticker when the last subscriber unsubscribes', async () => { + const clock = testClock(); + let abortCount = 0; + const governor = new RateLimitGovernor( + createRateLimitGovernorConfig({ maxConcurrency: 1, waitStateTickMs: 100 }), + { + now: clock.now, + random: () => 0, + sleep: async (_ms, signal) => + new Promise((_resolve, reject) => { + signal?.addEventListener( + 'abort', + () => { + abortCount += 1; + reject(new DOMException('Aborted', 'AbortError')); + }, + { once: true }, + ); + }), + }, + ); + const unsubscribe = governor.subscribe(() => undefined); + + governor.report({ provider: 'claude-subscription', status: 'rejected', retryAfterMs: 1_000 }); + await flushMicrotasks(1); + unsubscribe(); + await flushMicrotasks(1); + + expect(abortCount).toBe(1); + }); +}); diff --git a/packages/cli/src/context/llm/runtime-local-config.test.ts b/packages/cli/test/context/llm/runtime-local-config.test.ts similarity index 53% rename from packages/cli/src/context/llm/runtime-local-config.test.ts rename to packages/cli/test/context/llm/runtime-local-config.test.ts index e5516ffa..14adca7c 100644 --- a/packages/cli/src/context/llm/runtime-local-config.test.ts +++ b/packages/cli/test/context/llm/runtime-local-config.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { createLocalKtxLlmProviderFromConfig, createLocalKtxLlmRuntimeFromConfig } from './local-config.js'; +import { createLocalKtxLlmProviderFromConfig, createLocalKtxLlmRuntimeFromConfig } from '../../../src/context/llm/local-config.js'; describe('local KTX LLM runtime config', () => { it('creates a Claude Code runtime for claude-code backend without creating an AI SDK provider', () => { @@ -22,4 +22,25 @@ describe('local KTX LLM runtime config', () => { }), ).toBeNull(); }); + + it('creates a Codex runtime for codex backend without creating an AI SDK provider', () => { + const runtime = createLocalKtxLlmRuntimeFromConfig( + { + provider: { backend: 'codex' }, + models: { default: 'codex', triage: 'gpt-5.4' }, + }, + { env: {}, projectDir: '/tmp/project', createCodexRuntime: vi.fn((deps) => ({ deps }) as never) }, + ); + + expect(runtime).toMatchObject({ deps: expect.objectContaining({ projectDir: '/tmp/project' }) }); + }); + + it('returns null from the AI SDK provider factory for codex backend', () => { + expect( + createLocalKtxLlmProviderFromConfig({ + provider: { backend: 'codex' }, + models: { default: 'codex' }, + }), + ).toBeNull(); + }); }); diff --git a/packages/cli/src/context/llm/runtime-tools.test.ts b/packages/cli/test/context/llm/runtime-tools.test.ts similarity index 91% rename from packages/cli/src/context/llm/runtime-tools.test.ts rename to packages/cli/test/context/llm/runtime-tools.test.ts index c1276d7d..f3c1b7f8 100644 --- a/packages/cli/src/context/llm/runtime-tools.test.ts +++ b/packages/cli/test/context/llm/runtime-tools.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; import { z } from 'zod'; -import { createAiSdkToolSet, createClaudeSdkTools, normalizeKtxRuntimeToolOutput } from './runtime-tools.js'; -import type { KtxRuntimeToolDescriptor } from './runtime-port.js'; +import { createAiSdkToolSet, createClaudeSdkTools, normalizeKtxRuntimeToolOutput } from '../../../src/context/llm/runtime-tools.js'; +import type { KtxRuntimeToolDescriptor } from '../../../src/context/llm/runtime-port.js'; describe('runtime tool descriptors', () => { const descriptor: KtxRuntimeToolDescriptor<{ id: string }, { ok: boolean }> = { diff --git a/packages/cli/src/context/mcp/__snapshots__/mcp-tools-list.json b/packages/cli/test/context/mcp/__snapshots__/mcp-tools-list.json similarity index 96% rename from packages/cli/src/context/mcp/__snapshots__/mcp-tools-list.json rename to packages/cli/test/context/mcp/__snapshots__/mcp-tools-list.json index db84b328..b38851f4 100644 --- a/packages/cli/src/context/mcp/__snapshots__/mcp-tools-list.json +++ b/packages/cli/test/context/mcp/__snapshots__/mcp-tools-list.json @@ -65,7 +65,7 @@ }, "limit": { "default": 10, - "description": "Maximum wiki pages to return. Defaults to 10.", + "description": "Maximum wiki pages to return.", "type": "integer", "minimum": 1, "maximum": 50 @@ -307,7 +307,7 @@ { "name": "sl_query", "title": "Semantic Layer Query", - "description": "Execute a semantic-layer query and return rows, headers, generated SQL, and plan details. Example: sl_query({ connectionId: \"warehouse\", measures: [\"orders.order_count\"], dimensions: [{ dimension: \"orders.created_at\", granularity: \"month\" }] }).", + "description": "Execute a semantic-layer query and return headers, rows, and total row count, plus correctness notes (e.g. compile-only or fan-out) when relevant. The generated SQL and full query plan are omitted by default; request them with include: [\"sql\"] and/or include: [\"plan\"]. Example: sl_query({ connectionId: \"warehouse\", measures: [\"orders.order_count\"], dimensions: [{ field: \"orders.created_at\", granularity: \"month\" }], include: [\"sql\"] }).", "inputSchema": { "type": "object", "properties": { @@ -349,7 +349,8 @@ "description": "Measures to select. Use semantic-layer keys when available." }, "dimensions": { - "description": "Dimensions to group by. Strings and {dimension, granularity} are accepted.", + "default": [], + "description": "Dimensions to group by. Use {field, granularity?} entries.", "type": "array", "items": { "type": "object", @@ -389,7 +390,8 @@ } }, "order_by": { - "description": "Sort clauses. Strings and Cube-style {id, desc} are accepted.", + "default": [], + "description": "Sort clauses. Use {field, direction?} entries.", "type": "array", "items": { "type": "object", @@ -401,7 +403,7 @@ }, "direction": { "default": "asc", - "description": "Sort direction: \"asc\" or \"desc\". Defaults to \"asc\".", + "description": "Sort direction for this field.", "type": "string", "enum": [ "asc", @@ -416,15 +418,27 @@ }, "limit": { "default": 1000, - "description": "Maximum rows to return. Defaults to 1000.", + "description": "Maximum rows to return.", "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "include_empty": { "default": true, - "description": "Whether to include empty dimension groups. Defaults to true.", + "description": "Whether to include empty dimension groups.", "type": "boolean" + }, + "include": { + "default": [], + "description": "Extra detail to attach to the response: \"sql\" for the generated SQL, \"plan\" for the full query plan.", + "type": "array", + "items": { + "type": "string", + "enum": [ + "plan", + "sql" + ] + } } }, "required": [ @@ -441,9 +455,6 @@ "dialect": { "type": "string" }, - "sql": { - "type": "string" - }, "headers": { "type": "array", "items": { @@ -460,6 +471,15 @@ "totalRows": { "type": "number" }, + "notes": { + "type": "array", + "items": { + "type": "string" + } + }, + "sql": { + "type": "string" + }, "plan": { "type": "object", "propertyNames": { @@ -469,7 +489,6 @@ } }, "required": [ - "sql", "headers", "rows", "totalRows" @@ -489,7 +508,7 @@ { "name": "entity_details", "title": "Entity Details", - "description": "Read table and column metadata from the latest live-database scan snapshot. Example: entity_details({ connectionId: \"warehouse\", entities: [{ table: { schema: \"public\", table: \"orders\" }, columns: [\"id\"] }] }).", + "description": "Read table and column metadata from the latest live-database scan snapshot. Example: entity_details({ connectionId: \"warehouse\", entities: [{ table: { catalog: null, db: \"public\", name: \"orders\" }, columns: [\"id\"] }] }).", "inputSchema": { "type": "object", "properties": { @@ -549,7 +568,7 @@ ] } ], - "description": "Table display string or object ref. {schema, table} is accepted as an alias for {db, name}." + "description": "Table display string or canonical object ref." }, "columns": { "description": "Optional column filter.", @@ -560,7 +579,10 @@ "description": "Column name to inspect." } } - } + }, + "required": [ + "table" + ] }, "description": "Tables or columns to inspect. Maximum 20 entities." } @@ -1236,8 +1258,8 @@ } }, "limit": { - "description": "Maximum refs to return. Defaults to 15.", - "default": 15, + "description": "Maximum refs to return.", + "default": 10, "type": "integer", "minimum": 1, "maximum": 50 @@ -1391,7 +1413,7 @@ "description": "Parser-validated read-only SQL, e.g. \"select count(*) from public.orders\"." }, "maxRows": { - "description": "Maximum rows to return. Defaults to 1000.", + "description": "Maximum rows to return.", "default": 1000, "type": "integer", "minimum": 1, diff --git a/packages/cli/src/context/mcp/local-project-ports.test.ts b/packages/cli/test/context/mcp/local-project-ports.test.ts similarity index 98% rename from packages/cli/src/context/mcp/local-project-ports.test.ts rename to packages/cli/test/context/mcp/local-project-ports.test.ts index aa06b47e..afe85b4c 100644 --- a/packages/cli/src/context/mcp/local-project-ports.test.ts +++ b/packages/cli/test/context/mcp/local-project-ports.test.ts @@ -2,10 +2,10 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { initKtxProject } from '../../context/project/project.js'; -import { createKtxConnectorCapabilities, type KtxQueryResult, type KtxScanConnector, type KtxSchemaSnapshot } from '../../context/scan/types.js'; -import { writeLocalSlSource } from '../../context/sl/local-sl.js'; -import { createLocalProjectMcpContextPorts } from './local-project-ports.js'; +import { initKtxProject } from '../../../src/context/project/project.js'; +import { createKtxConnectorCapabilities, type KtxQueryResult, type KtxScanConnector, type KtxSchemaSnapshot } from '../../../src/context/scan/types.js'; +import { writeLocalSlSource } from '../../../src/context/sl/local-sl.js'; +import { createLocalProjectMcpContextPorts } from '../../../src/context/mcp/local-project-ports.js'; describe('createLocalProjectMcpContextPorts', () => { let tempDir: string; @@ -56,6 +56,8 @@ describe('createLocalProjectMcpContextPorts', () => { driver: snapshot.driver, capabilities: createKtxConnectorCapabilities({ readOnlySql: queryResult !== undefined }), introspect: vi.fn(async () => snapshot), + listSchemas: vi.fn(async () => []), + listTables: vi.fn(async () => []), executeReadOnly: queryResult === undefined ? undefined : vi.fn(async () => queryResult), cleanup: vi.fn(async () => {}), }; diff --git a/packages/cli/src/context/mcp/server.test.ts b/packages/cli/test/context/mcp/server.test.ts similarity index 73% rename from packages/cli/src/context/mcp/server.test.ts rename to packages/cli/test/context/mcp/server.test.ts index bee00c00..1359d346 100644 --- a/packages/cli/src/context/mcp/server.test.ts +++ b/packages/cli/test/context/mcp/server.test.ts @@ -1,15 +1,16 @@ -import { access, mkdtemp, readFile, rm } from 'node:fs/promises'; +import { access, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { createLocalProjectMemoryIngest } from '../../context/memory/local-memory.js'; -import { detectCaptureSignals } from '../../context/memory/capture-signals.js'; -import type { MemoryAgentInput } from '../../context/memory/types.js'; -import { initKtxProject } from '../../context/project/project.js'; -import { jsonToolResult } from './context-tools.js'; -import { createDefaultKtxMcpServer, createKtxMcpServer } from './server.js'; +import { createLocalProjectMemoryIngest } from '../../../src/context/memory/local-memory.js'; +import { detectCaptureSignals } from '../../../src/context/memory/capture-signals.js'; +import type { MemoryAgentInput } from '../../../src/context/memory/types.js'; +import { parseKtxProjectConfig, serializeKtxProjectConfig } from '../../../src/context/project/config.js'; +import { initKtxProject } from '../../../src/context/project/project.js'; +import { jsonToolResult } from '../../../src/context/mcp/context-tools.js'; +import { createDefaultKtxMcpServer, createKtxMcpServer } from '../../../src/context/mcp/server.js'; import type { KtxDiscoverDataMcpPort, KtxDictionarySearchMcpPort, @@ -21,7 +22,13 @@ import type { KtxSqlExecutionMcpPort, KtxSqlExecutionResponse, MemoryIngestPort, -} from './types.js'; +} from '../../../src/context/mcp/types.js'; + +const reportExceptionMock = vi.hoisted(() => vi.fn(async () => {})); + +vi.mock('../../../src/telemetry/exception.js', () => ({ + reportException: reportExceptionMock, +})); type RegisteredTool = { name: string; @@ -47,10 +54,10 @@ function makeFakeServer() { }; } -function makeIo() { +function makeIo(stdoutIsTTY = true) { let stderr = ''; return { - stdout: { isTTY: true, write() {} }, + stdout: { isTTY: stdoutIsTTY, write() {} }, stderr: { write(chunk: string) { stderr += chunk; @@ -272,8 +279,102 @@ describe('createKtxMcpServer', () => { expect(io.stderrText()).toContain('"event":"mcp_request_completed"'); expect(io.stderrText()).toContain('"toolName":"wiki_search"'); - expect(io.stderrText()).toContain('"sampleRate":0.1'); + expect(io.stderrText()).toContain('"sampleRate":1'); expect(io.stderrText()).not.toContain(projectDir); + // No client connected through the SDK here, so getClientInfo is absent: the + // event still emits and the optional client fields are simply omitted. + expect(io.stderrText()).not.toContain('mcpClientName'); + expect(io.stderrText()).not.toContain('mcpClientVersion'); + }); + + it('reports MCP tool exceptions with a tool-derived source', async () => { + reportExceptionMock.mockClear(); + vi.stubEnv('ANTHROPIC_API_KEY', 'mcp-anthropic-secret'); // pragma: allowlist secret + const fake = makeFakeServer(); + const io = makeIo(); + const projectDir = await mkdtemp(join(tmpdir(), 'ktx-mcp-exception-')); + try { + await initKtxProject({ projectDir }); + const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')); + await writeFile( + join(projectDir, 'ktx.yaml'), + serializeKtxProjectConfig({ + ...config, + llm: { + ...config.llm, + provider: { + backend: 'anthropic', + anthropic: { api_key: 'env:ANTHROPIC_API_KEY' }, // pragma: allowlist secret + }, + models: { default: 'claude-sonnet-4-6' }, + }, + }), + 'utf-8', + ); + + createKtxMcpServer({ + server: fake.server, + userContext: { userId: 'local-user' }, + projectDir, + io, + contextTools: { + knowledge: { + search: vi.fn().mockRejectedValue(new Error('wiki failed')), + read: vi.fn().mockResolvedValue(null), + }, + }, + }); + + await expect(getTool(fake.tools, 'wiki_search').handler({ query: 'revenue recognition', limit: 5 })).resolves.toMatchObject({ + isError: true, + }); + + expect(reportExceptionMock).toHaveBeenCalledWith( + expect.objectContaining({ + context: expect.objectContaining({ source: 'mcp:wiki_search', handled: true, fatal: false }), + projectDir, + redactionSecrets: expect.arrayContaining(['mcp-anthropic-secret']), + }), + ); + } finally { + await rm(projectDir, { recursive: true, force: true }); + } + }); + + it('captures the connecting MCP client name and version', async () => { + vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); + vi.stubEnv('CI', ''); + // Non-TTY io keeps the test hermetic (no ~/.ktx/telemetry.json is created) + // and mirrors a real headless MCP server; debug mode still emits the payload. + const io = makeIo(false); + + const server = createDefaultKtxMcpServer({ + name: 'ktx-test', + version: '0.0.0-test', + userContext: { userId: 'mcp-user' }, + projectDir: '/tmp/ktx-mcp-client-telemetry', + io, + contextTools: { + knowledge: { + search: vi.fn().mockResolvedValue({ results: [], totalFound: 0 }), + read: vi.fn().mockResolvedValue(null), + }, + }, + }); + const client = new Client({ name: 'test-agent', version: '9.9.9' }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await Promise.all([server.connect(serverTransport), client.connect(clientTransport)]); + + try { + await client.callTool({ name: 'wiki_search', arguments: { query: 'revenue recognition', limit: 5 } }); + } finally { + await client.close(); + await server.close(); + } + + expect(io.stderrText()).toContain('"event":"mcp_request_completed"'); + expect(io.stderrText()).toContain('"mcpClientName":"test-agent"'); + expect(io.stderrText()).toContain('"mcpClientVersion":"9.9.9"'); }); it('registers parser-gated sql_execution when the host provides a SQL execution port', async () => { @@ -307,16 +408,12 @@ describe('createKtxMcpServer', () => { content: [ { type: 'text', - text: JSON.stringify( - { - headers: ['status', 'count'], - headerTypes: ['text', 'bigint'], - rows: [['paid', 42]], - rowCount: 1, - }, - null, - 2, - ), + text: JSON.stringify({ + headers: ['status', 'count'], + headerTypes: ['text', 'bigint'], + rows: [['paid', 42]], + rowCount: 1, + }), }, ], structuredContent: { @@ -466,7 +563,43 @@ describe('createKtxMcpServer', () => { }); }); - it('sl_query normalizes order_by from cube-style {id, desc} and bare strings to {field, direction}', async () => { + it('sl_query rejects cube-style order_by aliases and bare strings', async () => { + const fake = makeFakeServer(); + const semanticLayer: KtxSemanticLayerMcpPort = { + readSource: vi.fn(), + query: vi.fn().mockResolvedValue({ + sql: '', + headers: [], + rows: [], + totalRows: 0, + }), + }; + + createKtxMcpServer({ + server: fake.server, + userContext: { userId: 'local-user' }, + contextTools: { semanticLayer }, + }); + + await expect( + getTool(fake.tools, 'sl_query').handler({ + connectionId: 'warehouse', + measures: ['orders.count'], + order_by: [{ id: 'orders.quarter_label', desc: false }], + }), + ).resolves.toMatchObject({ isError: true }); + await expect( + getTool(fake.tools, 'sl_query').handler({ + connectionId: 'warehouse', + measures: ['orders.count'], + order_by: ['orders.segment'], + }), + ).resolves.toMatchObject({ isError: true }); + + expect(semanticLayer.query).not.toHaveBeenCalled(); + }); + + it('sl_query accepts canonical order_by entries', async () => { const fake = makeFakeServer(); const semanticLayer: KtxSemanticLayerMcpPort = { readSource: vi.fn(), @@ -489,9 +622,7 @@ describe('createKtxMcpServer', () => { measures: ['orders.count'], order_by: [ { field: 'orders.total', direction: 'desc' }, - { id: 'orders.quarter_label', desc: false }, - { id: 'orders.created_at', desc: true }, - 'orders.segment', + { field: 'orders.segment' }, ], }); @@ -501,8 +632,6 @@ describe('createKtxMcpServer', () => { query: expect.objectContaining({ order_by: [ { field: 'orders.total', direction: 'desc' }, - { field: 'orders.quarter_label', direction: 'asc' }, - { field: 'orders.created_at', direction: 'desc' }, { field: 'orders.segment', direction: 'asc' }, ], }), @@ -511,7 +640,35 @@ describe('createKtxMcpServer', () => { ); }); - it('sl_query normalizes cube-style dimensions to field dimensions', async () => { + it('sl_query rejects cube-style dimensions and bare strings', async () => { + const fake = makeFakeServer(); + const semanticLayer = makeAllContextTools().semanticLayer!; + + createKtxMcpServer({ + server: fake.server, + userContext: { userId: 'local-user' }, + contextTools: { semanticLayer }, + }); + + await expect( + getTool(fake.tools, 'sl_query').handler({ + connectionId: 'warehouse', + measures: ['orders.count'], + dimensions: [{ dimension: 'orders.created_at', granularity: 'month' }], + }), + ).resolves.toMatchObject({ isError: true }); + await expect( + getTool(fake.tools, 'sl_query').handler({ + connectionId: 'warehouse', + measures: ['orders.count'], + dimensions: ['orders.status'], + }), + ).resolves.toMatchObject({ isError: true }); + + expect(semanticLayer.query).not.toHaveBeenCalled(); + }); + + it('sl_query accepts canonical field dimensions', async () => { const fake = makeFakeServer(); const semanticLayer = makeAllContextTools().semanticLayer!; @@ -524,7 +681,7 @@ describe('createKtxMcpServer', () => { await getTool(fake.tools, 'sl_query').handler({ connectionId: 'warehouse', measures: ['orders.count'], - dimensions: [{ dimension: 'orders.created_at', granularity: 'month' }, 'orders.status'], + dimensions: [{ field: 'orders.created_at', granularity: 'month' }, { field: 'orders.status' }], }); expect(semanticLayer.query).toHaveBeenCalledWith( @@ -538,7 +695,113 @@ describe('createKtxMcpServer', () => { ); }); - it('entity_details normalizes sql-style schema table refs', async () => { + it('sl_query default response omits plan and sql but keeps compile-only and fan-out notes', async () => { + const fake = makeFakeServer(); + const semanticLayer: KtxSemanticLayerMcpPort = { + readSource: vi.fn(), + query: vi.fn().mockResolvedValue({ + connectionId: 'warehouse', + dialect: 'postgres', + sql: 'select count(*) from public.orders', + headers: ['order_count'], + rows: [], + totalRows: 0, + plan: { + sources_used: ['orders'], + has_fan_out: true, + fan_out_description: 'orders fans out across line_items', + execution: { mode: 'compile_only', reason: 'No execution adapter configured.' }, + }, + }), + }; + + createKtxMcpServer({ + server: fake.server, + userContext: { userId: 'local-user' }, + contextTools: { semanticLayer }, + }); + + const result = await getTool(fake.tools, 'sl_query').handler({ + connectionId: 'warehouse', + measures: ['orders.order_count'], + }); + + expect(result).toMatchObject({ + structuredContent: { + connectionId: 'warehouse', + dialect: 'postgres', + headers: ['order_count'], + rows: [], + totalRows: 0, + notes: ['No execution adapter configured.', 'orders fans out across line_items'], + }, + }); + const structured = (result as { structuredContent: Record }).structuredContent; + expect(structured.sql).toBeUndefined(); + expect(structured.plan).toBeUndefined(); + }); + + it('sl_query attaches sql and plan only when include requests them', async () => { + const fake = makeFakeServer(); + const plan = { sources_used: ['orders'], execution: { mode: 'executed' } }; + const semanticLayer: KtxSemanticLayerMcpPort = { + readSource: vi.fn(), + query: vi.fn().mockResolvedValue({ + connectionId: 'warehouse', + dialect: 'postgres', + sql: 'select count(*) from public.orders', + headers: ['order_count'], + rows: [[3]], + totalRows: 1, + plan, + }), + }; + + createKtxMcpServer({ + server: fake.server, + userContext: { userId: 'local-user' }, + contextTools: { semanticLayer }, + }); + + const result = await getTool(fake.tools, 'sl_query').handler({ + connectionId: 'warehouse', + measures: ['orders.order_count'], + include: ['plan', 'sql'], + }); + + expect(result).toMatchObject({ + structuredContent: { + sql: 'select count(*) from public.orders', + plan, + rows: [[3]], + totalRows: 1, + }, + }); + const structured = (result as { structuredContent: Record }).structuredContent; + expect(structured.notes).toBeUndefined(); + }); + + it('entity_details rejects sql-style schema table ref aliases', async () => { + const fake = makeFakeServer(); + const entityDetails = makeAllContextTools().entityDetails!; + + createKtxMcpServer({ + server: fake.server, + userContext: { userId: 'local-user' }, + contextTools: { entityDetails }, + }); + + await expect( + getTool(fake.tools, 'entity_details').handler({ + connectionId: 'warehouse', + entities: [{ table: { schema: 'public', table: 'orders' }, columns: ['id'] }], + }), + ).resolves.toMatchObject({ isError: true }); + + expect(entityDetails.read).not.toHaveBeenCalled(); + }); + + it('entity_details accepts canonical table refs', async () => { const fake = makeFakeServer(); const entityDetails = makeAllContextTools().entityDetails!; @@ -550,7 +813,7 @@ describe('createKtxMcpServer', () => { await getTool(fake.tools, 'entity_details').handler({ connectionId: 'warehouse', - entities: [{ table: { schema: 'public', table: 'orders' }, columns: ['id'] }], + entities: [{ table: { catalog: null, db: 'public', name: 'orders' }, columns: ['id'] }], }); expect(entityDetails.read).toHaveBeenCalledWith({ @@ -718,7 +981,7 @@ describe('createKtxMcpServer', () => { connectionId: '00000000-0000-4000-8000-000000000001', }), ).resolves.toEqual({ - content: [{ type: 'text', text: JSON.stringify({ runId: 'run-1' }, null, 2) }], + content: [{ type: 'text', text: JSON.stringify({ runId: 'run-1' }) }], structuredContent: { runId: 'run-1' }, }); expect(ingest.ingest).toHaveBeenCalledWith({ @@ -745,21 +1008,17 @@ describe('createKtxMcpServer', () => { content: [ { type: 'text', - text: JSON.stringify( - { - runId: 'run-1', - status: 'done', - stage: 'done', - done: true, - captured: { wiki: ['revenue'], sl: [], xrefs: [] }, - error: null, - commitHash: 'abc123', - skillsLoaded: ['wiki_capture'], - signalDetected: true, - }, - null, - 2, - ), + text: JSON.stringify({ + runId: 'run-1', + status: 'done', + stage: 'done', + done: true, + captured: { wiki: ['revenue'], sl: [], xrefs: [] }, + error: null, + commitHash: 'abc123', + skillsLoaded: ['wiki_capture'], + signalDetected: true, + }), }, ], structuredContent: { @@ -967,19 +1226,15 @@ describe('createKtxMcpServer', () => { content: [ { type: 'text', - text: JSON.stringify( - { - connections: [ - { - id: '00000000-0000-4000-8000-000000000001', - name: 'Warehouse', - connectionType: 'POSTGRES', - }, - ], - }, - null, - 2, - ), + text: JSON.stringify({ + connections: [ + { + id: '00000000-0000-4000-8000-000000000001', + name: 'Warehouse', + connectionType: 'POSTGRES', + }, + ], + }), }, ], structuredContent: { @@ -1018,7 +1273,7 @@ describe('createKtxMcpServer', () => { await getTool(fake.tools, 'sl_query').handler({ connectionId: '00000000-0000-4000-8000-000000000001', measures: ['orders.count'], - dimensions: ['orders.created_at'], + dimensions: [{ field: 'orders.created_at' }], filters: ['orders.status = paid'], limit: 25, }); diff --git a/packages/cli/src/context/memory/local-memory.test.ts b/packages/cli/test/context/memory/local-memory.test.ts similarity index 96% rename from packages/cli/src/context/memory/local-memory.test.ts rename to packages/cli/test/context/memory/local-memory.test.ts index 1a7240c9..23a93cc4 100644 --- a/packages/cli/src/context/memory/local-memory.test.ts +++ b/packages/cli/test/context/memory/local-memory.test.ts @@ -2,9 +2,9 @@ import { access, mkdtemp, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { initKtxProject } from '../../context/project/project.js'; -import { createLocalProjectMemoryIngest } from './local-memory.js'; -import { LocalMemoryRunStore } from './local-memory-runs.js'; +import { initKtxProject } from '../../../src/context/project/project.js'; +import { createLocalProjectMemoryIngest } from '../../../src/context/memory/local-memory.js'; +import { LocalMemoryRunStore } from '../../../src/context/memory/local-memory-runs.js'; vi.mock('ai', () => ({ generateText: vi.fn().mockResolvedValue({ text: '', toolCalls: [] }), diff --git a/packages/cli/src/context/memory/memory-agent.service.ingest.test.ts b/packages/cli/test/context/memory/memory-agent.service.ingest.test.ts similarity index 98% rename from packages/cli/src/context/memory/memory-agent.service.ingest.test.ts rename to packages/cli/test/context/memory/memory-agent.service.ingest.test.ts index 1c13bdd2..acb1c2f8 100644 --- a/packages/cli/src/context/memory/memory-agent.service.ingest.test.ts +++ b/packages/cli/test/context/memory/memory-agent.service.ingest.test.ts @@ -13,8 +13,8 @@ vi.mock('ai', () => ({ // Imported AFTER vi.mock so the mocked module is used. import { generateText } from 'ai'; -import { SYSTEM_GIT_AUTHOR } from '../../context/tools/authors.js'; -import { MemoryAgentService } from './memory-agent.service.js'; +import { SYSTEM_GIT_AUTHOR } from '../../../src/context/tools/authors.js'; +import { MemoryAgentService } from '../../../src/context/memory/memory-agent.service.js'; interface BuiltMocks { appSettings: any; diff --git a/packages/cli/src/context/memory/memory-agent.service.test.ts b/packages/cli/test/context/memory/memory-agent.service.test.ts similarity index 98% rename from packages/cli/src/context/memory/memory-agent.service.test.ts rename to packages/cli/test/context/memory/memory-agent.service.test.ts index cea83674..ba91444e 100644 --- a/packages/cli/src/context/memory/memory-agent.service.test.ts +++ b/packages/cli/test/context/memory/memory-agent.service.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it, vi } from 'vitest'; -import { validateSingleSource } from '../../context/sl/tools/sl-warehouse-validation.js'; -import { createTouchedSlSources, hasTouchedSlSource } from '../../context/tools/touched-sl-sources.js'; -import { detectCaptureSignals, isWorthAnalyzing } from './capture-signals.js'; -import { MemoryAgentService } from './memory-agent.service.js'; +import { validateSingleSource } from '../../../src/context/sl/tools/sl-warehouse-validation.js'; +import { createTouchedSlSources, hasTouchedSlSource } from '../../../src/context/tools/touched-sl-sources.js'; +import { detectCaptureSignals, isWorthAnalyzing } from '../../../src/context/memory/capture-signals.js'; +import { MemoryAgentService } from '../../../src/context/memory/memory-agent.service.js'; const passthroughValidator = { validateSingleSource: (d: unknown, c: string, n: string) => validateSingleSource(d as never, c, n), diff --git a/packages/cli/src/context/memory/memory-runs.test.ts b/packages/cli/test/context/memory/memory-runs.test.ts similarity index 95% rename from packages/cli/src/context/memory/memory-runs.test.ts rename to packages/cli/test/context/memory/memory-runs.test.ts index e4d0d4ba..b515750e 100644 --- a/packages/cli/src/context/memory/memory-runs.test.ts +++ b/packages/cli/test/context/memory/memory-runs.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { MemoryAgentInput, MemoryAgentResult } from '../../context/memory/types.js'; -import type { MemoryAgentService } from '../../context/memory/memory-agent.service.js'; -import { MemoryIngestService, type MemoryRunStorePort } from './memory-runs.js'; +import type { MemoryAgentInput, MemoryAgentResult } from '../../../src/context/memory/types.js'; +import type { MemoryAgentService } from '../../../src/context/memory/memory-agent.service.js'; +import { MemoryIngestService, type MemoryRunStorePort } from '../../../src/context/memory/memory-runs.js'; class InMemoryRunStore implements MemoryRunStorePort { readonly rows = new Map< diff --git a/packages/cli/src/context/memory/memory-runtime-assets.test.ts b/packages/cli/test/context/memory/memory-runtime-assets.test.ts similarity index 94% rename from packages/cli/src/context/memory/memory-runtime-assets.test.ts rename to packages/cli/test/context/memory/memory-runtime-assets.test.ts index 55d9047c..ab6ff324 100644 --- a/packages/cli/src/context/memory/memory-runtime-assets.test.ts +++ b/packages/cli/test/context/memory/memory-runtime-assets.test.ts @@ -2,13 +2,13 @@ import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; -import { PromptService } from '../../context/prompts/prompt.service.js'; -import { SkillsRegistryService } from '../../context/skills/skills-registry.service.js'; -import { DEFAULT_SKILL_NAMES, promptNameFor } from '../../context/memory/capture-signals.js'; -import type { MemoryAgentSourceType } from '../../context/memory/types.js'; +import { PromptService } from '../../../src/context/prompts/prompt.service.js'; +import { SkillsRegistryService } from '../../../src/context/skills/skills-registry.service.js'; +import { DEFAULT_SKILL_NAMES, promptNameFor } from '../../../src/context/memory/capture-signals.js'; +import type { MemoryAgentSourceType } from '../../../src/context/memory/types.js'; -const promptsDir = fileURLToPath(new URL('../../prompts', import.meta.url)); -const skillsDir = fileURLToPath(new URL('../../skills', import.meta.url)); +const promptsDir = fileURLToPath(new URL('../../../src/prompts', import.meta.url)); +const skillsDir = fileURLToPath(new URL('../../../src/skills', import.meta.url)); const memorySourceTypes: MemoryAgentSourceType[] = ['research', 'external_ingest', 'backfill']; const expectedSkillHeadings: Record = { wiki_capture: '# Wiki Capture', diff --git a/packages/cli/src/context/project/config.test.ts b/packages/cli/test/context/project/config.test.ts similarity index 86% rename from packages/cli/src/context/project/config.test.ts rename to packages/cli/test/context/project/config.test.ts index 28e00f74..e5911a25 100644 --- a/packages/cli/src/context/project/config.test.ts +++ b/packages/cli/test/context/project/config.test.ts @@ -5,7 +5,7 @@ import { parseKtxProjectConfig, serializeKtxProjectConfig, validateKtxProjectConfig, -} from './config.js'; +} from '../../../src/context/project/config.js'; describe('KTX project config', () => { it.each(['status', 'replay', 'run', 'watch'])('accepts former ingest subcommand name "%s" as a connection id', (connectionId) => { @@ -50,6 +50,18 @@ connections: maxConcurrency: 1, failureMode: 'continue', }, + rateLimit: { + enabled: true, + throttleThreshold: 0.8, + minConcurrencyUnderPressure: 1, + retry: { + maxAttempts: 6, + baseDelayMs: 1_000, + maxDelayMs: 60_000, + jitter: true, + }, + }, + profile: false, }, agent: { run_research: { @@ -156,6 +168,58 @@ ingest: }); }); + it('parses the ingest.profile flag (false default, true, or "json")', () => { + expect(parseKtxProjectConfig('ingest:\n adapters: []\n').ingest.profile).toBe(false); + expect(parseKtxProjectConfig('ingest:\n profile: true\n').ingest.profile).toBe(true); + expect(parseKtxProjectConfig('ingest:\n profile: json\n').ingest.profile).toBe('json'); + }); + + it('defaults ingest rate-limit settings', () => { + const config = buildDefaultKtxProjectConfig(); + expect(config.ingest.rateLimit).toEqual({ + enabled: true, + throttleThreshold: 0.8, + minConcurrencyUnderPressure: 1, + retry: { + maxAttempts: 6, + baseDelayMs: 1_000, + maxDelayMs: 60_000, + jitter: true, + }, + }); + }); + + it('validates ingest rate-limit retry settings', () => { + const config = parseKtxProjectConfig(` +llm: + provider: + backend: none +ingest: + rateLimit: + enabled: true + throttleThreshold: 0.7 + minConcurrencyUnderPressure: 2 + maxWaitMs: 300000 + retry: + maxAttempts: 4 + baseDelayMs: 500 + maxDelayMs: 30000 + jitter: false +`); + expect(config.ingest.rateLimit).toEqual({ + enabled: true, + throttleThreshold: 0.7, + minConcurrencyUnderPressure: 2, + maxWaitMs: 300_000, + retry: { + maxAttempts: 4, + baseDelayMs: 500, + maxDelayMs: 30_000, + jitter: false, + }, + }); + }); + it('parses global Vertex LLM config', () => { const config = parseKtxProjectConfig(` llm: @@ -224,6 +288,31 @@ llm: }); }); + it('parses Codex as a first-class LLM backend', () => { + const config = parseKtxProjectConfig(` +llm: + provider: + backend: codex + models: + default: gpt-5.3-codex + triage: gpt-5.3-codex + candidateExtraction: gpt-5.3-codex + curator: gpt-5.3-codex + reconcile: gpt-5.3-codex + repair: gpt-5.3-codex +`); + + expect(config.llm.provider.backend).toBe('codex'); + expect(config.llm.models).toEqual({ + default: 'gpt-5.3-codex', + triage: 'gpt-5.3-codex', + candidateExtraction: 'gpt-5.3-codex', + curator: 'gpt-5.3-codex', + reconcile: 'gpt-5.3-codex', + repair: 'gpt-5.3-codex', + }); + }); + it('parses gateway LLM, OpenAI scan embeddings, and sentence-transformers ingest embeddings', () => { const config = parseKtxProjectConfig(` llm: @@ -523,7 +612,7 @@ describe('generateKtxProjectConfigJsonSchema', () => { const llm = (schema.properties as Record }>).llm; const provider = llm?.properties?.provider as { properties?: Record }; const backend = provider?.properties?.backend as { enum?: readonly string[] }; - expect(backend?.enum).toEqual(['none', 'anthropic', 'vertex', 'gateway', 'claude-code']); + expect(backend?.enum).toEqual(['none', 'anthropic', 'vertex', 'gateway', 'claude-code', 'codex']); const storage = (schema.properties as Record }>).storage; const state = storage?.properties?.state as { enum?: readonly string[] }; diff --git a/packages/cli/src/context/project/driver-schemas.test.ts b/packages/cli/test/context/project/driver-schemas.test.ts similarity index 94% rename from packages/cli/src/context/project/driver-schemas.test.ts rename to packages/cli/test/context/project/driver-schemas.test.ts index 252f428f..c83a27a1 100644 --- a/packages/cli/src/context/project/driver-schemas.test.ts +++ b/packages/cli/test/context/project/driver-schemas.test.ts @@ -1,10 +1,9 @@ import { describe, expect, it } from 'vitest'; -import { connectionConfigSchema } from './driver-schemas.js'; +import { connectionConfigSchema } from '../../../src/context/project/driver-schemas.js'; describe('connectionConfigSchema (driver discriminated union)', () => { it.each([ ['postgres', 'postgres://user:pass@host:5432/db'], // pragma: allowlist secret - ['postgresql', 'postgresql://user:pass@host:5432/db'], // pragma: allowlist secret ['mysql', 'mysql://user:pass@host:3306/db'], // pragma: allowlist secret ['snowflake', 'snowflake://account/db'], ['bigquery', 'bigquery://project/dataset'], @@ -32,6 +31,10 @@ describe('connectionConfigSchema (driver discriminated union)', () => { it('rejects an unknown driver', () => { expect(() => connectionConfigSchema.parse({ driver: 'nope', url: 'x' })).toThrow(); }); + + it('rejects legacy warehouse driver aliases', () => { + expect(() => connectionConfigSchema.parse({ driver: 'postgresql', url: 'postgresql://host/db' })).toThrow(); + }); }); describe('connectionConfigSchema - context source drivers with mappings', () => { diff --git a/packages/cli/src/context/project/local-git-file-store.test.ts b/packages/cli/test/context/project/local-git-file-store.test.ts similarity index 94% rename from packages/cli/src/context/project/local-git-file-store.test.ts rename to packages/cli/test/context/project/local-git-file-store.test.ts index 1bee3c1e..9a7a6948 100644 --- a/packages/cli/src/context/project/local-git-file-store.test.ts +++ b/packages/cli/test/context/project/local-git-file-store.test.ts @@ -2,9 +2,9 @@ import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { GitService } from '../../context/core/git.service.js'; -import type { KtxCoreConfig } from '../../context/core/config.js'; -import { LocalGitFileStore } from './local-git-file-store.js'; +import { GitService } from '../../../src/context/core/git.service.js'; +import type { KtxCoreConfig } from '../../../src/context/core/config.js'; +import { LocalGitFileStore } from '../../../src/context/project/local-git-file-store.js'; describe('LocalGitFileStore', () => { let tempDir: string; diff --git a/packages/cli/src/context/project/mappings-yaml-schema.test.ts b/packages/cli/test/context/project/mappings-yaml-schema.test.ts similarity index 98% rename from packages/cli/src/context/project/mappings-yaml-schema.test.ts rename to packages/cli/test/context/project/mappings-yaml-schema.test.ts index f7001a70..917cd808 100644 --- a/packages/cli/src/context/project/mappings-yaml-schema.test.ts +++ b/packages/cli/test/context/project/mappings-yaml-schema.test.ts @@ -7,7 +7,7 @@ import { parseLookmlMappingBootstrap, parseLookerMappingBootstrap, parseMetabaseMappingBootstrap, -} from './mappings-yaml-schema.js'; +} from '../../../src/context/project/mappings-yaml-schema.js'; describe('ktx.yaml mapping bootstrap schema', () => { it('parses Metabase mapping intent with CLI syncMode default ALL', () => { diff --git a/packages/cli/src/context/project/project.test.ts b/packages/cli/test/context/project/project.test.ts similarity index 96% rename from packages/cli/src/context/project/project.test.ts rename to packages/cli/test/context/project/project.test.ts index 21e27d6a..668fa264 100644 --- a/packages/cli/src/context/project/project.test.ts +++ b/packages/cli/test/context/project/project.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, loadKtxProject } from './project.js'; +import { initKtxProject, loadKtxProject } from '../../../src/context/project/project.js'; describe('KTX local project runtime', () => { let tempDir: string; diff --git a/packages/cli/src/context/project/setup-config.test.ts b/packages/cli/test/context/project/setup-config.test.ts similarity index 93% rename from packages/cli/src/context/project/setup-config.test.ts rename to packages/cli/test/context/project/setup-config.test.ts index 88c5376e..948d9d54 100644 --- a/packages/cli/src/context/project/setup-config.test.ts +++ b/packages/cli/test/context/project/setup-config.test.ts @@ -2,13 +2,13 @@ import { mkdtemp, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { buildDefaultKtxProjectConfig } from './config.js'; +import { buildDefaultKtxProjectConfig } from '../../../src/context/project/config.js'; import { markKtxSetupStateStepComplete, mergeKtxSetupGitignoreEntries, readKtxSetupState, setKtxSetupDatabaseConnectionIds, -} from './setup-config.js'; +} from '../../../src/context/project/setup-config.js'; describe('KTX setup config helpers', () => { let tempDir: string; diff --git a/packages/cli/src/context/prompts/prompt.service.test.ts b/packages/cli/test/context/prompts/prompt.service.test.ts similarity index 95% rename from packages/cli/src/context/prompts/prompt.service.test.ts rename to packages/cli/test/context/prompts/prompt.service.test.ts index 046b777b..df2e407c 100644 --- a/packages/cli/src/context/prompts/prompt.service.test.ts +++ b/packages/cli/test/context/prompts/prompt.service.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { PromptService } from './prompt.service.js'; +import { PromptService } from '../../../src/context/prompts/prompt.service.js'; describe('PromptService', () => { let dir: string; diff --git a/packages/cli/test/context/scan/constraint-discovery.test.ts b/packages/cli/test/context/scan/constraint-discovery.test.ts new file mode 100644 index 00000000..0a06f1f1 --- /dev/null +++ b/packages/cli/test/context/scan/constraint-discovery.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest'; +import { constraintDiscoveryWarning, tryConstraintQuery } from '../../../src/context/scan/constraint-discovery.js'; + +describe('tryConstraintQuery', () => { + it('returns the query value when the query succeeds', async () => { + await expect( + tryConstraintQuery( + { + schema: 'public', + kind: 'primary_key', + isDeniedError: () => false, + }, + async () => ['id'], + ), + ).resolves.toEqual({ ok: true, value: ['id'] }); + }); + + it('returns a recoverable warning when the classifier recognizes denial', async () => { + const error = Object.assign(new Error('permission denied'), { code: '42501' }); + + await expect( + tryConstraintQuery( + { + schema: 'analytics', + kind: 'foreign_key', + isDeniedError: (candidate) => candidate === error, + }, + async () => { + throw error; + }, + ), + ).resolves.toEqual({ + ok: false, + warning: { + code: 'constraint_discovery_unauthorized', + message: 'Skipped foreign-key discovery in analytics (insufficient grants on system catalogs)', + recoverable: true, + metadata: { schema: 'analytics', kind: 'foreign_key' }, + }, + }); + }); + + it('rethrows non-denial errors unchanged', async () => { + const error = Object.assign(new Error('connection reset'), { code: 'ECONNRESET' }); + + await expect( + tryConstraintQuery( + { + schema: 'public', + kind: 'primary_key', + isDeniedError: () => false, + }, + async () => { + throw error; + }, + ), + ).rejects.toBe(error); + }); +}); + +describe('constraintDiscoveryWarning', () => { + it('formats stable primary-key warning text and metadata', () => { + expect(constraintDiscoveryWarning({ schema: 'public', kind: 'primary_key' })).toEqual({ + code: 'constraint_discovery_unauthorized', + message: 'Skipped primary-key discovery in public (insufficient grants on system catalogs)', + recoverable: true, + metadata: { schema: 'public', kind: 'primary_key' }, + }); + }); +}); diff --git a/packages/cli/src/context/scan/credentials.test.ts b/packages/cli/test/context/scan/credentials.test.ts similarity index 96% rename from packages/cli/src/context/scan/credentials.test.ts rename to packages/cli/test/context/scan/credentials.test.ts index 891c58a9..62ee2952 100644 --- a/packages/cli/src/context/scan/credentials.test.ts +++ b/packages/cli/test/context/scan/credentials.test.ts @@ -1,13 +1,13 @@ import { describe, expect, it } from 'vitest'; -import { REDACTED_KTX_CREDENTIAL_VALUE } from '../core/redaction.js'; +import { REDACTED_KTX_CREDENTIAL_VALUE } from '../../../src/context/core/redaction.js'; import { redactKtxCredentialEnvelope, redactKtxCredentialValue, redactKtxScanMetadata, redactKtxScanReport, redactKtxScanWarning, -} from './credentials.js'; -import type { KtxCredentialEnvelope, KtxScanReport, KtxScanWarning } from './types.js'; +} from '../../../src/context/scan/credentials.js'; +import type { KtxCredentialEnvelope, KtxScanReport, KtxScanWarning } from '../../../src/context/scan/types.js'; describe('KTX scan credential redaction', () => { it('keeps credential references inspectable', () => { diff --git a/packages/cli/src/context/scan/data-dictionary.test.ts b/packages/cli/test/context/scan/data-dictionary.test.ts similarity index 99% rename from packages/cli/src/context/scan/data-dictionary.test.ts rename to packages/cli/test/context/scan/data-dictionary.test.ts index b8b39376..daf20559 100644 --- a/packages/cli/src/context/scan/data-dictionary.test.ts +++ b/packages/cli/test/context/scan/data-dictionary.test.ts @@ -3,7 +3,7 @@ import { defaultKtxDataDictionarySettings, isKtxDataDictionaryCandidate, shouldKtxSampleColumnForDictionary, -} from './data-dictionary.js'; +} from '../../../src/context/scan/data-dictionary.js'; const defaultPatterns = defaultKtxDataDictionarySettings.excludePatterns; diff --git a/packages/cli/src/context/scan/description-generation.test.ts b/packages/cli/test/context/scan/description-generation.test.ts similarity index 99% rename from packages/cli/src/context/scan/description-generation.test.ts rename to packages/cli/test/context/scan/description-generation.test.ts index bc7b1e25..811752e5 100644 --- a/packages/cli/src/context/scan/description-generation.test.ts +++ b/packages/cli/test/context/scan/description-generation.test.ts @@ -12,8 +12,8 @@ import { buildKtxTableDescriptionPrompt, type KtxDescriptionCachePort, KtxDescriptionGenerator, -} from './description-generation.js'; -import { createKtxConnectorCapabilities, type KtxScanConnector } from './types.js'; +} from '../../../src/context/scan/description-generation.js'; +import { createKtxConnectorCapabilities, type KtxScanConnector } from '../../../src/context/scan/types.js'; function createCache(initial: Record = {}): KtxDescriptionCachePort { const data = new Map(Object.entries(initial)); @@ -72,6 +72,8 @@ function createConnector(): KtxScanConnector { introspect: vi.fn(async () => { throw new Error('introspection is not used by description generation'); }), + listSchemas: vi.fn(async () => []), + listTables: vi.fn(async () => []), sampleColumn: vi.fn(async () => ({ values: ['paid', 'refunded', null], nullCount: 1, diff --git a/packages/cli/src/context/scan/embedding-text.test.ts b/packages/cli/test/context/scan/embedding-text.test.ts similarity index 94% rename from packages/cli/src/context/scan/embedding-text.test.ts rename to packages/cli/test/context/scan/embedding-text.test.ts index ee019bce..523d1d5c 100644 --- a/packages/cli/src/context/scan/embedding-text.test.ts +++ b/packages/cli/test/context/scan/embedding-text.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { buildKtxColumnEmbeddingText } from './embedding-text.js'; +import { buildKtxColumnEmbeddingText } from '../../../src/context/scan/embedding-text.js'; describe('KTX scan embedding text', () => { it('builds column embedding text with table, description, FK, and sample-value context', () => { diff --git a/packages/cli/src/context/scan/enrichment-state.test.ts b/packages/cli/test/context/scan/enrichment-state.test.ts similarity index 95% rename from packages/cli/src/context/scan/enrichment-state.test.ts rename to packages/cli/test/context/scan/enrichment-state.test.ts index 4ae597c6..24b4bae3 100644 --- a/packages/cli/src/context/scan/enrichment-state.test.ts +++ b/packages/cli/test/context/scan/enrichment-state.test.ts @@ -6,9 +6,9 @@ import { completedKtxScanEnrichmentStateSummary, computeKtxScanEnrichmentInputHash, summarizeKtxScanEnrichmentState, -} from './enrichment-state.js'; -import { SqliteLocalScanEnrichmentStateStore } from './sqlite-local-enrichment-state-store.js'; -import type { KtxSchemaSnapshot } from './types.js'; +} from '../../../src/context/scan/enrichment-state.js'; +import { SqliteLocalScanEnrichmentStateStore } from '../../../src/context/scan/sqlite-local-enrichment-state-store.js'; +import type { KtxSchemaSnapshot } from '../../../src/context/scan/types.js'; const snapshot: KtxSchemaSnapshot = { connectionId: 'warehouse', diff --git a/packages/cli/src/context/scan/enrichment-summary.test.ts b/packages/cli/test/context/scan/enrichment-summary.test.ts similarity index 96% rename from packages/cli/src/context/scan/enrichment-summary.test.ts rename to packages/cli/test/context/scan/enrichment-summary.test.ts index f320046b..783e18cd 100644 --- a/packages/cli/src/context/scan/enrichment-summary.test.ts +++ b/packages/cli/test/context/scan/enrichment-summary.test.ts @@ -3,7 +3,7 @@ import { failedKtxScanEnrichmentSummary, ktxScanErrorMessage, skippedKtxScanEnrichmentSummary, -} from './enrichment-summary.js'; +} from '../../../src/context/scan/enrichment-summary.js'; describe('KTX scan enrichment summaries', () => { it('keeps structural scans skipped when no enrichment was requested', () => { diff --git a/packages/cli/src/context/scan/enrichment-types.test.ts b/packages/cli/test/context/scan/enrichment-types.test.ts similarity index 99% rename from packages/cli/src/context/scan/enrichment-types.test.ts rename to packages/cli/test/context/scan/enrichment-types.test.ts index 3f7828dc..72e7b247 100644 --- a/packages/cli/src/context/scan/enrichment-types.test.ts +++ b/packages/cli/test/context/scan/enrichment-types.test.ts @@ -9,7 +9,7 @@ import type { KtxRelationshipUpdate, KtxScanMetadataStore, KtxStructuralSyncPlan, -} from './enrichment-types.js'; +} from '../../../src/context/scan/enrichment-types.js'; describe('KTX scan enrichment contracts', () => { it('models an enriched schema with reusable table, column, and relationship metadata', () => { diff --git a/packages/cli/src/context/scan/entity-details.test.ts b/packages/cli/test/context/scan/entity-details.test.ts similarity index 91% rename from packages/cli/src/context/scan/entity-details.test.ts rename to packages/cli/test/context/scan/entity-details.test.ts index ddccef87..ea8d01b7 100644 --- a/packages/cli/src/context/scan/entity-details.test.ts +++ b/packages/cli/test/context/scan/entity-details.test.ts @@ -2,9 +2,9 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; -import { createKtxEntityDetailsService } from './entity-details.js'; -import type { KtxConnectionDriver, KtxScanReport, KtxSchemaTable } from './types.js'; +import { initKtxProject, type KtxLocalProject } from '../../../src/context/project/project.js'; +import { createKtxEntityDetailsService } from '../../../src/context/scan/entity-details.js'; +import type { KtxConnectionDriver, KtxScanReport, KtxSchemaTable } from '../../../src/context/scan/types.js'; describe('createKtxEntityDetailsService', () => { let tempDir: string; @@ -201,6 +201,22 @@ describe('createKtxEntityDetailsService', () => { }); }); + it('resolves quoted qualified display strings through the dialect parser', async () => { + await seedScan({ syncId: 'sync-1', runId: 'scan-1' }); + const service = createKtxEntityDetailsService(project); + + const result = await service.read({ + connectionId: 'warehouse', + entities: [{ table: '"public"."orders"' }], + }); + + expect(result.results[0]).toMatchObject({ + ok: true, + display: 'public.orders', + tableRef: { catalog: null, db: 'public', name: 'orders' }, + }); + }); + it('filters requested columns while keeping full-table foreign keys', async () => { await seedScan({ syncId: 'sync-1', runId: 'scan-1' }); const service = createKtxEntityDetailsService(project); diff --git a/packages/cli/src/context/scan/local-enrichment-artifacts.test.ts b/packages/cli/test/context/scan/local-enrichment-artifacts.test.ts similarity index 98% rename from packages/cli/src/context/scan/local-enrichment-artifacts.test.ts rename to packages/cli/test/context/scan/local-enrichment-artifacts.test.ts index 8a49fc78..638bafb2 100644 --- a/packages/cli/src/context/scan/local-enrichment-artifacts.test.ts +++ b/packages/cli/test/context/scan/local-enrichment-artifacts.test.ts @@ -3,10 +3,10 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import YAML from 'yaml'; -import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; -import type { KtxLocalScanEnrichmentResult } from './local-enrichment.js'; -import { writeLocalScanEnrichmentArtifacts, writeLocalScanManifestShards } from './local-enrichment-artifacts.js'; -import type { KtxSchemaSnapshot } from './types.js'; +import { initKtxProject, type KtxLocalProject } from '../../../src/context/project/project.js'; +import type { KtxLocalScanEnrichmentResult } from '../../../src/context/scan/local-enrichment.js'; +import { writeLocalScanEnrichmentArtifacts, writeLocalScanManifestShards } from '../../../src/context/scan/local-enrichment-artifacts.js'; +import type { KtxSchemaSnapshot } from '../../../src/context/scan/types.js'; const snapshot: KtxSchemaSnapshot = { connectionId: 'warehouse', diff --git a/packages/cli/src/context/scan/local-enrichment.test.ts b/packages/cli/test/context/scan/local-enrichment.test.ts similarity index 96% rename from packages/cli/src/context/scan/local-enrichment.test.ts rename to packages/cli/test/context/scan/local-enrichment.test.ts index 9647c8b9..9704d071 100644 --- a/packages/cli/src/context/scan/local-enrichment.test.ts +++ b/packages/cli/test/context/scan/local-enrichment.test.ts @@ -1,17 +1,17 @@ import Database from 'better-sqlite3'; import { describe, expect, it, vi } from 'vitest'; -import { buildDefaultKtxProjectConfig } from '../project/config.js'; +import { buildDefaultKtxProjectConfig } from '../../../src/context/project/config.js'; import type { KtxScanEnrichmentCompletedStage, KtxScanEnrichmentFailedStage, KtxScanEnrichmentStageLookup, KtxScanEnrichmentStateStore, -} from './enrichment-state.js'; +} from '../../../src/context/scan/enrichment-state.js'; import { createDeterministicLocalScanEnrichmentProviders, runLocalScanEnrichment, snapshotToKtxEnrichedSchema, -} from './local-enrichment.js'; +} from '../../../src/context/scan/local-enrichment.js'; import { createKtxConnectorCapabilities, type KtxQueryResult, @@ -20,7 +20,7 @@ import { type KtxScanConnector, type KtxScanContext, type KtxSchemaSnapshot, -} from './types.js'; +} from '../../../src/context/scan/types.js'; function fakeScanEmbedding(options: { dimensions: number; maxBatchSize?: number }): KtxEmbeddingPort { return { @@ -104,6 +104,8 @@ function connector(): KtxScanConnector { columnStats: true, }), introspect: vi.fn(async () => snapshot), + listSchemas: vi.fn(async () => []), + listTables: vi.fn(async () => []), sampleTable: vi.fn(async () => ({ headers: ['id', 'customer_id'], rows: [[1, 10]], @@ -331,6 +333,27 @@ describe('local scan enrichment', () => { expect(scanConnector.introspect).toHaveBeenCalledTimes(1); }); + it('fails when connector driver and snapshot driver differ', async () => { + const mismatchedConnector: KtxScanConnector = { + ...connector(), + driver: 'mysql', + }; + + await expect( + runLocalScanEnrichment({ + connectionId: 'warehouse', + mode: 'relationships', + detectRelationships: true, + connector: mismatchedConnector, + snapshot, + context: { runId: 'scan-run-driver-mismatch' }, + providers: null, + }), + ).rejects.toThrow( + 'ktx scan connector driver "mysql" does not match snapshot driver "postgres" for connection "warehouse"', + ); + }); + it('runs deterministic relationship detection for relationship scans', async () => { const result = await runLocalScanEnrichment({ connectionId: 'warehouse', diff --git a/packages/cli/src/context/scan/local-scan.test.ts b/packages/cli/test/context/scan/local-scan.test.ts similarity index 95% rename from packages/cli/src/context/scan/local-scan.test.ts rename to packages/cli/test/context/scan/local-scan.test.ts index 7b5af5b0..931fd6b0 100644 --- a/packages/cli/src/context/scan/local-scan.test.ts +++ b/packages/cli/test/context/scan/local-scan.test.ts @@ -3,18 +3,23 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import YAML from 'yaml'; -import type { SourceAdapter } from '../../context/ingest/types.js'; -import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; -import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js'; -import { resolveEnabledTables } from './enabled-tables.js'; -import { getLocalScanReport, getLocalScanStatus, runLocalScan } from './local-scan.js'; -import { tableRefKey, tableRefSet, type KtxTableRefKey } from './table-ref.js'; +import type { SourceAdapter } from '../../../src/context/ingest/types.js'; +import type { KtxLlmRuntimePort } from '../../../src/context/llm/runtime-port.js'; +import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../../src/context/project/project.js'; +import { resolveEnabledTables } from '../../../src/context/scan/enabled-tables.js'; +import { getLocalScanReport, getLocalScanStatus, runLocalScan } from '../../../src/context/scan/local-scan.js'; +import { tableRefKey, tableRefSet, type KtxTableRefKey } from '../../../src/context/scan/table-ref.js'; import type { KtxQueryResult, KtxReadOnlyQueryInput, KtxScanConnector, KtxSchemaSnapshot, -} from './types.js'; +} from '../../../src/context/scan/types.js'; + +const connectorScopeListing = { + listSchemas: vi.fn(async () => []), + listTables: vi.fn(async () => []), +}; function relationshipSqlResult( input: KtxReadOnlyQueryInput, @@ -180,6 +185,13 @@ function fetchOnlyAdapter(options: { extractedAt?: () => string; snapshot?: KtxS 'utf-8', ); await writeFile(join(stagedDir, 'foreign-keys.json'), '{"foreignKeys":[]}\n', 'utf-8'); + if (scanSnapshot.warnings?.length) { + await writeFile( + join(stagedDir, 'warnings.json'), + `${JSON.stringify({ warnings: scanSnapshot.warnings })}\n`, + 'utf-8', + ); + } for (const table of scanSnapshot.tables) { await writeFile(join(stagedDir, 'tables', `${table.name}.json`), `${JSON.stringify(table)}\n`, 'utf-8'); } @@ -247,6 +259,7 @@ function nativeScanConnector(options: { cleanup?: () => Promise } = {}): K formalForeignKeys: false, estimatedRowCounts: false, }, + ...connectorScopeListing, introspect: vi.fn(async () => nativeScanSnapshot()), sampleTable: vi.fn(async () => ({ headers: ['id'], rows: [[1]], totalRows: 1 })), sampleColumn: vi.fn(async () => ({ values: ['1'], nullCount: 0, distinctCount: 1 })), @@ -336,6 +349,48 @@ describe('local scan', () => { }); }); + it('threads structural snapshot warnings into the final scan report', async () => { + const result = await runLocalScan({ + project, + adapters: [ + fetchOnlyAdapter({ + snapshot: { + ...defaultFetchSnapshot(), + warnings: [ + { + code: 'constraint_discovery_unauthorized', + message: 'Skipped primary-key discovery in public (insufficient grants on system catalogs)', + recoverable: true, + metadata: { schema: 'public', kind: 'primary_key' }, + }, + ], + }, + }), + ], + connectionId: 'warehouse', + jobId: 'scan-run-structural-warnings', + now: () => new Date('2026-04-29T09:01:00.000Z'), + }); + + expect(result.report.warnings).toEqual([ + { + code: 'constraint_discovery_unauthorized', + message: 'Skipped primary-key discovery in public (insufficient grants on system catalogs)', + recoverable: true, + metadata: { schema: 'public', kind: 'primary_key' }, + }, + ]); + await expect( + readFile( + join( + project.projectDir, + 'raw-sources/warehouse/live-database/2026-04-29-090100-scan-run-structural-warnings/scan-report.json', + ), + 'utf-8', + ), + ).resolves.toContain('"constraint_discovery_unauthorized"'); + }); + it('passes enabled_tables as fetch context tableScope and does not post-filter staged snapshots', async () => { project.config.connections.warehouse = { ...project.config.connections.warehouse, @@ -607,6 +662,7 @@ describe('local scan', () => { formalForeignKeys: false, estimatedRowCounts: false, }, + ...connectorScopeListing, async introspect() { return { connectionId: 'warehouse', @@ -692,6 +748,7 @@ describe('local scan', () => { formalForeignKeys: false, estimatedRowCounts: true, }, + ...connectorScopeListing, async introspect() { return { connectionId: 'warehouse', @@ -881,6 +938,14 @@ describe('local scan', () => { }; } + async listSchemas(): Promise { + return []; + } + + async listTables() { + return []; + } + async executeReadOnly(input: KtxReadOnlyQueryInput): Promise { return relationshipSqlResult(input); } @@ -923,6 +988,7 @@ describe('local scan', () => { formalForeignKeys: false, estimatedRowCounts: true, }, + ...connectorScopeListing, async introspect() { return { connectionId: 'warehouse', @@ -1024,6 +1090,7 @@ describe('local scan', () => { formalForeignKeys: false, estimatedRowCounts: true, }, + ...connectorScopeListing, async introspect() { return { connectionId: 'warehouse', @@ -1151,6 +1218,7 @@ describe('local scan', () => { formalForeignKeys: false, estimatedRowCounts: true, }, + ...connectorScopeListing, async introspect() { return { connectionId: 'warehouse', @@ -1291,6 +1359,7 @@ describe('local scan', () => { formalForeignKeys: false, estimatedRowCounts: true, }, + ...connectorScopeListing, async introspect() { return { connectionId: 'warehouse', @@ -1406,6 +1475,7 @@ describe('local scan', () => { formalForeignKeys: false, estimatedRowCounts: false, }, + ...connectorScopeListing, async introspect() { return { connectionId: 'warehouse', @@ -1501,6 +1571,7 @@ describe('local scan', () => { formalForeignKeys: false, estimatedRowCounts: false, }, + ...connectorScopeListing, async introspect() { return { connectionId: 'warehouse', @@ -1617,6 +1688,7 @@ describe('local scan', () => { formalForeignKeys: false, estimatedRowCounts: false, }, + ...connectorScopeListing, async introspect() { return { connectionId: 'warehouse', @@ -1878,6 +1950,15 @@ describe('resolveEnabledTables', () => { expect(result!.has(tableRefKey({ catalog: null, db: 'public', name: 'orders' }))).toBe(true); }); + it('ignores legacy enabled_tables object entries', () => { + expect( + resolveEnabledTables({ + driver: 'postgres', + enabled_tables: [{ catalog: null, db: 'public', name: 'orders' }], + }), + ).toBeNull(); + }); + it('returns null for undefined connection', () => { expect(resolveEnabledTables(undefined)).toBeNull(); }); diff --git a/packages/cli/src/context/scan/local-structural-artifacts.test.ts b/packages/cli/test/context/scan/local-structural-artifacts.test.ts similarity index 63% rename from packages/cli/src/context/scan/local-structural-artifacts.test.ts rename to packages/cli/test/context/scan/local-structural-artifacts.test.ts index 71019597..519bb26d 100644 --- a/packages/cli/src/context/scan/local-structural-artifacts.test.ts +++ b/packages/cli/test/context/scan/local-structural-artifacts.test.ts @@ -2,8 +2,8 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; -import { readLocalScanStructuralSnapshot } from './local-structural-artifacts.js'; +import { initKtxProject, type KtxLocalProject } from '../../../src/context/project/project.js'; +import { readLocalScanStructuralSnapshot } from '../../../src/context/scan/local-structural-artifacts.js'; describe('readLocalScanStructuralSnapshot', () => { let tempDir: string; @@ -165,6 +165,61 @@ describe('readLocalScanStructuralSnapshot', () => { }); }); + it('rebuilds scan warnings from persisted live-database warning files', async () => { + const rawRoot = 'raw-sources/warehouse/live-database/sync-warnings'; + await project.fileStore.writeFile( + `${rawRoot}/connection.json`, + '{"connectionId":"warehouse","metadata":{}}\n', + 'ktx', + 'ktx@example.com', + 'Seed connection artifact', + ); + await project.fileStore.writeFile( + `${rawRoot}/warnings.json`, + `${JSON.stringify( + { + warnings: [ + { + code: 'constraint_discovery_unauthorized', + message: 'Skipped foreign-key discovery in public (insufficient grants on system catalogs)', + recoverable: true, + metadata: { schema: 'public', kind: 'foreign_key' }, + }, + ], + }, + null, + 2, + )}\n`, + 'ktx', + 'ktx@example.com', + 'Seed warning artifact', + ); + await project.fileStore.writeFile( + `${rawRoot}/tables/orders.json`, + '{"name":"orders","catalog":null,"db":"public","kind":"table","comment":null,"estimatedRows":null,"columns":[{"name":"id","nativeType":"integer","normalizedType":"integer","dimensionType":"number","nullable":false,"primaryKey":false,"comment":null}],"foreignKeys":[]}\n', + 'ktx', + 'ktx@example.com', + 'Seed orders artifact', + ); + + const snapshot = await readLocalScanStructuralSnapshot({ + project, + connectionId: 'warehouse', + driver: 'postgres', + rawSourcesDir: rawRoot, + extractedAtFallback: '2026-04-29T13:00:00.000Z', + }); + + expect(snapshot.warnings).toEqual([ + { + code: 'constraint_discovery_unauthorized', + message: 'Skipped foreign-key discovery in public (insufficient grants on system catalogs)', + recoverable: true, + metadata: { schema: 'public', kind: 'foreign_key' }, + }, + ]); + }); + it('uses the scan report timestamp when connection.json omits extractedAt', async () => { const rawRoot = 'raw-sources/warehouse/live-database/sync-2'; await project.fileStore.writeFile( @@ -192,4 +247,32 @@ describe('readLocalScanStructuralSnapshot', () => { expect(snapshot.extractedAt).toBe('2026-04-29T13:00:00.000Z'); }); + + it('tolerates older live-database staged directories without warnings.json', async () => { + const rawRoot = 'raw-sources/warehouse/live-database/sync-no-warnings'; + await project.fileStore.writeFile( + `${rawRoot}/connection.json`, + '{"connectionId":"warehouse","metadata":{}}\n', + 'ktx', + 'ktx@example.com', + 'Seed connection artifact', + ); + await project.fileStore.writeFile( + `${rawRoot}/tables/orders.json`, + '{"name":"orders","catalog":null,"db":null,"kind":"table","comment":null,"estimatedRows":null,"columns":[{"name":"id","nativeType":"integer","normalizedType":"integer","dimensionType":"number","nullable":false,"primaryKey":true,"comment":null}],"foreignKeys":[]}\n', + 'ktx', + 'ktx@example.com', + 'Seed orders artifact', + ); + + const snapshot = await readLocalScanStructuralSnapshot({ + project, + connectionId: 'warehouse', + driver: 'postgres', + rawSourcesDir: rawRoot, + extractedAtFallback: '2026-04-29T13:00:00.000Z', + }); + + expect(snapshot.warnings).toEqual([]); + }); }); diff --git a/packages/cli/src/context/scan/relationship-benchmark-report.test.ts b/packages/cli/test/context/scan/relationship-benchmark-report.test.ts similarity index 99% rename from packages/cli/src/context/scan/relationship-benchmark-report.test.ts rename to packages/cli/test/context/scan/relationship-benchmark-report.test.ts index 8941eec1..0837ed3c 100644 --- a/packages/cli/src/context/scan/relationship-benchmark-report.test.ts +++ b/packages/cli/test/context/scan/relationship-benchmark-report.test.ts @@ -2,12 +2,12 @@ import { describe, expect, it } from 'vitest'; import { buildKtxRelationshipBenchmarkReport, formatKtxRelationshipBenchmarkReportMarkdown, -} from './relationship-benchmark-report.js'; +} from '../../../src/context/scan/relationship-benchmark-report.js'; import type { KtxRelationshipBenchmarkCaseResult, KtxRelationshipBenchmarkFixture, KtxRelationshipBenchmarkSuiteResult, -} from './relationship-benchmarks.js'; +} from '../../../src/context/scan/relationship-benchmarks.js'; type CaseResultOverrides = Omit, 'metrics'> & { metrics?: Partial; diff --git a/packages/cli/src/context/scan/relationship-benchmarks.test.ts b/packages/cli/test/context/scan/relationship-benchmarks.test.ts similarity index 96% rename from packages/cli/src/context/scan/relationship-benchmarks.test.ts rename to packages/cli/test/context/scan/relationship-benchmarks.test.ts index aff025aa..daaa2142 100644 --- a/packages/cli/src/context/scan/relationship-benchmarks.test.ts +++ b/packages/cli/test/context/scan/relationship-benchmarks.test.ts @@ -5,7 +5,7 @@ import { describe, expect, it } from 'vitest'; import type { KtxRelationshipBenchmarkExpectedLinks, KtxRelationshipBenchmarkFixture, -} from './relationship-benchmarks.js'; +} from '../../../src/context/scan/relationship-benchmarks.js'; import { currentKtxRelationshipBenchmarkDetector, loadKtxRelationshipBenchmarkFixture, @@ -13,8 +13,8 @@ import { maskKtxRelationshipBenchmarkSnapshot, runKtxRelationshipBenchmarkCase, runKtxRelationshipBenchmarkSuite, -} from './relationship-benchmarks.js'; -import type { KtxSchemaSnapshot } from './types.js'; +} from '../../../src/context/scan/relationship-benchmarks.js'; +import type { KtxSchemaSnapshot } from '../../../src/context/scan/types.js'; const EXPECTED_LINKS: KtxRelationshipBenchmarkExpectedLinks = { expectedPks: [ @@ -140,7 +140,7 @@ function snapshot(): KtxSchemaSnapshot { describe('relationship benchmarks', () => { it('keeps the current benchmark detector on the relationship-discovery path only', async () => { - const source = await readFile(new URL('relationship-benchmarks.ts', import.meta.url), 'utf-8'); + const source = await readFile(new URL('../../../src/context/scan/relationship-benchmarks.ts', import.meta.url), 'utf-8'); expect(source).not.toMatch(/KtxRelationshipDetector/); expect(source).not.toMatch(/relationship-detection\.js/); @@ -261,7 +261,7 @@ describe('relationship benchmarks', () => { }); it('loads the composite-key fixture and accepts composite ground truth as headline evidence', async () => { - const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url); + const fixtureRoot = new URL('../../fixtures/relationship-benchmarks', import.meta.url); const fixture = await loadKtxRelationshipBenchmarkFixture( join(fixtureRoot.pathname, 'composite_keys_no_declared_constraints'), ); @@ -586,7 +586,7 @@ describe('relationship benchmarks', () => { }); it('loads every checked-in relationship benchmark fixture with explicit provenance', async () => { - const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url); + const fixtureRoot = new URL('../../fixtures/relationship-benchmarks', import.meta.url); const fixtureDirs = (await readdir(fixtureRoot, { withFileTypes: true })) .filter((entry) => entry.isDirectory()) .map((entry) => entry.name) @@ -601,7 +601,7 @@ describe('relationship benchmarks', () => { }); it('loads May 8 evidence-fusion adversarial fixtures as reported synthetic evidence', async () => { - const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url); + const fixtureRoot = new URL('../../fixtures/relationship-benchmarks', import.meta.url); const fixtures = await loadKtxRelationshipBenchmarkFixtures(fixtureRoot.pathname); const byId = new Map(fixtures.map((fixture) => [fixture.id, fixture])); const adversarialIds = [ @@ -634,7 +634,7 @@ describe('relationship benchmarks', () => { }); it('loads the May 8 scale stress fixture with bounded benchmark validation', async () => { - const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url); + const fixtureRoot = new URL('../../fixtures/relationship-benchmarks', import.meta.url); const fixture = await loadKtxRelationshipBenchmarkFixture( join(fixtureRoot.pathname, 'scale_stress_no_declared_constraints'), ); @@ -651,7 +651,7 @@ describe('relationship benchmarks', () => { }); adHocRelationshipBenchmarkIt('runs the scale stress fixture inside the benchmark validation budget', async () => { - const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url); + const fixtureRoot = new URL('../../fixtures/relationship-benchmarks', import.meta.url); const fixture = await loadKtxRelationshipBenchmarkFixture( join(fixtureRoot.pathname, 'scale_stress_no_declared_constraints'), ); @@ -846,7 +846,7 @@ describe('relationship benchmarks', () => { }); it('loads the packaged B2B demo fixtures and records the current relationship-discovery baseline', async () => { - const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url); + const fixtureRoot = new URL('../../fixtures/relationship-benchmarks', import.meta.url); const declared = await loadKtxRelationshipBenchmarkFixture( join(fixtureRoot.pathname, 'demo_b2b_declared_metadata'), ); @@ -927,7 +927,7 @@ describe('relationship benchmarks', () => { }); it('loads the public Chinook benchmark fixture with declared metadata', async () => { - const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url); + const fixtureRoot = new URL('../../fixtures/relationship-benchmarks', import.meta.url); const fixture = await loadKtxRelationshipBenchmarkFixture( join(fixtureRoot.pathname, 'chinook_with_declared_metadata'), ); @@ -945,7 +945,7 @@ describe('relationship benchmarks', () => { }); it('loads the public Northwind benchmark fixture with declared metadata', async () => { - const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url); + const fixtureRoot = new URL('../../fixtures/relationship-benchmarks', import.meta.url); const fixture = await loadKtxRelationshipBenchmarkFixture( join(fixtureRoot.pathname, 'northwind_with_declared_metadata'), ); @@ -961,7 +961,7 @@ describe('relationship benchmarks', () => { }); it('loads the public Sakila benchmark fixture with declared metadata', async () => { - const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url); + const fixtureRoot = new URL('../../fixtures/relationship-benchmarks', import.meta.url); const fixture = await loadKtxRelationshipBenchmarkFixture( join(fixtureRoot.pathname, 'sakila_with_declared_metadata'), ); @@ -977,7 +977,7 @@ describe('relationship benchmarks', () => { }); it('loads the public AdventureWorksLT benchmark fixture with declared metadata', async () => { - const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url); + const fixtureRoot = new URL('../../fixtures/relationship-benchmarks', import.meta.url); const fixture = await loadKtxRelationshipBenchmarkFixture( join(fixtureRoot.pathname, 'adventureworkslt_with_declared_metadata'), ); @@ -1037,7 +1037,7 @@ describe('relationship benchmarks', () => { }); it('loads the full AdventureWorks OLTP benchmark fixture with declared metadata', async () => { - const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url); + const fixtureRoot = new URL('../../fixtures/relationship-benchmarks', import.meta.url); const fixture = await loadKtxRelationshipBenchmarkFixture( join(fixtureRoot.pathname, 'adventureworks_oltp_with_declared_metadata'), ); @@ -1097,7 +1097,7 @@ describe('relationship benchmarks', () => { }); it('loads the row-bearing natural-key fixture and counts it as headline evidence', async () => { - const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url); + const fixtureRoot = new URL('../../fixtures/relationship-benchmarks', import.meta.url); const naturalKeys = await loadKtxRelationshipBenchmarkFixture( join(fixtureRoot.pathname, 'natural_keys_no_declared_constraints'), ); @@ -1131,7 +1131,7 @@ describe('relationship benchmarks', () => { }); it('accepts plan-code suffix relationships only when validation is available', async () => { - const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url); + const fixtureRoot = new URL('../../fixtures/relationship-benchmarks', import.meta.url); const fixture = await loadKtxRelationshipBenchmarkFixture( join(fixtureRoot.pathname, 'plan_code_no_declared_constraints'), ); @@ -1192,7 +1192,7 @@ describe('relationship benchmarks', () => { }); it('uses embedding fixtures for semantic alias relationship benchmark cases', async () => { - const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url); + const fixtureRoot = new URL('../../fixtures/relationship-benchmarks', import.meta.url); const fixture = await loadKtxRelationshipBenchmarkFixture( join(fixtureRoot.pathname, 'semantic_embedding_aliases_no_declared_constraints'), ); @@ -1223,7 +1223,7 @@ describe('relationship benchmarks', () => { }); it('loads the Orbit-style product fixture as curated relationship-discovery benchmark evidence', async () => { - const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url); + const fixtureRoot = new URL('../../fixtures/relationship-benchmarks', import.meta.url); const fixture = await loadKtxRelationshipBenchmarkFixture( join(fixtureRoot.pathname, 'orbit_style_product_no_declared_constraints'), ); diff --git a/packages/cli/src/context/scan/relationship-budget.test.ts b/packages/cli/test/context/scan/relationship-budget.test.ts similarity index 97% rename from packages/cli/src/context/scan/relationship-budget.test.ts rename to packages/cli/test/context/scan/relationship-budget.test.ts index 479e5b23..b3c6edcf 100644 --- a/packages/cli/src/context/scan/relationship-budget.test.ts +++ b/packages/cli/test/context/scan/relationship-budget.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { applyKtxRelationshipValidationBudget, defaultKtxRelationshipValidationBudget } from './relationship-budget.js'; +import { applyKtxRelationshipValidationBudget, defaultKtxRelationshipValidationBudget } from '../../../src/context/scan/relationship-budget.js'; interface Candidate { id: string; diff --git a/packages/cli/src/context/scan/relationship-candidates.test.ts b/packages/cli/test/context/scan/relationship-candidates.test.ts similarity index 98% rename from packages/cli/src/context/scan/relationship-candidates.test.ts rename to packages/cli/test/context/scan/relationship-candidates.test.ts index 795d7791..cfe5ce2c 100644 --- a/packages/cli/src/context/scan/relationship-candidates.test.ts +++ b/packages/cli/test/context/scan/relationship-candidates.test.ts @@ -1,12 +1,12 @@ import { describe, expect, it } from 'vitest'; -import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from './enrichment-types.js'; -import { normalizeKtxRelationshipName } from './relationship-name-similarity.js'; +import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from '../../../src/context/scan/enrichment-types.js'; +import { normalizeKtxRelationshipName } from '../../../src/context/scan/relationship-name-similarity.js'; import { generateKtxRelationshipDiscoveryCandidates, inferKtxRelationshipTargetPks, mergeKtxRelationshipDiscoveryCandidates, -} from './relationship-candidates.js'; -import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js'; +} from '../../../src/context/scan/relationship-candidates.js'; +import type { KtxRelationshipProfileArtifact } from '../../../src/context/scan/relationship-profiling.js'; function column( tableId: string, diff --git a/packages/cli/src/context/scan/relationship-composite-candidates.test.ts b/packages/cli/test/context/scan/relationship-composite-candidates.test.ts similarity index 80% rename from packages/cli/src/context/scan/relationship-composite-candidates.test.ts rename to packages/cli/test/context/scan/relationship-composite-candidates.test.ts index abf495e1..e0a9ca6c 100644 --- a/packages/cli/src/context/scan/relationship-composite-candidates.test.ts +++ b/packages/cli/test/context/scan/relationship-composite-candidates.test.ts @@ -1,11 +1,12 @@ import Database from 'better-sqlite3'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { snapshotToKtxEnrichedSchema } from './local-enrichment.js'; -import { loadKtxRelationshipBenchmarkFixture, maskKtxRelationshipBenchmarkSnapshot } from './relationship-benchmarks.js'; -import { discoverKtxCompositeRelationships } from './relationship-composite-candidates.js'; -import { profileKtxRelationshipSchema, type KtxRelationshipReadOnlyExecutor } from './relationship-profiling.js'; -import type { KtxQueryResult, KtxReadOnlyQueryInput, KtxScanContext } from './types.js'; +import { getDialectForDriver } from '../../../src/context/connections/dialects.js'; +import { snapshotToKtxEnrichedSchema } from '../../../src/context/scan/local-enrichment.js'; +import { loadKtxRelationshipBenchmarkFixture, maskKtxRelationshipBenchmarkSnapshot } from '../../../src/context/scan/relationship-benchmarks.js'; +import { discoverKtxCompositeRelationships } from '../../../src/context/scan/relationship-composite-candidates.js'; +import { profileKtxRelationshipSchema, type KtxRelationshipReadOnlyExecutor } from '../../../src/context/scan/relationship-profiling.js'; +import type { KtxQueryResult, KtxReadOnlyQueryInput, KtxScanContext } from '../../../src/context/scan/types.js'; class TestSqliteExecutor implements KtxRelationshipReadOnlyExecutor { private readonly db: Database.Database; @@ -32,7 +33,7 @@ class TestSqliteExecutor implements KtxRelationshipReadOnlyExecutor { describe('composite relationship discovery detector', () => { it('infers composite primary keys and validates composite foreign keys from row evidence', async () => { - const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url); + const fixtureRoot = new URL('../../fixtures/relationship-benchmarks', import.meta.url); const fixture = await loadKtxRelationshipBenchmarkFixture( join(fixtureRoot.pathname, 'composite_keys_no_declared_constraints'), ); @@ -41,7 +42,7 @@ describe('composite relationship discovery detector', () => { const executor = new TestSqliteExecutor(fixture.dataPath ?? ''); const profiles = await profileKtxRelationshipSchema({ connectionId: snapshot.connectionId, - driver: snapshot.driver, + dialect: getDialectForDriver(snapshot.driver), schema, executor, ctx: { runId: 'test:composite-profile' }, @@ -49,7 +50,7 @@ describe('composite relationship discovery detector', () => { const result = await discoverKtxCompositeRelationships({ connectionId: snapshot.connectionId, - driver: snapshot.driver, + dialect: getDialectForDriver(snapshot.driver), schema, profiles, executor, diff --git a/packages/cli/src/context/scan/relationship-diagnostics.test.ts b/packages/cli/test/context/scan/relationship-diagnostics.test.ts similarity index 98% rename from packages/cli/src/context/scan/relationship-diagnostics.test.ts rename to packages/cli/test/context/scan/relationship-diagnostics.test.ts index 7c1dbb76..647ac931 100644 --- a/packages/cli/src/context/scan/relationship-diagnostics.test.ts +++ b/packages/cli/test/context/scan/relationship-diagnostics.test.ts @@ -1,11 +1,11 @@ import { describe, expect, it } from 'vitest'; -import type { KtxEnrichedRelationship, KtxRelationshipEndpoint } from './enrichment-types.js'; -import type { KtxResolvedRelationshipDiscoveryCandidate } from './relationship-graph-resolver.js'; +import type { KtxEnrichedRelationship, KtxRelationshipEndpoint } from '../../../src/context/scan/enrichment-types.js'; +import type { KtxResolvedRelationshipDiscoveryCandidate } from '../../../src/context/scan/relationship-graph-resolver.js'; import { buildKtxRelationshipArtifacts, buildKtxRelationshipDiagnostics, emptyKtxRelationshipProfileArtifact, -} from './relationship-diagnostics.js'; +} from '../../../src/context/scan/relationship-diagnostics.js'; function endpoint(table: string, column: string): KtxRelationshipEndpoint { return { diff --git a/packages/cli/src/context/scan/relationship-discovery.test.ts b/packages/cli/test/context/scan/relationship-discovery.test.ts similarity index 94% rename from packages/cli/src/context/scan/relationship-discovery.test.ts rename to packages/cli/test/context/scan/relationship-discovery.test.ts index 400fae62..55341645 100644 --- a/packages/cli/src/context/scan/relationship-discovery.test.ts +++ b/packages/cli/test/context/scan/relationship-discovery.test.ts @@ -1,15 +1,16 @@ import Database from 'better-sqlite3'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; -import { buildDefaultKtxProjectConfig } from '../project/config.js'; -import { snapshotToKtxEnrichedSchema } from './local-enrichment.js'; +import type { KtxLlmRuntimePort } from '../../../src/context/llm/runtime-port.js'; +import { getDialectForDriver } from '../../../src/context/connections/dialects.js'; +import { buildDefaultKtxProjectConfig } from '../../../src/context/project/config.js'; +import { snapshotToKtxEnrichedSchema } from '../../../src/context/scan/local-enrichment.js'; import { loadKtxRelationshipBenchmarkFixture, maskKtxRelationshipBenchmarkSnapshot, -} from './relationship-benchmarks.js'; -import { discoverKtxRelationships } from './relationship-discovery.js'; -import { createKtxConnectorCapabilities } from './types.js'; -import type { KtxQueryResult, KtxReadOnlyQueryInput, KtxScanConnector, KtxScanContext, KtxSchemaSnapshot } from './types.js'; +} from '../../../src/context/scan/relationship-benchmarks.js'; +import { discoverKtxRelationships } from '../../../src/context/scan/relationship-discovery.js'; +import { createKtxConnectorCapabilities } from '../../../src/context/scan/types.js'; +import type { KtxQueryResult, KtxReadOnlyQueryInput, KtxScanConnector, KtxScanContext, KtxSchemaSnapshot } from '../../../src/context/scan/types.js'; class InMemorySqliteExecutor { readonly db = new Database(':memory:'); @@ -212,6 +213,8 @@ function connector(executor: InMemorySqliteExecutor | null): KtxScanConnector { columnSampling: false, }), introspect: async () => snapshot(), + listSchemas: async () => [], + listTables: async () => [], executeReadOnly: executor ? executor.executeReadOnly.bind(executor) : undefined, }; } @@ -308,7 +311,7 @@ describe('production relationship discovery', () => { const result = await discoverKtxRelationships({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), connector: connector(executor), schema: snapshotToKtxEnrichedSchema(snapshot()), context: { runId: 'relationship-run-1' }, @@ -347,7 +350,7 @@ describe('production relationship discovery', () => { const schema = naturalKeySnapshot(); const result = await discoverKtxRelationships({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), connector: { ...connector(executor), introspect: async () => schema, @@ -397,7 +400,7 @@ describe('production relationship discovery', () => { const result = await discoverKtxRelationships({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), connector: { ...connector(executor), introspect: async () => sourceSnapshot, @@ -430,7 +433,7 @@ describe('production relationship discovery', () => { it('keeps candidates review-only when read-only SQL is unavailable', async () => { const result = await discoverKtxRelationships({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), connector: connector(null), schema: snapshotToKtxEnrichedSchema(snapshot()), context: { runId: 'relationship-run-no-sql' }, @@ -456,7 +459,7 @@ describe('production relationship discovery', () => { const sourceSnapshot = declaredForeignKeySnapshot(); const result = await discoverKtxRelationships({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), connector: connector(null), schema: snapshotToKtxEnrichedSchema(sourceSnapshot), context: { runId: 'formal-metadata-no-sql' }, @@ -503,7 +506,7 @@ describe('production relationship discovery', () => { const result = await discoverKtxRelationships({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), connector: connector(executor), schema: snapshotToKtxEnrichedSchema(llmOnlyRelationshipSnapshot()), context: { runId: 'llm-relationship-orchestrator' }, @@ -543,7 +546,7 @@ describe('production relationship discovery', () => { const result = await discoverKtxRelationships({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), connector: connector(executor), schema: snapshotToKtxEnrichedSchema(snapshot()), context: { runId: 'configured-thresholds' }, @@ -604,7 +607,7 @@ describe('production relationship discovery', () => { const result = await discoverKtxRelationships({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), connector: { ...connector(executor), introspect: async () => richSnapshot, @@ -628,7 +631,7 @@ describe('production relationship discovery', () => { it('accepts SQL-validated composite relationships in production relationship-discovery detection', async () => { const fixtureRoot = new URL( - '../../test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints', + '../../fixtures/relationship-benchmarks/composite_keys_no_declared_constraints', import.meta.url, ); const fixture = await loadKtxRelationshipBenchmarkFixture(fixtureRoot.pathname); @@ -644,6 +647,8 @@ describe('production relationship discovery', () => { columnSampling: false, }), introspect: async () => maskedSnapshot, + listSchemas: async () => [], + listTables: async () => [], executeReadOnly: async (input) => { const rows = database.prepare(input.sql).all() as Record[]; const headers = Object.keys(rows[0] ?? {}); @@ -658,7 +663,7 @@ describe('production relationship discovery', () => { const result = await discoverKtxRelationships({ connectionId: maskedSnapshot.connectionId, - driver: maskedSnapshot.driver, + dialect: getDialectForDriver(maskedSnapshot.driver), connector: testConnector, schema: snapshotToKtxEnrichedSchema(maskedSnapshot, new Map()), context: { runId: 'test:production-composite' }, diff --git a/packages/cli/src/context/scan/relationship-formal-metadata.test.ts b/packages/cli/test/context/scan/relationship-formal-metadata.test.ts similarity index 96% rename from packages/cli/src/context/scan/relationship-formal-metadata.test.ts rename to packages/cli/test/context/scan/relationship-formal-metadata.test.ts index 8e4a57ff..3fd94a42 100644 --- a/packages/cli/src/context/scan/relationship-formal-metadata.test.ts +++ b/packages/cli/test/context/scan/relationship-formal-metadata.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import type { KtxEnrichedRelationship, KtxEnrichedSchema } from './enrichment-types.js'; -import { collectKtxFormalMetadataRelationships } from './relationship-formal-metadata.js'; +import type { KtxEnrichedRelationship, KtxEnrichedSchema } from '../../../src/context/scan/enrichment-types.js'; +import { collectKtxFormalMetadataRelationships } from '../../../src/context/scan/relationship-formal-metadata.js'; function schema(relationships: KtxEnrichedRelationship[]): KtxEnrichedSchema { return { diff --git a/packages/cli/src/context/scan/relationship-graph-resolver.test.ts b/packages/cli/test/context/scan/relationship-graph-resolver.test.ts similarity index 98% rename from packages/cli/src/context/scan/relationship-graph-resolver.test.ts rename to packages/cli/test/context/scan/relationship-graph-resolver.test.ts index 945e8257..643d7956 100644 --- a/packages/cli/src/context/scan/relationship-graph-resolver.test.ts +++ b/packages/cli/test/context/scan/relationship-graph-resolver.test.ts @@ -4,10 +4,10 @@ import type { KtxEnrichedSchema, KtxEnrichedTable, KtxRelationshipEndpoint, -} from './enrichment-types.js'; -import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js'; -import type { KtxValidatedRelationshipDiscoveryCandidate } from './relationship-validation.js'; -import { resolveKtxRelationshipGraph } from './relationship-graph-resolver.js'; +} from '../../../src/context/scan/enrichment-types.js'; +import type { KtxRelationshipProfileArtifact } from '../../../src/context/scan/relationship-profiling.js'; +import type { KtxValidatedRelationshipDiscoveryCandidate } from '../../../src/context/scan/relationship-validation.js'; +import { resolveKtxRelationshipGraph } from '../../../src/context/scan/relationship-graph-resolver.js'; function column(tableId: string, name: string, overrides: Partial = {}): KtxEnrichedColumn { const tableRef = overrides.tableRef ?? { catalog: null, db: null, name: tableId }; diff --git a/packages/cli/src/context/scan/relationship-llm-proposal.test.ts b/packages/cli/test/context/scan/relationship-llm-proposal.test.ts similarity index 94% rename from packages/cli/src/context/scan/relationship-llm-proposal.test.ts rename to packages/cli/test/context/scan/relationship-llm-proposal.test.ts index 46e22dcb..0713a1c6 100644 --- a/packages/cli/src/context/scan/relationship-llm-proposal.test.ts +++ b/packages/cli/test/context/scan/relationship-llm-proposal.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it, vi } from 'vitest'; -import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; -import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from './enrichment-types.js'; -import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js'; -import { proposeKtxRelationshipCandidatesWithLlm } from './relationship-llm-proposal.js'; +import type { KtxLlmRuntimePort } from '../../../src/context/llm/runtime-port.js'; +import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from '../../../src/context/scan/enrichment-types.js'; +import type { KtxRelationshipProfileArtifact } from '../../../src/context/scan/relationship-profiling.js'; +import { proposeKtxRelationshipCandidatesWithLlm } from '../../../src/context/scan/relationship-llm-proposal.js'; function llmRuntime(output?: unknown): KtxLlmRuntimePort { return { diff --git a/packages/cli/src/context/scan/relationship-locality.test.ts b/packages/cli/test/context/scan/relationship-locality.test.ts similarity index 96% rename from packages/cli/src/context/scan/relationship-locality.test.ts rename to packages/cli/test/context/scan/relationship-locality.test.ts index 85dd4350..1a7f09fd 100644 --- a/packages/cli/src/context/scan/relationship-locality.test.ts +++ b/packages/cli/test/context/scan/relationship-locality.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import type { KtxEnrichedColumn, KtxEnrichedTable } from './enrichment-types.js'; -import { localCandidateTables } from './relationship-locality.js'; +import type { KtxEnrichedColumn, KtxEnrichedTable } from '../../../src/context/scan/enrichment-types.js'; +import { localCandidateTables } from '../../../src/context/scan/relationship-locality.js'; function column( tableId: string, diff --git a/packages/cli/src/context/scan/relationship-name-similarity.test.ts b/packages/cli/test/context/scan/relationship-name-similarity.test.ts similarity index 97% rename from packages/cli/src/context/scan/relationship-name-similarity.test.ts rename to packages/cli/test/context/scan/relationship-name-similarity.test.ts index 34730c81..0f8f437c 100644 --- a/packages/cli/src/context/scan/relationship-name-similarity.test.ts +++ b/packages/cli/test/context/scan/relationship-name-similarity.test.ts @@ -5,7 +5,7 @@ import { singularizeKtxRelationshipToken, tokenSimilarity, tokenizeKtxRelationshipName, -} from './relationship-name-similarity.js'; +} from '../../../src/context/scan/relationship-name-similarity.js'; describe('relationship name similarity', () => { it('tokenizes common warehouse naming styles', () => { diff --git a/packages/cli/src/context/scan/relationship-profiling.test.ts b/packages/cli/test/context/scan/relationship-profiling.test.ts similarity index 90% rename from packages/cli/src/context/scan/relationship-profiling.test.ts rename to packages/cli/test/context/scan/relationship-profiling.test.ts index 76151d23..7983d958 100644 --- a/packages/cli/src/context/scan/relationship-profiling.test.ts +++ b/packages/cli/test/context/scan/relationship-profiling.test.ts @@ -2,16 +2,12 @@ import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import Database from 'better-sqlite3'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from './enrichment-types.js'; -import { snapshotToKtxEnrichedSchema } from './local-enrichment.js'; -import { loadKtxRelationshipBenchmarkFixture, maskKtxRelationshipBenchmarkSnapshot } from './relationship-benchmarks.js'; -import { - createKtxRelationshipProfileCache, - formatKtxRelationshipTableRef, - profileKtxRelationshipSchema, - quoteKtxRelationshipIdentifier, -} from './relationship-profiling.js'; -import type { KtxQueryResult, KtxReadOnlyQueryInput, KtxScanContext } from './types.js'; +import { getDialectForDriver } from '../../../src/context/connections/dialects.js'; +import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from '../../../src/context/scan/enrichment-types.js'; +import { snapshotToKtxEnrichedSchema } from '../../../src/context/scan/local-enrichment.js'; +import { loadKtxRelationshipBenchmarkFixture, maskKtxRelationshipBenchmarkSnapshot } from '../../../src/context/scan/relationship-benchmarks.js'; +import { createKtxRelationshipProfileCache, profileKtxRelationshipSchema } from '../../../src/context/scan/relationship-profiling.js'; +import type { KtxQueryResult, KtxReadOnlyQueryInput, KtxScanContext } from '../../../src/context/scan/types.js'; class InMemorySqliteExecutor { readonly db = new Database(':memory:'); @@ -104,7 +100,7 @@ describe('relationship profiling', () => { }); it('keeps profiling on the batched table path', async () => { - const source = await readFile(new URL('relationship-profiling.ts', import.meta.url), 'utf-8'); + const source = await readFile(new URL('../../../src/context/scan/relationship-profiling.ts', import.meta.url), 'utf-8'); expect(source).not.toMatch(new RegExp('queryColumn' + 'Profile')); expect(source).not.toMatch(/for \(const column of table\.columns\)[\s\S]*executeReadOnly/); @@ -112,16 +108,6 @@ describe('relationship profiling', () => { expect(source).toMatch(/UNION ALL/); }); - it('quotes identifiers and formats table refs for supported local SQL drivers', () => { - expect(quoteKtxRelationshipIdentifier('sqlite', 'odd"name')).toBe('"odd""name"'); - expect(quoteKtxRelationshipIdentifier('mysql', 'odd`name')).toBe('`odd``name`'); - expect(quoteKtxRelationshipIdentifier('sqlserver', 'odd]name')).toBe('[odd]]name]'); - expect(formatKtxRelationshipTableRef('sqlite', { catalog: null, db: null, name: 'accounts' })).toBe('"accounts"'); - expect(formatKtxRelationshipTableRef('postgres', { catalog: null, db: 'analytics', name: 'accounts' })).toBe( - '"analytics"."accounts"', - ); - }); - it('profiles row count, null rate, uniqueness, sample values, and text lengths', async () => { executor = new InMemorySqliteExecutor(); executor.db.exec(` @@ -135,7 +121,7 @@ describe('relationship profiling', () => { const result = await profileKtxRelationshipSchema({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), schema: schema([ table('accounts', [ column('accounts', 'id', { primaryKey: false, nullable: false }), @@ -197,7 +183,7 @@ describe('relationship profiling', () => { const result = await profileKtxRelationshipSchema({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), schema: schema([ table('accounts', [ column('accounts', 'id', { nullable: false }), @@ -240,7 +226,7 @@ describe('relationship profiling', () => { const profiles = await profileKtxRelationshipSchema({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), schema: schema([ table('accounts', [ column('accounts', 'id', { nullable: false }), @@ -291,7 +277,7 @@ describe('relationship profiling', () => { const first = await profileKtxRelationshipSchema({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), schema: relationshipSchema, executor, ctx: { runId: 'profile-cache-run' }, @@ -299,7 +285,7 @@ describe('relationship profiling', () => { }); const second = await profileKtxRelationshipSchema({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), schema: relationshipSchema, executor, ctx: { runId: 'profile-cache-run' }, @@ -307,7 +293,7 @@ describe('relationship profiling', () => { }); const third = await profileKtxRelationshipSchema({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), schema: relationshipSchema, executor, ctx: { runId: 'profile-cache-fresh-run' }, @@ -323,7 +309,7 @@ describe('relationship profiling', () => { }); it('profiles the checked-in scale stress fixture with one query per table', async () => { - const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url); + const fixtureRoot = new URL('../../fixtures/relationship-benchmarks', import.meta.url); const fixture = await loadKtxRelationshipBenchmarkFixture(join(fixtureRoot.pathname, 'scale_stress_no_declared_constraints')); if (!fixture.dataPath) { throw new Error('scale_stress_no_declared_constraints is missing data.sqlite'); @@ -336,7 +322,7 @@ describe('relationship profiling', () => { try { const result = await profileKtxRelationshipSchema({ connectionId: fixture.snapshot.connectionId, - driver: fixture.snapshot.driver, + dialect: getDialectForDriver(fixture.snapshot.driver), schema: snapshotToKtxEnrichedSchema(maskedSnapshot, new Map()), executor: scaleExecutor, ctx: { runId: 'scale-stress-profile-query-count' }, @@ -381,7 +367,7 @@ describe('relationship profiling', () => { await profileKtxRelationshipSchema({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), schema: schemaWithTables(['accounts', 'orders', 'payments', 'refunds']), executor, ctx: { runId: 'profile-concurrency' }, @@ -417,7 +403,7 @@ describe('relationship profiling', () => { const result = await profileKtxRelationshipSchema({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), schema: schemaWithTables(['accounts', 'orders']), executor, ctx: { runId: 'profile-error-isolated' }, diff --git a/packages/cli/src/context/scan/relationship-scoring.test.ts b/packages/cli/test/context/scan/relationship-scoring.test.ts similarity index 98% rename from packages/cli/src/context/scan/relationship-scoring.test.ts rename to packages/cli/test/context/scan/relationship-scoring.test.ts index 30127913..bd683f55 100644 --- a/packages/cli/src/context/scan/relationship-scoring.test.ts +++ b/packages/cli/test/context/scan/relationship-scoring.test.ts @@ -5,7 +5,7 @@ import { normalizeKtxRelationshipScoreWeights, scoreKtxRelationshipCandidate, type KtxRelationshipSignalVector, -} from './relationship-scoring.js'; +} from '../../../src/context/scan/relationship-scoring.js'; function signals(overrides: Partial = {}): KtxRelationshipSignalVector { return { diff --git a/packages/cli/src/context/scan/relationship-validation.test.ts b/packages/cli/test/context/scan/relationship-validation.test.ts similarity index 93% rename from packages/cli/src/context/scan/relationship-validation.test.ts rename to packages/cli/test/context/scan/relationship-validation.test.ts index 856cf60a..cd7771d9 100644 --- a/packages/cli/src/context/scan/relationship-validation.test.ts +++ b/packages/cli/test/context/scan/relationship-validation.test.ts @@ -1,11 +1,12 @@ import Database from 'better-sqlite3'; import { afterEach, describe, expect, it } from 'vitest'; -import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from './enrichment-types.js'; -import { generateKtxRelationshipDiscoveryCandidates } from './relationship-candidates.js'; -import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js'; -import { profileKtxRelationshipSchema } from './relationship-profiling.js'; -import { validateKtxRelationshipDiscoveryCandidates } from './relationship-validation.js'; -import type { KtxQueryResult, KtxReadOnlyQueryInput, KtxScanContext } from './types.js'; +import { getDialectForDriver } from '../../../src/context/connections/dialects.js'; +import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from '../../../src/context/scan/enrichment-types.js'; +import { generateKtxRelationshipDiscoveryCandidates } from '../../../src/context/scan/relationship-candidates.js'; +import type { KtxRelationshipProfileArtifact } from '../../../src/context/scan/relationship-profiling.js'; +import { profileKtxRelationshipSchema } from '../../../src/context/scan/relationship-profiling.js'; +import { validateKtxRelationshipDiscoveryCandidates } from '../../../src/context/scan/relationship-validation.js'; +import type { KtxQueryResult, KtxReadOnlyQueryInput, KtxScanContext } from '../../../src/context/scan/types.js'; class InMemorySqliteExecutor { readonly db = new Database(':memory:'); @@ -99,7 +100,7 @@ describe('relationship validation', () => { const testSchema = schema(); const profiles = await profileKtxRelationshipSchema({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), schema: testSchema, executor, ctx: { runId: 'validate-test' }, @@ -110,7 +111,7 @@ describe('relationship validation', () => { const validated = await validateKtxRelationshipDiscoveryCandidates({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), candidates, profiles, executor, @@ -148,7 +149,7 @@ describe('relationship validation', () => { const testSchema = schema(); const profiles = await profileKtxRelationshipSchema({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), schema: testSchema, executor, ctx: { runId: 'validate-test' }, @@ -159,7 +160,7 @@ describe('relationship validation', () => { const validated = await validateKtxRelationshipDiscoveryCandidates({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), candidates, profiles, executor, @@ -198,7 +199,7 @@ describe('relationship validation', () => { const testSchema = schema(); const profiles = await profileKtxRelationshipSchema({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), schema: testSchema, executor, ctx: { runId: 'validate-budget-profile' }, @@ -211,7 +212,7 @@ describe('relationship validation', () => { const validated = await validateKtxRelationshipDiscoveryCandidates({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), candidates, profiles, executor, @@ -253,7 +254,7 @@ describe('relationship validation', () => { ]); const profiles = await profileKtxRelationshipSchema({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), schema: testSchema, executor, ctx: { runId: 'validate-zero-budget-profile' }, @@ -263,7 +264,7 @@ describe('relationship validation', () => { const validated = await validateKtxRelationshipDiscoveryCandidates({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), candidates, profiles, executor, @@ -300,7 +301,7 @@ describe('relationship validation', () => { ]); const profiles = await profileKtxRelationshipSchema({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), schema: testSchema, executor, ctx: { runId: 'llm-rejected-validation' }, @@ -329,7 +330,7 @@ describe('relationship validation', () => { const [validated] = await validateKtxRelationshipDiscoveryCandidates({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), candidates: [llmCandidate], profiles, executor, @@ -374,7 +375,7 @@ describe('relationship validation', () => { ]); const profiles = await profileKtxRelationshipSchema({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), schema: testSchema, executor, ctx: { runId: 'validation-concurrency-profile' }, @@ -383,7 +384,7 @@ describe('relationship validation', () => { await validateKtxRelationshipDiscoveryCandidates({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), candidates, profiles, executor: throttled, @@ -475,7 +476,7 @@ describe('relationship validation', () => { const [validated] = await validateKtxRelationshipDiscoveryCandidates({ connectionId: 'warehouse', - driver: 'sqlite', + dialect: getDialectForDriver('sqlite'), candidates: [candidate], profiles, executor, diff --git a/packages/cli/src/context/scan/table-ref.test.ts b/packages/cli/test/context/scan/table-ref.test.ts similarity index 91% rename from packages/cli/src/context/scan/table-ref.test.ts rename to packages/cli/test/context/scan/table-ref.test.ts index eb52ac9b..233af18c 100644 --- a/packages/cli/src/context/scan/table-ref.test.ts +++ b/packages/cli/test/context/scan/table-ref.test.ts @@ -5,7 +5,7 @@ import { tableRefKey, tableRefSet, type KtxTableRefKey, -} from './table-ref.js'; +} from '../../../src/context/scan/table-ref.js'; describe('tableRefKey roundtrip', () => { it('encodes and decodes a three-part ref', () => { @@ -47,9 +47,9 @@ describe('scopedTableNames', () => { expect(scopedTableNames(scope, { catalog: 'ANALYTICS', db: 'STAGING' })).toEqual(['LISTINGS']); }); - it('treats null in the scope entry as a wildcard for that segment', () => { + it('requires non-null scope segments to match the namespace', () => { const scope = tableRefSet([{ catalog: null, db: 'public', name: 'users' }]); - expect(scopedTableNames(scope, { catalog: 'any-catalog', db: 'public' })).toEqual(['users']); + expect(scopedTableNames(scope, { catalog: 'any-catalog', db: 'public' })).toEqual([]); }); it('returns empty when no scope entry matches the namespace', () => { @@ -57,7 +57,7 @@ describe('scopedTableNames', () => { expect(scopedTableNames(scope, { catalog: 'X', db: 'Y' })).toEqual([]); }); - it('dedupes when the same name appears under different catalog projections', () => { + it('dedupes exact namespace matches only', () => { const scope: ReadonlySet = tableRefSet([ { catalog: null, db: 'public', name: 'users' }, { catalog: 'A', db: 'public', name: 'users' }, diff --git a/packages/cli/src/context/scan/type-normalization.test.ts b/packages/cli/test/context/scan/type-normalization.test.ts similarity index 92% rename from packages/cli/src/context/scan/type-normalization.test.ts rename to packages/cli/test/context/scan/type-normalization.test.ts index 5dc0adf2..fa19df32 100644 --- a/packages/cli/src/context/scan/type-normalization.test.ts +++ b/packages/cli/test/context/scan/type-normalization.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { inferKtxDimensionType, ktxColumnTypeMappingFromNative, normalizeKtxNativeType } from './type-normalization.js'; +import { inferKtxDimensionType, ktxColumnTypeMappingFromNative, normalizeKtxNativeType } from '../../../src/context/scan/type-normalization.js'; describe('KTX scan type normalization', () => { it('normalizes native database type strings', () => { diff --git a/packages/cli/src/context/scan/types.test.ts b/packages/cli/test/context/scan/types.test.ts similarity index 97% rename from packages/cli/src/context/scan/types.test.ts rename to packages/cli/test/context/scan/types.test.ts index 309db88e..8aa55dba 100644 --- a/packages/cli/src/context/scan/types.test.ts +++ b/packages/cli/test/context/scan/types.test.ts @@ -15,7 +15,7 @@ import { type KtxScanContext, type KtxScanInput, type KtxSchemaSnapshot, -} from './types.js'; +} from '../../../src/context/scan/types.js'; describe('KTX scan contract types', () => { it('defaults to structural-only connector capabilities', () => { @@ -93,6 +93,8 @@ describe('KTX scan contract types', () => { expect(ctx.runId).toBe('scan-run-1'); return snapshot; }, + listSchemas: async () => [], + listTables: async () => [], }; await expect( @@ -164,6 +166,8 @@ describe('KTX scan contract types', () => { tables: [], }; }, + listSchemas: async () => [], + listTables: async () => [], }; await expect( diff --git a/packages/cli/src/context/scan/warehouse-catalog.test.ts b/packages/cli/test/context/scan/warehouse-catalog.test.ts similarity index 92% rename from packages/cli/src/context/scan/warehouse-catalog.test.ts rename to packages/cli/test/context/scan/warehouse-catalog.test.ts index 6ef1f03a..300eba91 100644 --- a/packages/cli/src/context/scan/warehouse-catalog.test.ts +++ b/packages/cli/test/context/scan/warehouse-catalog.test.ts @@ -2,8 +2,8 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; -import { WarehouseCatalogService } from './warehouse-catalog.js'; +import { initKtxProject, type KtxLocalProject } from '../../../src/context/project/project.js'; +import { WarehouseCatalogService } from '../../../src/context/scan/warehouse-catalog.js'; describe('WarehouseCatalogService', () => { let tempDir: string; @@ -156,6 +156,17 @@ describe('WarehouseCatalogService', () => { }); }); + it('keeps one-part table display fallback for loose catalog resolution', async () => { + await seedLiveDatabaseScan(); + const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); + + await expect(catalog.resolveDisplay('warehouse', 'orders')).resolves.toMatchObject({ + resolved: { catalog: null, db: 'public', name: 'orders' }, + candidates: [], + dialect: 'postgres', + }); + }); + it('treats two-part BigQuery identifiers as ambiguous instead of guessing', async () => { await seedLiveDatabaseScan('warehouse', 'sync-bigquery', 'bigquery'); const catalog = new WarehouseCatalogService({ fileStore: project.fileStore }); diff --git a/packages/cli/src/context/search/backend-conformance.test-utils.test.ts b/packages/cli/test/context/search/backend-conformance.test-utils.test.ts similarity index 95% rename from packages/cli/src/context/search/backend-conformance.test-utils.test.ts rename to packages/cli/test/context/search/backend-conformance.test-utils.test.ts index c9ecebb7..b49f866a 100644 --- a/packages/cli/src/context/search/backend-conformance.test-utils.test.ts +++ b/packages/cli/test/context/search/backend-conformance.test-utils.test.ts @@ -2,22 +2,22 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, it } from 'vitest'; -import { SqliteContextEvidenceStore } from '../ingest/context-evidence/sqlite-context-evidence-store.js'; -import type { JsonValue } from '../ingest/ports.js'; -import { initKtxProject, type KtxLocalProject } from '../project/project.js'; -import { type LocalSlSourceSearchResult, searchLocalSlSources, writeLocalSlSource } from '../sl/local-sl.js'; -import type { ContextEvidenceSearchResult } from '../tools/context-evidence-tool-store.js'; +import { SqliteContextEvidenceStore } from '../../../src/context/ingest/context-evidence/sqlite-context-evidence-store.js'; +import type { JsonValue } from '../../../src/context/ingest/ports.js'; +import { initKtxProject, type KtxLocalProject } from '../../../src/context/project/project.js'; +import { type LocalSlSourceSearchResult, searchLocalSlSources, writeLocalSlSource } from '../../../src/context/sl/local-sl.js'; +import type { ContextEvidenceSearchResult } from '../../../src/context/tools/context-evidence-tool-store.js'; import { type LocalKnowledgeSearchResult, searchLocalKnowledgePages, writeLocalKnowledgePage, -} from '../wiki/local-knowledge.js'; +} from '../../../src/context/wiki/local-knowledge.js'; import { assertSearchBackendCapabilities, assertSearchBackendConformanceCase, type SearchBackendConformanceResult, } from './backend-conformance.test-utils.js'; -import type { SearchBackendCapabilities } from './types.js'; +import type { SearchBackendCapabilities } from '../../../src/context/search/types.js'; const SQLITE_SEARCH_CAPABILITIES = { fts: true, diff --git a/packages/cli/src/context/search/backend-conformance.test-utils.ts b/packages/cli/test/context/search/backend-conformance.test-utils.ts similarity index 99% rename from packages/cli/src/context/search/backend-conformance.test-utils.ts rename to packages/cli/test/context/search/backend-conformance.test-utils.ts index fa6070b2..506cc22c 100644 --- a/packages/cli/src/context/search/backend-conformance.test-utils.ts +++ b/packages/cli/test/context/search/backend-conformance.test-utils.ts @@ -1,4 +1,4 @@ -import type { SearchBackendCapabilities, SearchLaneStatus } from './types.js'; +import type { SearchBackendCapabilities, SearchLaneStatus } from '../../../src/context/search/types.js'; export interface SearchBackendConformanceLane { lane: string; diff --git a/packages/cli/src/context/search/discover.test.ts b/packages/cli/test/context/search/discover.test.ts similarity index 97% rename from packages/cli/src/context/search/discover.test.ts rename to packages/cli/test/context/search/discover.test.ts index 931de2be..77e35e18 100644 --- a/packages/cli/src/context/search/discover.test.ts +++ b/packages/cli/test/context/search/discover.test.ts @@ -2,9 +2,9 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; -import { writeLocalKnowledgePage } from '../wiki/local-knowledge.js'; -import { createKtxDiscoverDataService } from './discover.js'; +import { initKtxProject, type KtxLocalProject } from '../../../src/context/project/project.js'; +import { writeLocalKnowledgePage } from '../../../src/context/wiki/local-knowledge.js'; +import { createKtxDiscoverDataService } from '../../../src/context/search/discover.js'; describe('createKtxDiscoverDataService', () => { let tempDir: string; diff --git a/packages/cli/src/context/search/hybrid-search-core.test.ts b/packages/cli/test/context/search/hybrid-search-core.test.ts similarity index 96% rename from packages/cli/src/context/search/hybrid-search-core.test.ts rename to packages/cli/test/context/search/hybrid-search-core.test.ts index 2350e2ed..5952e3ee 100644 --- a/packages/cli/src/context/search/hybrid-search-core.test.ts +++ b/packages/cli/test/context/search/hybrid-search-core.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { HybridSearchCore } from './hybrid-search-core.js'; -import type { SearchCandidateGenerator } from './types.js'; +import { HybridSearchCore } from '../../../src/context/search/hybrid-search-core.js'; +import type { SearchCandidateGenerator } from '../../../src/context/search/types.js'; function generator( lane: string, diff --git a/packages/cli/src/context/search/pglite-owner-process.test.ts b/packages/cli/test/context/search/pglite-owner-process.test.ts similarity index 98% rename from packages/cli/src/context/search/pglite-owner-process.test.ts rename to packages/cli/test/context/search/pglite-owner-process.test.ts index 3a15eea9..df3d096b 100644 --- a/packages/cli/src/context/search/pglite-owner-process.test.ts +++ b/packages/cli/test/context/search/pglite-owner-process.test.ts @@ -4,8 +4,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { Client } from 'pg'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { assertSearchBackendConformanceCase } from '../../context/search/backend-conformance.test-utils.js'; -import { KtxPGliteOwnerProcess } from './pglite-owner-process.js'; +import { assertSearchBackendConformanceCase } from './backend-conformance.test-utils.js'; +import { KtxPGliteOwnerProcess } from '../../../src/context/search/pglite-owner-process.js'; async function allocatePort(): Promise { const server = createServer(); diff --git a/packages/cli/src/context/search/pglite-runtime-boundary.test.ts b/packages/cli/test/context/search/pglite-runtime-boundary.test.ts similarity index 96% rename from packages/cli/src/context/search/pglite-runtime-boundary.test.ts rename to packages/cli/test/context/search/pglite-runtime-boundary.test.ts index feb7443d..c9fe80d2 100644 --- a/packages/cli/src/context/search/pglite-runtime-boundary.test.ts +++ b/packages/cli/test/context/search/pglite-runtime-boundary.test.ts @@ -3,7 +3,7 @@ import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; -const ktxRoot = fileURLToPath(new URL('../../../../../', import.meta.url)); +const ktxRoot = fileURLToPath(new URL('../../../../..', import.meta.url)); function readKtxFile(relativePath: string): string { return readFileSync(join(ktxRoot, relativePath), 'utf8'); diff --git a/packages/cli/src/context/search/pglite-spike.test.ts b/packages/cli/test/context/search/pglite-spike.test.ts similarity index 98% rename from packages/cli/src/context/search/pglite-spike.test.ts rename to packages/cli/test/context/search/pglite-spike.test.ts index 470000da..2183630b 100644 --- a/packages/cli/src/context/search/pglite-spike.test.ts +++ b/packages/cli/test/context/search/pglite-spike.test.ts @@ -5,8 +5,8 @@ import { PGlite, type PGliteInterface } from '@electric-sql/pglite'; import { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm'; import { vector } from '@electric-sql/pglite/vector'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { assertSearchBackendCapabilities, assertSearchBackendConformanceCase } from '../../context/search/backend-conformance.test-utils.js'; -import type { SearchBackendCapabilities } from '../../context/search/types.js'; +import { assertSearchBackendCapabilities, assertSearchBackendConformanceCase } from './backend-conformance.test-utils.js'; +import type { SearchBackendCapabilities } from '../../../src/context/search/types.js'; type PGliteDb = PGliteInterface; diff --git a/packages/cli/src/context/search/query.test.ts b/packages/cli/test/context/search/query.test.ts similarity index 95% rename from packages/cli/src/context/search/query.test.ts rename to packages/cli/test/context/search/query.test.ts index 64f1fd0b..b8e7660f 100644 --- a/packages/cli/src/context/search/query.test.ts +++ b/packages/cli/test/context/search/query.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { defaultLaneCandidatePoolLimit, normalizeSearchQuery } from './query.js'; +import { defaultLaneCandidatePoolLimit, normalizeSearchQuery } from '../../../src/context/search/query.js'; describe('search query helpers', () => { it('normalizes punctuation and duplicate terms into stable lowercase tokens', () => { diff --git a/packages/cli/src/context/search/rrf.test.ts b/packages/cli/test/context/search/rrf.test.ts similarity index 90% rename from packages/cli/src/context/search/rrf.test.ts rename to packages/cli/test/context/search/rrf.test.ts index cbb4065b..42890989 100644 --- a/packages/cli/src/context/search/rrf.test.ts +++ b/packages/cli/test/context/search/rrf.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { compareFusedSearchCandidates, DEFAULT_SEARCH_LANE_WEIGHTS, rrfContribution } from './rrf.js'; -import type { FusedSearchCandidate } from './types.js'; +import { compareFusedSearchCandidates, DEFAULT_SEARCH_LANE_WEIGHTS, rrfContribution } from '../../../src/context/search/rrf.js'; +import type { FusedSearchCandidate } from '../../../src/context/search/types.js'; describe('RRF scoring', () => { it('uses the shared lane weights from the hybrid search spec', () => { diff --git a/packages/cli/src/context/skills/skills-registry.service.test.ts b/packages/cli/test/context/skills/skills-registry.service.test.ts similarity index 98% rename from packages/cli/src/context/skills/skills-registry.service.test.ts rename to packages/cli/test/context/skills/skills-registry.service.test.ts index 9bb716dd..2f6f1aaf 100644 --- a/packages/cli/src/context/skills/skills-registry.service.test.ts +++ b/packages/cli/test/context/skills/skills-registry.service.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { SkillsRegistryService } from './skills-registry.service.js'; +import { SkillsRegistryService } from '../../../src/context/skills/skills-registry.service.js'; describe('SkillsRegistryService', () => { let service: SkillsRegistryService; diff --git a/packages/cli/src/context/sl/dictionary-search.test.ts b/packages/cli/test/context/sl/dictionary-search.test.ts similarity index 97% rename from packages/cli/src/context/sl/dictionary-search.test.ts rename to packages/cli/test/context/sl/dictionary-search.test.ts index 1838f0d9..b7a6beeb 100644 --- a/packages/cli/src/context/sl/dictionary-search.test.ts +++ b/packages/cli/test/context/sl/dictionary-search.test.ts @@ -2,8 +2,8 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; -import { createKtxDictionarySearchService } from './dictionary-search.js'; +import { initKtxProject, type KtxLocalProject } from '../../../src/context/project/project.js'; +import { createKtxDictionarySearchService } from '../../../src/context/sl/dictionary-search.js'; describe('createKtxDictionarySearchService', () => { let tempDir: string; diff --git a/packages/cli/src/context/sl/local-query.test.ts b/packages/cli/test/context/sl/local-query.test.ts similarity index 97% rename from packages/cli/src/context/sl/local-query.test.ts rename to packages/cli/test/context/sl/local-query.test.ts index 800bdb95..4137f596 100644 --- a/packages/cli/src/context/sl/local-query.test.ts +++ b/packages/cli/test/context/sl/local-query.test.ts @@ -2,9 +2,9 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js'; -import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; -import { compileLocalSlQuery } from './local-query.js'; +import type { KtxSemanticLayerComputePort } from '../../../src/context/daemon/semantic-layer-compute.js'; +import { initKtxProject, type KtxLocalProject } from '../../../src/context/project/project.js'; +import { compileLocalSlQuery } from '../../../src/context/sl/local-query.js'; describe('compileLocalSlQuery', () => { let tempDir: string; diff --git a/packages/cli/src/context/sl/local-sl.test.ts b/packages/cli/test/context/sl/local-sl.test.ts similarity index 80% rename from packages/cli/src/context/sl/local-sl.test.ts rename to packages/cli/test/context/sl/local-sl.test.ts index 18cc7392..b3a9b7d6 100644 --- a/packages/cli/src/context/sl/local-sl.test.ts +++ b/packages/cli/test/context/sl/local-sl.test.ts @@ -2,14 +2,15 @@ import { access, mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; +import { initKtxProject, type KtxLocalProject } from '../../../src/context/project/project.js'; import { listLocalSlSources, readLocalSlSource, + resolveLocalSlSource, searchLocalSlSources, validateLocalSlSource, writeLocalSlSource, -} from './local-sl.js'; +} from '../../../src/context/sl/local-sl.js'; const ORDERS_YAML = [ 'name: orders', @@ -90,6 +91,101 @@ describe('local semantic-layer helpers', () => { await expect(validateLocalSlSource(ORDERS_YAML)).resolves.toEqual({ valid: true, errors: [] }); }); + it('resolves a scoped source by connection id', async () => { + await writeLocalSlSource(project, { + connectionId: 'warehouse', + sourceName: 'orders', + yaml: ORDERS_YAML, + }); + + await expect( + resolveLocalSlSource(project, { + connectionId: 'warehouse', + sourceName: 'orders', + }), + ).resolves.toEqual({ + kind: 'found', + source: expect.objectContaining({ + connectionId: 'warehouse', + name: 'orders', + path: 'semantic-layer/warehouse/orders.yaml', + yaml: ORDERS_YAML, + }), + }); + }); + + it('returns not-found for a missing scoped source', async () => { + await writeLocalSlSource(project, { + connectionId: 'warehouse', + sourceName: 'orders', + yaml: ORDERS_YAML, + }); + + await expect( + resolveLocalSlSource(project, { + connectionId: 'warehouse', + sourceName: 'missing_orders', + }), + ).resolves.toEqual({ kind: 'not-found' }); + }); + + it('resolves a unique source name across all connections', async () => { + await writeLocalSlSource(project, { + connectionId: 'warehouse', + sourceName: 'orders', + yaml: ORDERS_YAML, + }); + await writeLocalSlSource(project, { + connectionId: 'analytics', + sourceName: 'tickets', + yaml: SUPPORT_YAML, + }); + + await expect( + resolveLocalSlSource(project, { + sourceName: 'tickets', + }), + ).resolves.toEqual({ + kind: 'found', + source: expect.objectContaining({ + connectionId: 'analytics', + name: 'tickets', + path: 'semantic-layer/analytics/tickets.yaml', + yaml: SUPPORT_YAML, + }), + }); + }); + + it('returns not-found for a missing unscoped source', async () => { + await writeLocalSlSource(project, { + connectionId: 'warehouse', + sourceName: 'orders', + yaml: ORDERS_YAML, + }); + + await expect(resolveLocalSlSource(project, { sourceName: 'missing_orders' })).resolves.toEqual({ + kind: 'not-found', + }); + }); + + it('reports sorted ambiguous connection ids for duplicate source names', async () => { + await writeLocalSlSource(project, { + connectionId: 'warehouse', + sourceName: 'orders', + yaml: ORDERS_YAML, + }); + await writeLocalSlSource(project, { + connectionId: 'analytics', + sourceName: 'orders', + yaml: ORDERS_YAML, + }); + + await expect(resolveLocalSlSource(project, { sourceName: 'orders' })).resolves.toEqual({ + kind: 'ambiguous', + connectionIds: ['analytics', 'warehouse'], + }); + }); + it('validates table-backed sources against matching physical manifests when project context is provided', async () => { await project.fileStore.writeFile( 'semantic-layer/postgres-warehouse/_schema/orbit_analytics.yaml', @@ -392,7 +488,7 @@ describe('local semantic-layer helpers', () => { ).rejects.toThrow('Invalid semantic-layer source'); }); - it('reports legacy overlay column patches with a file-attributed migration hint', async () => { + it('reports overlay columns that are not computed columns', async () => { const invalidYaml = [ 'name: orders', 'columns:', @@ -406,9 +502,7 @@ describe('local semantic-layer helpers', () => { validateLocalSlSource(invalidYaml, { project, connectionId: 'warehouse', sourceName: 'orders' }), ).resolves.toEqual({ valid: false, - errors: [ - "semantic-layer/warehouse/orders.yaml: column 'status' patches a manifest column but is in 'columns:' — move it to 'column_overrides:'", - ], + errors: expect.arrayContaining([expect.stringContaining('columns.0.type')]), }); }); diff --git a/packages/cli/src/context/sl/pglite-sl-search-prototype.test.ts b/packages/cli/test/context/sl/pglite-sl-search-prototype.test.ts similarity index 95% rename from packages/cli/src/context/sl/pglite-sl-search-prototype.test.ts rename to packages/cli/test/context/sl/pglite-sl-search-prototype.test.ts index 372f8668..8ebb8646 100644 --- a/packages/cli/src/context/sl/pglite-sl-search-prototype.test.ts +++ b/packages/cli/test/context/sl/pglite-sl-search-prototype.test.ts @@ -3,10 +3,10 @@ import { createServer } from 'node:net'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; -import { assertSearchBackendConformanceCase } from '../../context/search/backend-conformance.test-utils.js'; -import { searchLocalSlSources, writeLocalSlSource, type LocalSlSourceSearchResult } from './local-sl.js'; -import { searchLocalSlSourcesWithPglitePrototype } from './pglite-sl-search-prototype.js'; +import { initKtxProject, type KtxLocalProject } from '../../../src/context/project/project.js'; +import { assertSearchBackendConformanceCase } from '../search/backend-conformance.test-utils.js'; +import { searchLocalSlSources, writeLocalSlSource, type LocalSlSourceSearchResult } from '../../../src/context/sl/local-sl.js'; +import { searchLocalSlSourcesWithPglitePrototype } from '../../../src/context/sl/pglite-sl-search-prototype.js'; const ORDERS_YAML = [ 'name: orders', diff --git a/packages/cli/src/context/sl/schemas.contract.test.ts b/packages/cli/test/context/sl/schemas.contract.test.ts similarity index 88% rename from packages/cli/src/context/sl/schemas.contract.test.ts rename to packages/cli/test/context/sl/schemas.contract.test.ts index 1b0dac20..9b1293f1 100644 --- a/packages/cli/src/context/sl/schemas.contract.test.ts +++ b/packages/cli/test/context/sl/schemas.contract.test.ts @@ -2,14 +2,14 @@ import { execFileSync } from 'node:child_process'; import { Ajv2020 } from 'ajv/dist/2020.js'; import { describe, expect, it } from 'vitest'; -import { resolvedSourceSchema } from './schemas.js'; -import { toResolvedWire } from './semantic-layer.service.js'; -import type { SemanticLayerSource } from './types.js'; +import { resolvedSourceSchema } from '../../../src/context/sl/schemas.js'; +import { toResolvedWire } from '../../../src/context/sl/semantic-layer.service.js'; +import type { SemanticLayerSource } from '../../../src/context/sl/types.js'; function loadPythonSourceDefinitionSchema(): Record | null { try { const stdout = execFileSync('uv', ['run', 'python', '-m', 'semantic_layer', 'dump-schema'], { - cwd: new URL('../../../../', import.meta.url), + cwd: new URL('../../../..', import.meta.url), encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], }); diff --git a/packages/cli/src/context/sl/semantic-layer.service.test.ts b/packages/cli/test/context/sl/semantic-layer.service.test.ts similarity index 98% rename from packages/cli/src/context/sl/semantic-layer.service.test.ts rename to packages/cli/test/context/sl/semantic-layer.service.test.ts index 0844a3c5..f8b919bb 100644 --- a/packages/cli/src/context/sl/semantic-layer.service.test.ts +++ b/packages/cli/test/context/sl/semantic-layer.service.test.ts @@ -11,9 +11,9 @@ import { SemanticLayerService, toResolvedWire, UnknownColumnOverrideError, -} from './semantic-layer.service.js'; -import { resolvedSourceSchema, sourceDefinitionSchema, sourceOverlaySchema } from './schemas.js'; -import type { SemanticLayerSource } from './types.js'; +} from '../../../src/context/sl/semantic-layer.service.js'; +import { resolvedSourceSchema, sourceDefinitionSchema, sourceOverlaySchema } from '../../../src/context/sl/schemas.js'; +import type { SemanticLayerSource } from '../../../src/context/sl/types.js'; const pythonPort = { validateSources: vi.fn(), @@ -847,7 +847,7 @@ describe('loadAllSources — standalone enrichment via inherits_columns_from', ( }); }); - it('reports file-attributed errors for legacy overlay column patches', async () => { + it('reports file-attributed errors for overlay columns that shadow manifest columns', async () => { const schemaPath = 'semantic-layer/conn-1/_schema/marts.yaml'; const overlayPath = 'semantic-layer/conn-1/orders.yaml'; configService.listFiles.mockResolvedValue({ files: [schemaPath, overlayPath] }); @@ -871,7 +871,8 @@ describe('loadAllSources — standalone enrichment via inherits_columns_from', ( const { loadErrors } = await service.loadAllSources('conn-1'); expect(loadErrors.join('\n')).toContain(overlayPath); - expect(loadErrors.join('\n')).toContain("move it to 'column_overrides:'"); + expect(loadErrors.join('\n')).toContain("column 'id' in columns already exists on manifest source 'orders'"); + expect(loadErrors.join('\n')).not.toContain('column_overrides'); }); it('reports and logs directory listing failures instead of treating them as empty sources', async () => { diff --git a/packages/cli/src/context/sl/sl-dictionary-profile.test.ts b/packages/cli/test/context/sl/sl-dictionary-profile.test.ts similarity index 94% rename from packages/cli/src/context/sl/sl-dictionary-profile.test.ts rename to packages/cli/test/context/sl/sl-dictionary-profile.test.ts index f7aa3854..7a36924f 100644 --- a/packages/cli/src/context/sl/sl-dictionary-profile.test.ts +++ b/packages/cli/test/context/sl/sl-dictionary-profile.test.ts @@ -2,8 +2,8 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; -import { loadLatestSlDictionaryEntries } from './sl-dictionary-profile.js'; +import { initKtxProject, type KtxLocalProject } from '../../../src/context/project/project.js'; +import { loadLatestSlDictionaryEntries } from '../../../src/context/sl/sl-dictionary-profile.js'; describe('loadLatestSlDictionaryEntries', () => { let tempDir: string; diff --git a/packages/cli/src/context/sl/sl-search.service.test.ts b/packages/cli/test/context/sl/sl-search.service.test.ts similarity index 98% rename from packages/cli/src/context/sl/sl-search.service.test.ts rename to packages/cli/test/context/sl/sl-search.service.test.ts index 164c3954..052cdee9 100644 --- a/packages/cli/src/context/sl/sl-search.service.test.ts +++ b/packages/cli/test/context/sl/sl-search.service.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { buildSemanticLayerSourceSearchText, SlSearchService } from './sl-search.service.js'; -import type { SemanticLayerSource } from './types.js'; +import { buildSemanticLayerSourceSearchText, SlSearchService } from '../../../src/context/sl/sl-search.service.js'; +import type { SemanticLayerSource } from '../../../src/context/sl/types.js'; describe('SlSearchService', () => { it('builds search text from source, columns, measures, and joins', () => { diff --git a/packages/cli/src/context/sl/sqlite-sl-sources-index.test.ts b/packages/cli/test/context/sl/sqlite-sl-sources-index.test.ts similarity index 98% rename from packages/cli/src/context/sl/sqlite-sl-sources-index.test.ts rename to packages/cli/test/context/sl/sqlite-sl-sources-index.test.ts index 91a7727e..d0002dae 100644 --- a/packages/cli/src/context/sl/sqlite-sl-sources-index.test.ts +++ b/packages/cli/test/context/sl/sqlite-sl-sources-index.test.ts @@ -2,7 +2,7 @@ import { access, mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { SqliteSlSourcesIndex } from './sqlite-sl-sources-index.js'; +import { SqliteSlSourcesIndex } from '../../../src/context/sl/sqlite-sl-sources-index.js'; describe('SqliteSlSourcesIndex', () => { let tempDir: string; diff --git a/packages/cli/src/context/sl/tools/connection-id-schema.test.ts b/packages/cli/test/context/sl/tools/connection-id-schema.test.ts similarity index 87% rename from packages/cli/src/context/sl/tools/connection-id-schema.test.ts rename to packages/cli/test/context/sl/tools/connection-id-schema.test.ts index 48e023e5..1108ce96 100644 --- a/packages/cli/src/context/sl/tools/connection-id-schema.test.ts +++ b/packages/cli/test/context/sl/tools/connection-id-schema.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { slToolConnectionIdSchema } from './connection-id-schema.js'; +import { slToolConnectionIdSchema } from '../../../../src/context/sl/tools/connection-id-schema.js'; describe('slToolConnectionIdSchema', () => { it('accepts app UUIDs and local project connection ids', () => { diff --git a/packages/cli/src/context/sl/tools/sl-discover.tool.test.ts b/packages/cli/test/context/sl/tools/sl-discover.tool.test.ts similarity index 86% rename from packages/cli/src/context/sl/tools/sl-discover.tool.test.ts rename to packages/cli/test/context/sl/tools/sl-discover.tool.test.ts index 6dc30478..6a673d2c 100644 --- a/packages/cli/src/context/sl/tools/sl-discover.tool.test.ts +++ b/packages/cli/test/context/sl/tools/sl-discover.tool.test.ts @@ -1,9 +1,9 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolContext } from '../../../context/tools/base-tool.js'; -import type { ToolSession } from '../../../context/tools/tool-session.js'; -import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; -import type { SemanticLayerSource } from '../types.js'; -import { SlDiscoverTool } from './sl-discover.tool.js'; +import type { ToolContext } from '../../../../src/context/tools/base-tool.js'; +import type { ToolSession } from '../../../../src/context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../../src/context/tools/touched-sl-sources.js'; +import type { SemanticLayerSource } from '../../../../src/context/sl/types.js'; +import { SlDiscoverTool } from '../../../../src/context/sl/tools/sl-discover.tool.js'; function makeTool() { const semanticLayerService = { diff --git a/packages/cli/src/context/sl/tools/sl-edit-source.tool.test.ts b/packages/cli/test/context/sl/tools/sl-edit-source.tool.test.ts similarity index 96% rename from packages/cli/src/context/sl/tools/sl-edit-source.tool.test.ts rename to packages/cli/test/context/sl/tools/sl-edit-source.tool.test.ts index cf66baf8..fee83ea6 100644 --- a/packages/cli/src/context/sl/tools/sl-edit-source.tool.test.ts +++ b/packages/cli/test/context/sl/tools/sl-edit-source.tool.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../../context/tools/tool-session.js'; -import { createTouchedSlSources, hasTouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; -import type { ToolContext } from '../../../context/tools/base-tool.js'; -import { SlEditSourceTool } from './sl-edit-source.tool.js'; +import type { ToolSession } from '../../../../src/context/tools/tool-session.js'; +import { createTouchedSlSources, hasTouchedSlSource } from '../../../../src/context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../../src/context/tools/base-tool.js'; +import { SlEditSourceTool } from '../../../../src/context/sl/tools/sl-edit-source.tool.js'; function makeTool(overrides: any = {}) { const semanticLayerService = { diff --git a/packages/cli/src/context/sl/tools/sl-read-source.tool.session.test.ts b/packages/cli/test/context/sl/tools/sl-read-source.tool.session.test.ts similarity index 87% rename from packages/cli/src/context/sl/tools/sl-read-source.tool.session.test.ts rename to packages/cli/test/context/sl/tools/sl-read-source.tool.session.test.ts index 481c4cfe..121a012b 100644 --- a/packages/cli/src/context/sl/tools/sl-read-source.tool.session.test.ts +++ b/packages/cli/test/context/sl/tools/sl-read-source.tool.session.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../../context/tools/tool-session.js'; -import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; -import type { ToolContext } from '../../../context/tools/base-tool.js'; -import { SlReadSourceTool } from './sl-read-source.tool.js'; +import type { ToolSession } from '../../../../src/context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../../src/context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../../src/context/tools/base-tool.js'; +import { SlReadSourceTool } from '../../../../src/context/sl/tools/sl-read-source.tool.js'; function makeTool(overrides: Partial> = {}) { const semanticLayerService = { diff --git a/packages/cli/src/context/sl/tools/sl-rollback.tool.test.ts b/packages/cli/test/context/sl/tools/sl-rollback.tool.test.ts similarity index 90% rename from packages/cli/src/context/sl/tools/sl-rollback.tool.test.ts rename to packages/cli/test/context/sl/tools/sl-rollback.tool.test.ts index 5a1927a4..6d2d787a 100644 --- a/packages/cli/src/context/sl/tools/sl-rollback.tool.test.ts +++ b/packages/cli/test/context/sl/tools/sl-rollback.tool.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../../context/tools/tool-session.js'; -import { createTouchedSlSources, hasTouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; -import type { ToolContext } from '../../../context/tools/base-tool.js'; -import { SlRollbackTool } from './sl-rollback.tool.js'; +import type { ToolSession } from '../../../../src/context/tools/tool-session.js'; +import { createTouchedSlSources, hasTouchedSlSource } from '../../../../src/context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../../src/context/tools/base-tool.js'; +import { SlRollbackTool } from '../../../../src/context/sl/tools/sl-rollback.tool.js'; function makeSession(overrides: Partial = {}): ToolSession { return { diff --git a/packages/cli/src/context/sl/tools/sl-validate.tool.test.ts b/packages/cli/test/context/sl/tools/sl-validate.tool.test.ts similarity index 84% rename from packages/cli/src/context/sl/tools/sl-validate.tool.test.ts rename to packages/cli/test/context/sl/tools/sl-validate.tool.test.ts index f0c18eac..ff7cee5f 100644 --- a/packages/cli/src/context/sl/tools/sl-validate.tool.test.ts +++ b/packages/cli/test/context/sl/tools/sl-validate.tool.test.ts @@ -1,10 +1,10 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../../context/tools/tool-session.js'; -import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; -import type { ToolContext } from '../../../context/tools/base-tool.js'; -import type { SemanticLayerService } from '../semantic-layer.service.js'; -import type { SemanticLayerSource } from '../types.js'; -import { SlValidateTool, validateSemanticLayerEndpoint } from './sl-validate.tool.js'; +import type { ToolSession } from '../../../../src/context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../../src/context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../../src/context/tools/base-tool.js'; +import type { SemanticLayerService } from '../../../../src/context/sl/semantic-layer.service.js'; +import type { SemanticLayerSource } from '../../../../src/context/sl/types.js'; +import { SlValidateTool, validateSemanticLayerEndpoint } from '../../../../src/context/sl/tools/sl-validate.tool.js'; describe('validateSemanticLayerEndpoint', () => { it('uses the connection warehouse dialect, not hardcoded postgres', async () => { diff --git a/packages/cli/src/context/sl/tools/sl-warehouse-validation.test.ts b/packages/cli/test/context/sl/tools/sl-warehouse-validation.test.ts similarity index 98% rename from packages/cli/src/context/sl/tools/sl-warehouse-validation.test.ts rename to packages/cli/test/context/sl/tools/sl-warehouse-validation.test.ts index 5796cdb7..d8d45a81 100644 --- a/packages/cli/src/context/sl/tools/sl-warehouse-validation.test.ts +++ b/packages/cli/test/context/sl/tools/sl-warehouse-validation.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { validateSingleSource } from './sl-warehouse-validation.js'; +import { validateSingleSource } from '../../../../src/context/sl/tools/sl-warehouse-validation.js'; function makeDeps(opts: { sourceYaml: string; executeQuery: ReturnType }) { return { diff --git a/packages/cli/src/context/sl/tools/sl-write-source.tool.test.ts b/packages/cli/test/context/sl/tools/sl-write-source.tool.test.ts similarity index 97% rename from packages/cli/src/context/sl/tools/sl-write-source.tool.test.ts rename to packages/cli/test/context/sl/tools/sl-write-source.tool.test.ts index f168095c..ab3ee308 100644 --- a/packages/cli/src/context/sl/tools/sl-write-source.tool.test.ts +++ b/packages/cli/test/context/sl/tools/sl-write-source.tool.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../../context/tools/tool-session.js'; -import { createTouchedSlSources, hasTouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; -import type { ToolContext } from '../../../context/tools/base-tool.js'; -import { SlWriteSourceTool } from './sl-write-source.tool.js'; +import type { ToolSession } from '../../../../src/context/tools/tool-session.js'; +import { createTouchedSlSources, hasTouchedSlSource } from '../../../../src/context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../../src/context/tools/base-tool.js'; +import { SlWriteSourceTool } from '../../../../src/context/sl/tools/sl-write-source.tool.js'; function makeTool(overrides: Partial> = {}) { const semanticLayerService = { diff --git a/packages/cli/src/context/sql-analysis/http-sql-analysis-port.test.ts b/packages/cli/test/context/sql-analysis/http-sql-analysis-port.test.ts similarity index 71% rename from packages/cli/src/context/sql-analysis/http-sql-analysis-port.test.ts rename to packages/cli/test/context/sql-analysis/http-sql-analysis-port.test.ts index 2d759369..df32fb8d 100644 --- a/packages/cli/src/context/sql-analysis/http-sql-analysis-port.test.ts +++ b/packages/cli/test/context/sql-analysis/http-sql-analysis-port.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { createHttpSqlAnalysisPort } from './http-sql-analysis-port.js'; +import { createHttpSqlAnalysisPort } from '../../../src/context/sql-analysis/http-sql-analysis-port.js'; describe('createHttpSqlAnalysisPort', () => { it('calls the SQL-analysis fingerprint endpoint and maps snake_case response fields', async () => { @@ -49,7 +49,10 @@ describe('createHttpSqlAnalysisPort', () => { const requestJson = vi.fn(async () => ({ results: { orders: { - tables_touched: ['public.orders', 'public.customers'], + tables_touched: [ + { catalog: null, db: 'public', name: 'orders' }, + { catalog: null, db: 'public', name: 'customers' }, + ], columns_by_clause: { select: ['status'], where: ['created_at'], @@ -79,7 +82,10 @@ describe('createHttpSqlAnalysisPort', () => { [ 'orders', { - tablesTouched: ['public.orders', 'public.customers'], + tablesTouched: [ + { catalog: null, db: 'public', name: 'orders' }, + { catalog: null, db: 'public', name: 'customers' }, + ], columnsByClause: { select: ['status'], where: ['created_at'], @@ -108,6 +114,62 @@ describe('createHttpSqlAnalysisPort', () => { }); }); + it('passes an optional catalog and maps structured table refs for SQL batch analysis', async () => { + const requestJson = vi.fn(async () => ({ + results: { + orders: { + tables_touched: [ + { catalog: null, db: 'orbit_raw', name: 'accounts' }, + { catalog: 'demo_project', db: 'orbit_analytics', name: 'orders' }, + ], + columns_by_clause: { select: ['id'] }, + error: null, + }, + }, + })); + const port = createHttpSqlAnalysisPort({ baseUrl: 'http://python.test', requestJson }); + + await expect( + port.analyzeBatch( + [{ id: 'orders', sql: 'select id from accounts' }], + 'postgres', + { + catalog: { + tables: [ + { catalog: null, db: 'orbit_raw', name: 'accounts', columns: ['id'] }, + { catalog: 'demo_project', db: 'orbit_analytics', name: 'orders', columns: ['id'] }, + ], + }, + }, + ), + ).resolves.toEqual( + new Map([ + [ + 'orders', + { + tablesTouched: [ + { catalog: null, db: 'orbit_raw', name: 'accounts' }, + { catalog: 'demo_project', db: 'orbit_analytics', name: 'orders' }, + ], + columnsByClause: { select: ['id'] }, + error: null, + }, + ], + ]), + ); + + expect(requestJson).toHaveBeenCalledWith('/sql/analyze-batch', { + dialect: 'postgres', + items: [{ id: 'orders', sql: 'select id from accounts' }], + catalog: { + tables: [ + { catalog: null, db: 'orbit_raw', name: 'accounts', columns: ['id'] }, + { catalog: 'demo_project', db: 'orbit_analytics', name: 'orders', columns: ['id'] }, + ], + }, + }); + }); + it('maps read-only SQL validation responses', async () => { const requests: Array<{ path: string; payload: Record }> = []; const port = createHttpSqlAnalysisPort({ @@ -150,7 +212,7 @@ describe('createHttpSqlAnalysisPort', () => { const requestJson = vi.fn(async () => ({ results: { orders: { - tables_touched: ['public.orders'], + tables_touched: [{ catalog: null, db: 'public', name: 'orders' }], columns_by_clause: { select: ['status'], where: [42] }, error: null, }, diff --git a/packages/cli/src/context/test/make-local-git-repo.ts b/packages/cli/test/context/test/make-local-git-repo.ts similarity index 95% rename from packages/cli/src/context/test/make-local-git-repo.ts rename to packages/cli/test/context/test/make-local-git-repo.ts index a7b4c662..c60187ac 100644 --- a/packages/cli/src/context/test/make-local-git-repo.ts +++ b/packages/cli/test/context/test/make-local-git-repo.ts @@ -1,7 +1,7 @@ import { cp, mkdir, rm, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import type { SimpleGit } from 'simple-git'; -import { createSimpleGit } from '../ingest/git-env.js'; +import { createSimpleGit } from '../../../src/context/ingest/git-env.js'; export interface LocalGitRepo { repoDir: string; diff --git a/packages/cli/src/context/tools/context-evidence-tools.test.ts b/packages/cli/test/context/tools/context-evidence-tools.test.ts similarity index 95% rename from packages/cli/src/context/tools/context-evidence-tools.test.ts rename to packages/cli/test/context/tools/context-evidence-tools.test.ts index 08a8654f..f8adb618 100644 --- a/packages/cli/src/context/tools/context-evidence-tools.test.ts +++ b/packages/cli/test/context/tools/context-evidence-tools.test.ts @@ -3,17 +3,17 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; -import { SqliteContextEvidenceStore } from '../ingest/context-evidence/sqlite-context-evidence-store.js'; -import { ContextCandidateMarkTool } from './context-candidate-mark.tool.js'; -import { ContextCandidateWriteTool } from './context-candidate-write.tool.js'; -import { ContextEvidenceNeighborsTool } from './context-evidence-neighbors.tool.js'; -import { ContextEvidenceReadTool } from './context-evidence-read.tool.js'; -import { ContextEvidenceSearchTool } from './context-evidence-search.tool.js'; -import type { ContextEvidenceToolStorePort } from './context-evidence-tool-store.js'; -import { createTouchedSlSources } from '../../context/tools/touched-sl-sources.js'; -import type { ToolContext } from '../../context/tools/base-tool.js'; -import type { ToolSession } from '../../context/tools/tool-session.js'; +import type { KtxEmbeddingPort } from '../../../src/context/core/embedding.js'; +import { SqliteContextEvidenceStore } from '../../../src/context/ingest/context-evidence/sqlite-context-evidence-store.js'; +import { ContextCandidateMarkTool } from '../../../src/context/tools/context-candidate-mark.tool.js'; +import { ContextCandidateWriteTool } from '../../../src/context/tools/context-candidate-write.tool.js'; +import { ContextEvidenceNeighborsTool } from '../../../src/context/tools/context-evidence-neighbors.tool.js'; +import { ContextEvidenceReadTool } from '../../../src/context/tools/context-evidence-read.tool.js'; +import { ContextEvidenceSearchTool } from '../../../src/context/tools/context-evidence-search.tool.js'; +import type { ContextEvidenceToolStorePort } from '../../../src/context/tools/context-evidence-tool-store.js'; +import { createTouchedSlSources } from '../../../src/context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../src/context/tools/base-tool.js'; +import type { ToolSession } from '../../../src/context/tools/tool-session.js'; const ingestContext = (): ToolContext => ({ sourceId: 'ingest', diff --git a/packages/cli/src/context/tools/touched-sl-sources.test.ts b/packages/cli/test/context/tools/touched-sl-sources.test.ts similarity index 96% rename from packages/cli/src/context/tools/touched-sl-sources.test.ts rename to packages/cli/test/context/tools/touched-sl-sources.test.ts index 818676d2..f84f5d6b 100644 --- a/packages/cli/src/context/tools/touched-sl-sources.test.ts +++ b/packages/cli/test/context/tools/touched-sl-sources.test.ts @@ -7,7 +7,7 @@ import { listTouchedSlSources, touchedSlSourceCount, touchedSlSourceNamesForConnection, -} from './touched-sl-sources.js'; +} from '../../../src/context/tools/touched-sl-sources.js'; describe('target-aware touched SL source helpers', () => { it('deduplicates by connectionId and sourceName while preserving target identity', () => { diff --git a/packages/cli/src/context/wiki/knowledge-wiki.service.test.ts b/packages/cli/test/context/wiki/knowledge-wiki.service.test.ts similarity index 98% rename from packages/cli/src/context/wiki/knowledge-wiki.service.test.ts rename to packages/cli/test/context/wiki/knowledge-wiki.service.test.ts index 88bd92ab..efc9c69c 100644 --- a/packages/cli/src/context/wiki/knowledge-wiki.service.test.ts +++ b/packages/cli/test/context/wiki/knowledge-wiki.service.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { KnowledgeWikiService, type WikiFrontmatter } from './knowledge-wiki.service.js'; +import { KnowledgeWikiService, type WikiFrontmatter } from '../../../src/context/wiki/knowledge-wiki.service.js'; function makeService() { const pagesRepository: Record> = { diff --git a/packages/cli/src/context/wiki/local-knowledge.test.ts b/packages/cli/test/context/wiki/local-knowledge.test.ts similarity index 90% rename from packages/cli/src/context/wiki/local-knowledge.test.ts rename to packages/cli/test/context/wiki/local-knowledge.test.ts index 8229d5e7..cda5ca1a 100644 --- a/packages/cli/src/context/wiki/local-knowledge.test.ts +++ b/packages/cli/test/context/wiki/local-knowledge.test.ts @@ -2,13 +2,14 @@ import { access, mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; +import { initKtxProject, type KtxLocalProject } from '../../../src/context/project/project.js'; import { + listLocalKnowledgePageKeys, listLocalKnowledgePages, readLocalKnowledgePage, searchLocalKnowledgePages, writeLocalKnowledgePage, -} from './local-knowledge.js'; +} from '../../../src/context/wiki/local-knowledge.js'; class FakeEmbeddingPort { readonly maxBatchSize = 16; @@ -102,6 +103,35 @@ describe('local knowledge helpers', () => { await expect(access(join(project.projectDir, '.ktx', 'db.sqlite'))).resolves.toBeUndefined(); }); + it('lists page keys across scopes, deduped and sorted, for completion', async () => { + await writeLocalKnowledgePage(project, { + key: 'metrics-revenue', + scope: 'GLOBAL', + summary: 'Revenue metric definition', + content: 'Revenue is recognized when an order is paid.', + }); + await writeLocalKnowledgePage(project, { + key: 'metrics-churn', + scope: 'USER', + userId: 'local', + summary: 'Churn metric definition', + content: 'Churn is measured monthly.', + }); + // Same key in both scopes must collapse to a single completion candidate. + await writeLocalKnowledgePage(project, { + key: 'metrics-revenue', + scope: 'USER', + userId: 'local', + summary: 'User override of revenue', + content: 'Local revenue note.', + }); + + await expect(listLocalKnowledgePageKeys(project, { userId: 'local' })).resolves.toEqual([ + 'metrics-churn', + 'metrics-revenue', + ]); + }); + it('adds the token lane alongside lexical wiki matches', async () => { await writeLocalKnowledgePage(project, { key: 'metrics-revenue', diff --git a/packages/cli/src/context/wiki/sqlite-knowledge-index.test.ts b/packages/cli/test/context/wiki/sqlite-knowledge-index.test.ts similarity index 98% rename from packages/cli/src/context/wiki/sqlite-knowledge-index.test.ts rename to packages/cli/test/context/wiki/sqlite-knowledge-index.test.ts index 940e9954..5a3b0dc1 100644 --- a/packages/cli/src/context/wiki/sqlite-knowledge-index.test.ts +++ b/packages/cli/test/context/wiki/sqlite-knowledge-index.test.ts @@ -2,7 +2,7 @@ import { access, mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { SqliteKnowledgeIndex, type SqliteKnowledgeIndexPage } from './sqlite-knowledge-index.js'; +import { SqliteKnowledgeIndex, type SqliteKnowledgeIndexPage } from '../../../src/context/wiki/sqlite-knowledge-index.js'; describe('SqliteKnowledgeIndex', () => { let tempDir: string; diff --git a/packages/cli/src/context/wiki/tools/wiki-list-tags.tool.test.ts b/packages/cli/test/context/wiki/tools/wiki-list-tags.tool.test.ts similarity index 91% rename from packages/cli/src/context/wiki/tools/wiki-list-tags.tool.test.ts rename to packages/cli/test/context/wiki/tools/wiki-list-tags.tool.test.ts index 49605c4f..6c6ec209 100644 --- a/packages/cli/src/context/wiki/tools/wiki-list-tags.tool.test.ts +++ b/packages/cli/test/context/wiki/tools/wiki-list-tags.tool.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolContext } from '../../../context/tools/base-tool.js'; -import { WikiListTagsTool } from './wiki-list-tags.tool.js'; +import type { ToolContext } from '../../../../src/context/tools/base-tool.js'; +import { WikiListTagsTool } from '../../../../src/context/wiki/tools/wiki-list-tags.tool.js'; describe('WikiListTagsTool', () => { const baseContext: ToolContext = { sourceId: 's', messageId: 'm', userId: 'u' }; diff --git a/packages/cli/src/context/wiki/tools/wiki-read.tool.test.ts b/packages/cli/test/context/wiki/tools/wiki-read.tool.test.ts similarity index 91% rename from packages/cli/src/context/wiki/tools/wiki-read.tool.test.ts rename to packages/cli/test/context/wiki/tools/wiki-read.tool.test.ts index ac75b174..f70b34ec 100644 --- a/packages/cli/src/context/wiki/tools/wiki-read.tool.test.ts +++ b/packages/cli/test/context/wiki/tools/wiki-read.tool.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../../context/tools/tool-session.js'; -import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; -import type { ToolContext } from '../../../context/tools/base-tool.js'; -import { WikiReadTool } from './wiki-read.tool.js'; +import type { ToolSession } from '../../../../src/context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../../src/context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../../src/context/tools/base-tool.js'; +import { WikiReadTool } from '../../../../src/context/wiki/tools/wiki-read.tool.js'; describe('WikiReadTool', () => { const baseContext: ToolContext = { sourceId: 's', messageId: 'm', userId: 'u' }; diff --git a/packages/cli/src/context/wiki/tools/wiki-remove.tool.test.ts b/packages/cli/test/context/wiki/tools/wiki-remove.tool.test.ts similarity index 93% rename from packages/cli/src/context/wiki/tools/wiki-remove.tool.test.ts rename to packages/cli/test/context/wiki/tools/wiki-remove.tool.test.ts index 8130613c..afabc8c0 100644 --- a/packages/cli/src/context/wiki/tools/wiki-remove.tool.test.ts +++ b/packages/cli/test/context/wiki/tools/wiki-remove.tool.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../../context/tools/tool-session.js'; -import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; -import type { ToolContext } from '../../../context/tools/base-tool.js'; -import { WikiRemoveTool } from './wiki-remove.tool.js'; +import type { ToolSession } from '../../../../src/context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../../src/context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../../src/context/tools/base-tool.js'; +import { WikiRemoveTool } from '../../../../src/context/wiki/tools/wiki-remove.tool.js'; describe('WikiRemoveTool', () => { const baseContext: ToolContext = { sourceId: 's', messageId: 'm', userId: 'u' }; diff --git a/packages/cli/src/context/wiki/tools/wiki-search.tool.test.ts b/packages/cli/test/context/wiki/tools/wiki-search.tool.test.ts similarity index 93% rename from packages/cli/src/context/wiki/tools/wiki-search.tool.test.ts rename to packages/cli/test/context/wiki/tools/wiki-search.tool.test.ts index 24840a4f..d21f9825 100644 --- a/packages/cli/src/context/wiki/tools/wiki-search.tool.test.ts +++ b/packages/cli/test/context/wiki/tools/wiki-search.tool.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { WikiSearchTool } from './wiki-search.tool.js'; +import { WikiSearchTool } from '../../../../src/context/wiki/tools/wiki-search.tool.js'; describe('WikiSearchTool', () => { it('searches through the injected wiki adapter port', async () => { diff --git a/packages/cli/src/context/wiki/tools/wiki-write.tool.test.ts b/packages/cli/test/context/wiki/tools/wiki-write.tool.test.ts similarity index 97% rename from packages/cli/src/context/wiki/tools/wiki-write.tool.test.ts rename to packages/cli/test/context/wiki/tools/wiki-write.tool.test.ts index ad2bc54b..deadd716 100644 --- a/packages/cli/src/context/wiki/tools/wiki-write.tool.test.ts +++ b/packages/cli/test/context/wiki/tools/wiki-write.tool.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../../context/tools/tool-session.js'; -import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; -import type { ToolContext } from '../../../context/tools/base-tool.js'; -import { WikiWriteTool } from './wiki-write.tool.js'; +import type { ToolSession } from '../../../../src/context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../../src/context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../../src/context/tools/base-tool.js'; +import { WikiWriteTool } from '../../../../src/context/wiki/tools/wiki-write.tool.js'; function makeTool(overrides: any = {}) { const wikiService = { diff --git a/packages/cli/src/context/wiki/wiki-ref-validation.test.ts b/packages/cli/test/context/wiki/wiki-ref-validation.test.ts similarity index 96% rename from packages/cli/src/context/wiki/wiki-ref-validation.test.ts rename to packages/cli/test/context/wiki/wiki-ref-validation.test.ts index 6e0e8563..b6fd0012 100644 --- a/packages/cli/src/context/wiki/wiki-ref-validation.test.ts +++ b/packages/cli/test/context/wiki/wiki-ref-validation.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { findDanglingWikiRefsForActions } from './wiki-ref-validation.js'; +import { findDanglingWikiRefsForActions } from '../../../src/context/wiki/wiki-ref-validation.js'; function makeWikiService(pages: Record) { return { diff --git a/packages/cli/src/database-tree-picker.test.ts b/packages/cli/test/database-tree-picker.test.ts similarity index 73% rename from packages/cli/src/database-tree-picker.test.ts rename to packages/cli/test/database-tree-picker.test.ts index 4dd1dca3..182f0235 100644 --- a/packages/cli/src/database-tree-picker.test.ts +++ b/packages/cli/test/database-tree-picker.test.ts @@ -4,9 +4,9 @@ import { type DatabaseScopePromptAdapter, type DatabaseTreePickerRenderer, type PickDatabaseScopeArgs, -} from './database-tree-picker.js'; -import type { TreePickerChrome, TreePickerResult } from './tree-picker-tui.js'; -import type { PickerState } from './tree-picker-state.js'; +} from '../src/database-tree-picker.js'; +import type { TreePickerChrome, TreePickerResult } from '../src/tree-picker-tui.js'; +import type { PickerState } from '../src/tree-picker-state.js'; function makeIo() { let stdout = ''; @@ -52,10 +52,10 @@ function captureRenderer(): { } const discovered = [ - { schema: 'analytics', name: 'customers', kind: 'table' as const }, - { schema: 'analytics', name: 'orders', kind: 'table' as const }, - { schema: 'public', name: 'events', kind: 'view' as const }, - { schema: 'public', name: 'sessions', kind: 'table' as const }, + { catalog: null, schema: 'analytics', name: 'customers', kind: 'table' as const }, + { catalog: null, schema: 'analytics', name: 'orders', kind: 'table' as const }, + { catalog: null, schema: 'public', name: 'events', kind: 'view' as const }, + { catalog: null, schema: 'public', name: 'sessions', kind: 'table' as const }, ]; function promptAdapter(overrides: Partial = {}): DatabaseScopePromptAdapter { @@ -88,7 +88,7 @@ describe('pickDatabaseScope', () => { select: vi.fn(async () => 'save'), }); const listTablesForSchemas = vi.fn(async () => [ - { schema: 'analytics', name: 'orders', kind: 'table' as const }, + { catalog: null, schema: 'analytics', name: 'orders', kind: 'table' as const }, ]); const result = await pickDatabaseScope( @@ -114,6 +114,58 @@ describe('pickDatabaseScope', () => { }); }); + it('emits fully-qualified catalog.schema.name ids for catalog-bearing drivers and round-trips existing selection', async () => { + const promptsSave = promptAdapter({ + autocompleteMultiselect: vi.fn(async () => ['analytics']), + select: vi.fn(async () => 'save'), + }); + const listTablesForSchemas = vi.fn(async () => [ + { catalog: 'project-1', schema: 'analytics', name: 'orders', kind: 'table' as const }, + { catalog: 'project-1', schema: 'analytics', name: 'customers', kind: 'table' as const }, + ]); + const saveResult = await pickDatabaseScope( + baseArgs({ + schemas: ['analytics'], + schemaSuggestion: { excluded: new Set(), suggested: new Set(['analytics']) }, + listTablesForSchemas, + prompts: promptsSave, + }), + makeIo().io, + captureRenderer().renderer, + ); + expect(saveResult).toEqual({ + kind: 'selected', + activeSchemas: ['analytics'], + enabledTables: ['project-1.analytics.orders', 'project-1.analytics.customers'], + }); + + const { renderer, capture, setResult } = captureRenderer(); + setResult({ + kind: 'save', + selectedIds: ['project-1.analytics.orders'], + }); + const refineResult = await pickDatabaseScope( + baseArgs({ + schemas: ['analytics'], + schemaSuggestion: { excluded: new Set(), suggested: new Set(['analytics']) }, + existing: { enabledTables: ['project-1.analytics.orders'] }, + listTablesForSchemas, + prompts: promptAdapter({ + autocompleteMultiselect: vi.fn(async () => ['analytics']), + select: vi.fn(async () => 'refine'), + }), + }), + makeIo().io, + renderer, + ); + expect(refineResult).toEqual({ + kind: 'selected', + activeSchemas: ['analytics'], + enabledTables: ['project-1.analytics.orders'], + }); + expect([...(capture.state?.checked ?? [])]).toContain('project-1.analytics.orders'); + }); + it('routes partial existing allowlists through Stage 2 so save preserves table selections', async () => { const { renderer, setResult } = captureRenderer(); setResult({ kind: 'save', selectedIds: ['analytics.customers'] }); @@ -122,8 +174,8 @@ describe('pickDatabaseScope', () => { select: vi.fn(async () => 'save'), }); const listTablesForSchemas = vi.fn(async () => [ - { schema: 'analytics', name: 'customers', kind: 'table' as const }, - { schema: 'analytics', name: 'orders', kind: 'table' as const }, + { catalog: null, schema: 'analytics', name: 'customers', kind: 'table' as const }, + { catalog: null, schema: 'analytics', name: 'orders', kind: 'table' as const }, ]); const result = await pickDatabaseScope( @@ -161,6 +213,7 @@ describe('pickDatabaseScope', () => { 'public.events', 'public.sessions', ]); + expect([...(capture.state?.expanded ?? [])].sort()).toEqual(['analytics', 'public']); expect(capture.state?.byId.get('public.events')?.title).toBe('events (view)'); }); diff --git a/packages/cli/src/demo-assets.test.ts b/packages/cli/test/demo-assets.test.ts similarity index 99% rename from packages/cli/src/demo-assets.test.ts rename to packages/cli/test/demo-assets.test.ts index 052eda83..80573d3d 100644 --- a/packages/cli/src/demo-assets.test.ts +++ b/packages/cli/test/demo-assets.test.ts @@ -10,7 +10,7 @@ import { defaultDemoProjectDir, ensureDemoProject, ensureSeededDemoProject, -} from './demo-assets.js'; +} from '../src/demo-assets.js'; const packagedDemoSource = 'packaged-orbit-demo'; diff --git a/packages/cli/src/demo-metrics.test.ts b/packages/cli/test/demo-metrics.test.ts similarity index 97% rename from packages/cli/src/demo-metrics.test.ts rename to packages/cli/test/demo-metrics.test.ts index 9e40be36..fcfe90c8 100644 --- a/packages/cli/src/demo-metrics.test.ts +++ b/packages/cli/test/demo-metrics.test.ts @@ -1,4 +1,4 @@ -import type { MemoryFlowEvent, MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; +import type { MemoryFlowEvent, MemoryFlowReplayInput } from '../src/context/ingest/memory-flow/types.js'; import { describe, expect, it } from 'vitest'; import { buildDemoMetrics, @@ -8,7 +8,7 @@ import { formatTokens, formatTokensPerSec, progressBar, -} from './demo-metrics.js'; +} from '../src/demo-metrics.js'; function snapshot(events: MemoryFlowEvent[], overrides: Partial = {}): MemoryFlowReplayInput { return { diff --git a/packages/cli/src/doctor.test.ts b/packages/cli/test/doctor.test.ts similarity index 91% rename from packages/cli/src/doctor.test.ts rename to packages/cli/test/doctor.test.ts index 6064cfeb..242331e8 100644 --- a/packages/cli/src/doctor.test.ts +++ b/packages/cli/test/doctor.test.ts @@ -7,7 +7,7 @@ import { runKtxDoctor, runSetupDoctorChecks, type DoctorCheck, -} from './doctor.js'; +} from '../src/doctor.js'; function makeIo() { let stdout = ''; @@ -30,6 +30,30 @@ function makeIo() { }; } +function fakeDoctorHistoricSqlRunner() { + return { + dialect: 'postgres' as const, + catalogName: 'pg_stat_statements', + async run() { + return { warnings: [], info: [] }; + }, + formatSuccessDetail(result: unknown) { + const typed = result as { pgServerVersion?: string; warnings: string[]; info?: string[] }; + const info = typed.info && typed.info.length > 0 ? `; ${typed.info.join('; ')}` : ''; + return { + detail: `pg_stat_statements ready (${typed.pgServerVersion ?? 'PostgreSQL 16.4'})${info}`, + warnings: typed.warnings, + }; + }, + fixAdvice(error: unknown) { + return { + failHeadline: error instanceof Error ? error.message : String(error), + remediation: 'Fix query-history grants.', + }; + }, + }; +} + describe('formatDoctorReport', () => { it('shows the failing check and its fix in plain output', () => { const checks: DoctorCheck[] = [ @@ -398,6 +422,8 @@ describe('runKtxDoctor', () => { 'llm:', ' provider:', ' backend: anthropic', + ' models:', + ' default: claude-sonnet-4-5', '', ].join('\n'), 'utf-8', @@ -519,6 +545,8 @@ describe('runKtxDoctor', () => { 'llm:', ' provider:', ' backend: anthropic', + ' models:', + ' default: claude-sonnet-4-5', 'ingest:', ' adapters:', ' - live-database', @@ -539,14 +567,19 @@ describe('runKtxDoctor', () => { { command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, testIo.io, { - postgresQueryHistoryProbe: async () => { + queryHistoryReadinessProbe: async () => { probeCalls += 1; return { - pgServerVersion: 'PostgreSQL 16.4', - warnings: [], - info: [ - 'pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', - ], + ok: true, + dialect: 'postgres', + runner: fakeDoctorHistoricSqlRunner(), + result: { + pgServerVersion: 'PostgreSQL 16.4', + warnings: [], + info: [ + 'pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', + ], + }, }; }, }, @@ -558,7 +591,7 @@ describe('runKtxDoctor', () => { expect(out).toContain('Query history'); expect(out).toContain('warehouse'); expect(out).toContain('pg_stat_statements ready (PostgreSQL 16.4)'); - expect(out).toContain('info: pg_stat_statements.max is 1000'); + expect(out).toContain('pg_stat_statements.max is 1000'); expect(out).not.toContain('Update the Postgres parameter group or config'); expect(out).toContain('ktx status --json'); expect(out).toContain('ktx sl'); @@ -597,7 +630,7 @@ describe('runKtxDoctor', () => { expect(testIo.stdout()).toContain('ktx setup'); }); - it('warns about stale and unsupported per-driver connection fields', async () => { + it('does not warn about removed-field migration hints', async () => { process.env.ANTHROPIC_API_KEY = 'test-key'; // pragma: allowlist secret process.env.WAREHOUSE_DATABASE_URL = 'postgresql://reader@example.test/warehouse'; process.env.NOTION_TOKEN = 'notion-secret'; @@ -623,6 +656,8 @@ describe('runKtxDoctor', () => { 'llm:', ' provider:', ' backend: anthropic', + ' models:', + ' default: claude-sonnet-4-5', '', ].join('\n'), 'utf-8', @@ -634,20 +669,24 @@ describe('runKtxDoctor', () => { { command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, testIo.io, { - postgresQueryHistoryProbe: async () => ({ - pgServerVersion: 'PostgreSQL 16.4', - warnings: [], - info: [], + queryHistoryReadinessProbe: async () => ({ + ok: true, + dialect: 'postgres', + runner: fakeDoctorHistoricSqlRunner(), + result: { + pgServerVersion: 'PostgreSQL 16.4', + warnings: [], + info: [], + }, }), }, ), ).resolves.toBe(0); const out = testIo.stdout(); - expect(out).toContain('Warnings'); - expect(out).toContain('connections.warehouse.readonly is no longer used.'); - expect(out).toContain('connections.local.file_path was removed.'); - expect(out).toContain('connections.docs.last_successful_cursor is local sync state.'); + expect(out).not.toContain('connections.warehouse.readonly is no longer used.'); + expect(out).not.toContain('connections.local.file_path was removed.'); + expect(out).not.toContain('connections.docs.last_successful_cursor is local sync state.'); delete process.env.ANTHROPIC_API_KEY; delete process.env.WAREHOUSE_DATABASE_URL; delete process.env.NOTION_TOKEN; @@ -665,6 +704,8 @@ describe('runKtxDoctor', () => { 'llm:', ' provider:', ' backend: anthropic', + ' models:', + ' default: claude-sonnet-4-5', 'ingest:', ' adapters:', ' - live-database', @@ -843,9 +884,14 @@ describe('runKtxDoctor', () => { { command: 'validate', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, testIo.io, { - postgresQueryHistoryProbe: async () => { + queryHistoryReadinessProbe: async () => { probeCalls += 1; - return { pgServerVersion: 'PostgreSQL 16.4', warnings: [], info: [] }; + return { + ok: true, + dialect: 'postgres', + runner: fakeDoctorHistoricSqlRunner(), + result: { pgServerVersion: 'PostgreSQL 16.4', warnings: [], info: [] }, + }; }, }, ), diff --git a/packages/cli/src/embedding-resolution.test.ts b/packages/cli/test/embedding-resolution.test.ts similarity index 94% rename from packages/cli/src/embedding-resolution.test.ts rename to packages/cli/test/embedding-resolution.test.ts index 40c71538..d9546c36 100644 --- a/packages/cli/src/embedding-resolution.test.ts +++ b/packages/cli/test/embedding-resolution.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it, vi } from 'vitest'; -import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; -import type { KtxLocalProject } from './context/project/project.js'; -import { resolveProjectEmbeddingProvider } from './embedding-resolution.js'; -import type { ManagedLocalEmbeddingsDaemon } from './managed-local-embeddings.js'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from '../src/context/project/config.js'; +import type { KtxLocalProject } from '../src/context/project/project.js'; +import { resolveProjectEmbeddingProvider } from '../src/embedding-resolution.js'; +import type { ManagedLocalEmbeddingsDaemon } from '../src/managed-local-embeddings.js'; function projectWithConfig(config: KtxProjectConfig): KtxLocalProject { return { diff --git a/packages/cli/src/example-smoke.test.ts b/packages/cli/test/example-smoke.test.ts similarity index 100% rename from packages/cli/src/example-smoke.test.ts rename to packages/cli/test/example-smoke.test.ts diff --git a/packages/cli/src/test/fixtures/lookml/extends-chain/orders.model.lkml b/packages/cli/test/fixtures/lookml/extends-chain/orders.model.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/extends-chain/orders.model.lkml rename to packages/cli/test/fixtures/lookml/extends-chain/orders.model.lkml diff --git a/packages/cli/src/test/fixtures/lookml/extends-chain/views/base.view.lkml b/packages/cli/test/fixtures/lookml/extends-chain/views/base.view.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/extends-chain/views/base.view.lkml rename to packages/cli/test/fixtures/lookml/extends-chain/views/base.view.lkml diff --git a/packages/cli/src/test/fixtures/lookml/extends-chain/views/orders.view.lkml b/packages/cli/test/fixtures/lookml/extends-chain/views/orders.view.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/extends-chain/views/orders.view.lkml rename to packages/cli/test/fixtures/lookml/extends-chain/views/orders.view.lkml diff --git a/packages/cli/src/test/fixtures/lookml/extends-chain/views/orders_ext.view.lkml b/packages/cli/test/fixtures/lookml/extends-chain/views/orders_ext.view.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/extends-chain/views/orders_ext.view.lkml rename to packages/cli/test/fixtures/lookml/extends-chain/views/orders_ext.view.lkml diff --git a/packages/cli/src/test/fixtures/lookml/multi-model/marketing.model.lkml b/packages/cli/test/fixtures/lookml/multi-model/marketing.model.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/multi-model/marketing.model.lkml rename to packages/cli/test/fixtures/lookml/multi-model/marketing.model.lkml diff --git a/packages/cli/src/test/fixtures/lookml/multi-model/orders.model.lkml b/packages/cli/test/fixtures/lookml/multi-model/orders.model.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/multi-model/orders.model.lkml rename to packages/cli/test/fixtures/lookml/multi-model/orders.model.lkml diff --git a/packages/cli/src/test/fixtures/lookml/multi-model/views/campaigns.view.lkml b/packages/cli/test/fixtures/lookml/multi-model/views/campaigns.view.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/multi-model/views/campaigns.view.lkml rename to packages/cli/test/fixtures/lookml/multi-model/views/campaigns.view.lkml diff --git a/packages/cli/src/test/fixtures/lookml/multi-model/views/orders.view.lkml b/packages/cli/test/fixtures/lookml/multi-model/views/orders.view.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/multi-model/views/orders.view.lkml rename to packages/cli/test/fixtures/lookml/multi-model/views/orders.view.lkml diff --git a/packages/cli/src/test/fixtures/lookml/multi-model/views/shared_dims.view.lkml b/packages/cli/test/fixtures/lookml/multi-model/views/shared_dims.view.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/multi-model/views/shared_dims.view.lkml rename to packages/cli/test/fixtures/lookml/multi-model/views/shared_dims.view.lkml diff --git a/packages/cli/src/test/fixtures/lookml/single-model/orders.model.lkml b/packages/cli/test/fixtures/lookml/single-model/orders.model.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/single-model/orders.model.lkml rename to packages/cli/test/fixtures/lookml/single-model/orders.model.lkml diff --git a/packages/cli/src/test/fixtures/lookml/single-model/views/customers.view.lkml b/packages/cli/test/fixtures/lookml/single-model/views/customers.view.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/single-model/views/customers.view.lkml rename to packages/cli/test/fixtures/lookml/single-model/views/customers.view.lkml diff --git a/packages/cli/src/test/fixtures/lookml/single-model/views/orders.view.lkml b/packages/cli/test/fixtures/lookml/single-model/views/orders.view.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/single-model/views/orders.view.lkml rename to packages/cli/test/fixtures/lookml/single-model/views/orders.view.lkml diff --git a/packages/cli/src/test/fixtures/lookml/three-churn/billing.model.lkml b/packages/cli/test/fixtures/lookml/three-churn/billing.model.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/three-churn/billing.model.lkml rename to packages/cli/test/fixtures/lookml/three-churn/billing.model.lkml diff --git a/packages/cli/src/test/fixtures/lookml/three-churn/customers.model.lkml b/packages/cli/test/fixtures/lookml/three-churn/customers.model.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/three-churn/customers.model.lkml rename to packages/cli/test/fixtures/lookml/three-churn/customers.model.lkml diff --git a/packages/cli/src/test/fixtures/lookml/three-churn/support.model.lkml b/packages/cli/test/fixtures/lookml/three-churn/support.model.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/three-churn/support.model.lkml rename to packages/cli/test/fixtures/lookml/three-churn/support.model.lkml diff --git a/packages/cli/src/test/fixtures/lookml/three-churn/views/billing/billing_churn_risk.view.lkml b/packages/cli/test/fixtures/lookml/three-churn/views/billing/billing_churn_risk.view.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/three-churn/views/billing/billing_churn_risk.view.lkml rename to packages/cli/test/fixtures/lookml/three-churn/views/billing/billing_churn_risk.view.lkml diff --git a/packages/cli/src/test/fixtures/lookml/three-churn/views/customers/customer_churn_risk.view.lkml b/packages/cli/test/fixtures/lookml/three-churn/views/customers/customer_churn_risk.view.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/three-churn/views/customers/customer_churn_risk.view.lkml rename to packages/cli/test/fixtures/lookml/three-churn/views/customers/customer_churn_risk.view.lkml diff --git a/packages/cli/src/test/fixtures/lookml/three-churn/views/support/support_churn_risk.view.lkml b/packages/cli/test/fixtures/lookml/three-churn/views/support/support_churn_risk.view.lkml similarity index 100% rename from packages/cli/src/test/fixtures/lookml/three-churn/views/support/support_churn_risk.view.lkml rename to packages/cli/test/fixtures/lookml/three-churn/views/support/support_churn_risk.view.lkml diff --git a/packages/cli/src/test/fixtures/metabase/card-ref/cards/10.json b/packages/cli/test/fixtures/metabase/card-ref/cards/10.json similarity index 100% rename from packages/cli/src/test/fixtures/metabase/card-ref/cards/10.json rename to packages/cli/test/fixtures/metabase/card-ref/cards/10.json diff --git a/packages/cli/src/test/fixtures/metabase/card-ref/cards/11.json b/packages/cli/test/fixtures/metabase/card-ref/cards/11.json similarity index 100% rename from packages/cli/src/test/fixtures/metabase/card-ref/cards/11.json rename to packages/cli/test/fixtures/metabase/card-ref/cards/11.json diff --git a/packages/cli/src/test/fixtures/metabase/card-ref/collections/5.json b/packages/cli/test/fixtures/metabase/card-ref/collections/5.json similarity index 100% rename from packages/cli/src/test/fixtures/metabase/card-ref/collections/5.json rename to packages/cli/test/fixtures/metabase/card-ref/collections/5.json diff --git a/packages/cli/src/test/fixtures/metabase/card-ref/databases/42.json b/packages/cli/test/fixtures/metabase/card-ref/databases/42.json similarity index 100% rename from packages/cli/src/test/fixtures/metabase/card-ref/databases/42.json rename to packages/cli/test/fixtures/metabase/card-ref/databases/42.json diff --git a/packages/cli/src/test/fixtures/metabase/card-ref/sync-config.json b/packages/cli/test/fixtures/metabase/card-ref/sync-config.json similarity index 100% rename from packages/cli/src/test/fixtures/metabase/card-ref/sync-config.json rename to packages/cli/test/fixtures/metabase/card-ref/sync-config.json diff --git a/packages/cli/src/test/fixtures/metabase/multi-collection/cards/1.json b/packages/cli/test/fixtures/metabase/multi-collection/cards/1.json similarity index 100% rename from packages/cli/src/test/fixtures/metabase/multi-collection/cards/1.json rename to packages/cli/test/fixtures/metabase/multi-collection/cards/1.json diff --git a/packages/cli/src/test/fixtures/metabase/multi-collection/cards/2.json b/packages/cli/test/fixtures/metabase/multi-collection/cards/2.json similarity index 100% rename from packages/cli/src/test/fixtures/metabase/multi-collection/cards/2.json rename to packages/cli/test/fixtures/metabase/multi-collection/cards/2.json diff --git a/packages/cli/src/test/fixtures/metabase/multi-collection/cards/3.json b/packages/cli/test/fixtures/metabase/multi-collection/cards/3.json similarity index 100% rename from packages/cli/src/test/fixtures/metabase/multi-collection/cards/3.json rename to packages/cli/test/fixtures/metabase/multi-collection/cards/3.json diff --git a/packages/cli/src/test/fixtures/metabase/multi-collection/collections/5.json b/packages/cli/test/fixtures/metabase/multi-collection/collections/5.json similarity index 100% rename from packages/cli/src/test/fixtures/metabase/multi-collection/collections/5.json rename to packages/cli/test/fixtures/metabase/multi-collection/collections/5.json diff --git a/packages/cli/src/test/fixtures/metabase/multi-collection/collections/6.json b/packages/cli/test/fixtures/metabase/multi-collection/collections/6.json similarity index 100% rename from packages/cli/src/test/fixtures/metabase/multi-collection/collections/6.json rename to packages/cli/test/fixtures/metabase/multi-collection/collections/6.json diff --git a/packages/cli/src/test/fixtures/metabase/multi-collection/databases/42.json b/packages/cli/test/fixtures/metabase/multi-collection/databases/42.json similarity index 100% rename from packages/cli/src/test/fixtures/metabase/multi-collection/databases/42.json rename to packages/cli/test/fixtures/metabase/multi-collection/databases/42.json diff --git a/packages/cli/src/test/fixtures/metabase/multi-collection/sync-config.json b/packages/cli/test/fixtures/metabase/multi-collection/sync-config.json similarity index 100% rename from packages/cli/src/test/fixtures/metabase/multi-collection/sync-config.json rename to packages/cli/test/fixtures/metabase/multi-collection/sync-config.json diff --git a/packages/cli/src/test/fixtures/metabase/simple/cards/1.json b/packages/cli/test/fixtures/metabase/simple/cards/1.json similarity index 100% rename from packages/cli/src/test/fixtures/metabase/simple/cards/1.json rename to packages/cli/test/fixtures/metabase/simple/cards/1.json diff --git a/packages/cli/src/test/fixtures/metabase/simple/cards/2.json b/packages/cli/test/fixtures/metabase/simple/cards/2.json similarity index 100% rename from packages/cli/src/test/fixtures/metabase/simple/cards/2.json rename to packages/cli/test/fixtures/metabase/simple/cards/2.json diff --git a/packages/cli/src/test/fixtures/metabase/simple/collections/5.json b/packages/cli/test/fixtures/metabase/simple/collections/5.json similarity index 100% rename from packages/cli/src/test/fixtures/metabase/simple/collections/5.json rename to packages/cli/test/fixtures/metabase/simple/collections/5.json diff --git a/packages/cli/src/test/fixtures/metabase/simple/databases/42.json b/packages/cli/test/fixtures/metabase/simple/databases/42.json similarity index 100% rename from packages/cli/src/test/fixtures/metabase/simple/databases/42.json rename to packages/cli/test/fixtures/metabase/simple/databases/42.json diff --git a/packages/cli/src/test/fixtures/metabase/simple/sync-config.json b/packages/cli/test/fixtures/metabase/simple/sync-config.json similarity index 100% rename from packages/cli/src/test/fixtures/metabase/simple/sync-config.json rename to packages/cli/test/fixtures/metabase/simple/sync-config.json diff --git a/packages/cli/src/test/fixtures/metricflow/dbt-mixed/dbt_project.yml b/packages/cli/test/fixtures/metricflow/dbt-mixed/dbt_project.yml similarity index 100% rename from packages/cli/src/test/fixtures/metricflow/dbt-mixed/dbt_project.yml rename to packages/cli/test/fixtures/metricflow/dbt-mixed/dbt_project.yml diff --git a/packages/cli/src/test/fixtures/metricflow/dbt-mixed/models/orders.yml b/packages/cli/test/fixtures/metricflow/dbt-mixed/models/orders.yml similarity index 100% rename from packages/cli/src/test/fixtures/metricflow/dbt-mixed/models/orders.yml rename to packages/cli/test/fixtures/metricflow/dbt-mixed/models/orders.yml diff --git a/packages/cli/src/test/fixtures/metricflow/extends-chain/metrics/orders_final.yml b/packages/cli/test/fixtures/metricflow/extends-chain/metrics/orders_final.yml similarity index 100% rename from packages/cli/src/test/fixtures/metricflow/extends-chain/metrics/orders_final.yml rename to packages/cli/test/fixtures/metricflow/extends-chain/metrics/orders_final.yml diff --git a/packages/cli/src/test/fixtures/metricflow/extends-chain/models/orders.yml b/packages/cli/test/fixtures/metricflow/extends-chain/models/orders.yml similarity index 100% rename from packages/cli/src/test/fixtures/metricflow/extends-chain/models/orders.yml rename to packages/cli/test/fixtures/metricflow/extends-chain/models/orders.yml diff --git a/packages/cli/src/test/fixtures/metricflow/extends-chain/models/orders_ext.yml b/packages/cli/test/fixtures/metricflow/extends-chain/models/orders_ext.yml similarity index 100% rename from packages/cli/src/test/fixtures/metricflow/extends-chain/models/orders_ext.yml rename to packages/cli/test/fixtures/metricflow/extends-chain/models/orders_ext.yml diff --git a/packages/cli/src/test/fixtures/metricflow/multi-component/models/marketing/campaigns.yml b/packages/cli/test/fixtures/metricflow/multi-component/models/marketing/campaigns.yml similarity index 100% rename from packages/cli/src/test/fixtures/metricflow/multi-component/models/marketing/campaigns.yml rename to packages/cli/test/fixtures/metricflow/multi-component/models/marketing/campaigns.yml diff --git a/packages/cli/src/test/fixtures/metricflow/multi-component/models/sales/orders.yml b/packages/cli/test/fixtures/metricflow/multi-component/models/sales/orders.yml similarity index 100% rename from packages/cli/src/test/fixtures/metricflow/multi-component/models/sales/orders.yml rename to packages/cli/test/fixtures/metricflow/multi-component/models/sales/orders.yml diff --git a/packages/cli/src/test/fixtures/metricflow/single-model/models/orders.yml b/packages/cli/test/fixtures/metricflow/single-model/models/orders.yml similarity index 100% rename from packages/cli/src/test/fixtures/metricflow/single-model/models/orders.yml rename to packages/cli/test/fixtures/metricflow/single-model/models/orders.yml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/data.sqlite b/packages/cli/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/data.sqlite similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/data.sqlite rename to packages/cli/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/data.sqlite diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/snapshot.json b/packages/cli/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/snapshot.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/snapshot.json rename to packages/cli/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/snapshot.json diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/snapshot.json b/packages/cli/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/snapshot.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/snapshot.json rename to packages/cli/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/snapshot.json diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/snapshot.json b/packages/cli/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/snapshot.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/snapshot.json rename to packages/cli/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/snapshot.json diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/data.sqlite b/packages/cli/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/data.sqlite similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/data.sqlite rename to packages/cli/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/data.sqlite diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/snapshot.json b/packages/cli/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/snapshot.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/snapshot.json rename to packages/cli/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/snapshot.json diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/snapshot.json b/packages/cli/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/snapshot.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/snapshot.json rename to packages/cli/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/snapshot.json diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/data.sqlite b/packages/cli/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/data.sqlite similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/data.sqlite rename to packages/cli/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/data.sqlite diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/snapshot.json b/packages/cli/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/snapshot.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/snapshot.json rename to packages/cli/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/snapshot.json diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/data.sqlite b/packages/cli/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/data.sqlite similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/data.sqlite rename to packages/cli/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/data.sqlite diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/snapshot.json b/packages/cli/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/snapshot.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/snapshot.json rename to packages/cli/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/snapshot.json diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/data.sqlite b/packages/cli/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/data.sqlite similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/data.sqlite rename to packages/cli/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/data.sqlite diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/snapshot.json b/packages/cli/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/snapshot.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/snapshot.json rename to packages/cli/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/snapshot.json diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/data.sqlite b/packages/cli/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/data.sqlite similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/data.sqlite rename to packages/cli/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/data.sqlite diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/snapshot.json b/packages/cli/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/snapshot.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/snapshot.json rename to packages/cli/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/snapshot.json diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/data.sqlite b/packages/cli/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/data.sqlite similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/data.sqlite rename to packages/cli/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/data.sqlite diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/snapshot.json b/packages/cli/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/snapshot.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/snapshot.json rename to packages/cli/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/snapshot.json diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/data.sqlite b/packages/cli/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/data.sqlite similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/data.sqlite rename to packages/cli/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/data.sqlite diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/snapshot.json b/packages/cli/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/snapshot.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/snapshot.json rename to packages/cli/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/snapshot.json diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/snapshot.json b/packages/cli/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/snapshot.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/snapshot.json rename to packages/cli/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/snapshot.json diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/data.sqlite b/packages/cli/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/data.sqlite similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/data.sqlite rename to packages/cli/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/data.sqlite diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/snapshot.json b/packages/cli/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/snapshot.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/snapshot.json rename to packages/cli/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/snapshot.json diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/data.sqlite b/packages/cli/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/data.sqlite similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/data.sqlite rename to packages/cli/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/data.sqlite diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/snapshot.json b/packages/cli/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/snapshot.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/snapshot.json rename to packages/cli/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/snapshot.json diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/data.sqlite b/packages/cli/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/data.sqlite similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/data.sqlite rename to packages/cli/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/data.sqlite diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/snapshot.json b/packages/cli/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/snapshot.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/snapshot.json rename to packages/cli/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/snapshot.json diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/snapshot.json b/packages/cli/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/snapshot.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/snapshot.json rename to packages/cli/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/snapshot.json diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/data.sqlite.gz b/packages/cli/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/data.sqlite.gz similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/data.sqlite.gz rename to packages/cli/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/data.sqlite.gz diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/snapshot.json.gz b/packages/cli/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/snapshot.json.gz similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/snapshot.json.gz rename to packages/cli/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/snapshot.json.gz diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/column-embeddings.json b/packages/cli/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/column-embeddings.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/column-embeddings.json rename to packages/cli/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/column-embeddings.json diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/data.sqlite b/packages/cli/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/data.sqlite similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/data.sqlite rename to packages/cli/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/data.sqlite diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/expected-links.yaml b/packages/cli/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/expected-links.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/expected-links.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/fixture.yaml b/packages/cli/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/fixture.yaml rename to packages/cli/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/fixture.yaml diff --git a/packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/snapshot.json b/packages/cli/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/snapshot.json similarity index 100% rename from packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/snapshot.json rename to packages/cli/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/snapshot.json diff --git a/packages/cli/src/index.test.ts b/packages/cli/test/index.test.ts similarity index 97% rename from packages/cli/src/index.test.ts rename to packages/cli/test/index.test.ts index c482e452..57ac4901 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/test/index.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject } from './context/project/project.js'; +import { initKtxProject } from '../src/context/project/project.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { @@ -15,7 +15,7 @@ import { sanitizeMemoryFlowTuiError, startLiveMemoryFlowTui, warnVizFallbackOnce, -} from './index.js'; +} from '../src/index.js'; const require = createRequire(import.meta.url); @@ -132,9 +132,12 @@ describe('runKtxCli', () => { } expect(testIo.stdout()).not.toMatch(/^ dev\s/m); expect(testIo.stdout()).not.toMatch(/^ scan\s/m); - for (const removed of ['demo', 'init', 'connect', 'ask', 'knowledge', 'agent', 'completion', 'serve']) { + for (const removed of ['demo', 'init', 'connect', 'ask', 'knowledge', 'agent', 'serve']) { expect(testIo.stdout()).not.toMatch(new RegExp(`^\\s+${removed}(?:\\s|\\[|$)`, 'm')); } + // `completion` is a public command; the internal `__complete` helper is hidden. + expect(testIo.stdout()).toMatch(/^\s+completion /m); + expect(testIo.stdout()).not.toContain('__complete'); expect(testIo.stdout()).toContain('--project-dir '); expect(testIo.stdout()).toContain('KTX_PROJECT_DIR'); expect(testIo.stdout()).toContain('--debug'); @@ -414,12 +417,17 @@ describe('runKtxCli', () => { const promptIo = makeIo(); await expect( - runKtxCli(['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count'], promptIo.io, { sl }), + runKtxCli( + ['--project-dir', tempDir, 'sl', 'query', '--connection-id', 'warehouse', '--measure', 'orders.order_count'], + promptIo.io, + { sl }, + ), ).resolves.toBe(0); expect(sl).toHaveBeenLastCalledWith( expect.objectContaining({ command: 'query', projectDir: tempDir, + connectionId: 'warehouse', cliVersion, runtimeInstallPolicy: 'prompt', query: expect.objectContaining({ measures: ['orders.order_count'], dimensions: [] }), @@ -429,9 +437,21 @@ describe('runKtxCli', () => { const autoIo = makeIo(); await expect( - runKtxCli(['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--yes'], autoIo.io, { - sl, - }), + runKtxCli( + [ + '--project-dir', + tempDir, + 'sl', + 'query', + '--connection-id', + 'warehouse', + '--measure', + 'orders.order_count', + '--yes', + ], + autoIo.io, + { sl }, + ), ).resolves.toBe(0); expect(sl).toHaveBeenLastCalledWith( expect.objectContaining({ @@ -444,7 +464,17 @@ describe('runKtxCli', () => { const noInputIo = makeIo(); await expect( runKtxCli( - ['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--no-input'], + [ + '--project-dir', + tempDir, + 'sl', + 'query', + '--connection-id', + 'warehouse', + '--measure', + 'orders.order_count', + '--no-input', + ], noInputIo.io, { sl }, ), @@ -464,7 +494,18 @@ describe('runKtxCli', () => { await expect( runKtxCli( - ['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--yes', '--no-input'], + [ + '--project-dir', + tempDir, + 'sl', + 'query', + '--connection-id', + 'warehouse', + '--measure', + 'orders.order_count', + '--yes', + '--no-input', + ], io.io, { sl }, ), @@ -702,7 +743,7 @@ describe('runKtxCli', () => { const publicIngest = vi.fn().mockResolvedValue(0); await expect( - runKtxCli(['--project-dir', tempDir, 'ingest', 'warehouse', '--fast', '--no-input'], testIo.io, { + runKtxCli(['--project-dir', tempDir, 'ingest', 'warehouse', '--no-input'], testIo.io, { publicIngest, }), ).resolves.toBe(0); @@ -715,7 +756,6 @@ describe('runKtxCli', () => { all: false, json: false, inputMode: 'disabled', - depth: 'fast', queryHistory: 'default', cliVersion, runtimeInstallPolicy: 'never', @@ -725,12 +765,12 @@ describe('runKtxCli', () => { expect(testIo.stderr()).toBe(`Project: ${tempDir}\n`); }); - it('routes public ingest --all --deep with JSON output', async () => { + it('routes public ingest --all with JSON output', async () => { const testIo = makeIo(); const publicIngest = vi.fn().mockResolvedValue(0); await expect( - runKtxCli(['--project-dir', tempDir, 'ingest', '--all', '--deep', '--json'], testIo.io, { + runKtxCli(['--project-dir', tempDir, 'ingest', '--all', '--json'], testIo.io, { publicIngest, }), ).resolves.toBe(0); @@ -742,7 +782,6 @@ describe('runKtxCli', () => { all: true, json: true, inputMode: 'auto', - depth: 'deep', queryHistory: 'default', cliVersion, runtimeInstallPolicy: 'prompt', @@ -786,20 +825,6 @@ describe('runKtxCli', () => { expect(testIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); }); - it('rejects mutually exclusive public ingest depth flags before dispatch', async () => { - const testIo = makeIo(); - const publicIngest = vi.fn().mockResolvedValue(0); - - await expect( - runKtxCli(['--project-dir', '/tmp/project', 'ingest', 'warehouse', '--fast', '--deep'], testIo.io, { - publicIngest, - }), - ).resolves.toBe(1); - - expect(publicIngest).not.toHaveBeenCalled(); - expect(testIo.stderr()).toMatch(/option '--(deep|fast)' cannot be used with option '--(fast|deep)'/); - }); - it.each(['run', 'status', 'watch', 'replay'])( 'routes former ingest subcommand name "%s" as a connection id', async (connectionId) => { @@ -890,8 +915,6 @@ describe('runKtxCli', () => { expect(testIo.stdout()).toContain('Usage: ktx ingest'); expect(testIo.stdout()).toContain('Build or inspect KTX context'); expect(testIo.stdout()).toContain('--all'); - expect(testIo.stdout()).toContain('--fast'); - expect(testIo.stdout()).toContain('--deep'); expect(testIo.stdout()).toContain('--query-history'); expect(testIo.stdout()).toContain('--no-query-history'); expect(testIo.stdout()).toContain('--query-history-window-days '); diff --git a/packages/cli/src/ingest-query-executor.test.ts b/packages/cli/test/ingest-query-executor.test.ts similarity index 89% rename from packages/cli/src/ingest-query-executor.test.ts rename to packages/cli/test/ingest-query-executor.test.ts index 14b714d9..372cd362 100644 --- a/packages/cli/src/ingest-query-executor.test.ts +++ b/packages/cli/test/ingest-query-executor.test.ts @@ -1,7 +1,7 @@ -import type { KtxLocalProject } from './context/project/project.js'; -import { createKtxConnectorCapabilities, type KtxScanConnector } from './context/scan/types.js'; +import type { KtxLocalProject } from '../src/context/project/project.js'; +import { createKtxConnectorCapabilities, type KtxScanConnector } from '../src/context/scan/types.js'; import { describe, expect, it, vi } from 'vitest'; -import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js'; +import { createKtxCliIngestQueryExecutor } from '../src/ingest-query-executor.js'; function project(): KtxLocalProject { return { @@ -31,6 +31,8 @@ function connector(overrides: Partial = {}): KtxScanConnector })), cleanup: vi.fn(async () => {}), ...overrides, + listSchemas: overrides.listSchemas ?? vi.fn(async () => []), + listTables: overrides.listTables ?? vi.fn(async () => []), }; } diff --git a/packages/cli/src/ingest-report-file.test.ts b/packages/cli/test/ingest-report-file.test.ts similarity index 96% rename from packages/cli/src/ingest-report-file.test.ts rename to packages/cli/test/ingest-report-file.test.ts index 5183764b..071876cf 100644 --- a/packages/cli/src/ingest-report-file.test.ts +++ b/packages/cli/test/ingest-report-file.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { readIngestReportSnapshotFile } from './ingest-report-file.js'; +import { readIngestReportSnapshotFile } from '../src/ingest-report-file.js'; function reportSnapshot() { return { diff --git a/packages/cli/src/ingest-viz.test.ts b/packages/cli/test/ingest-viz.test.ts similarity index 99% rename from packages/cli/src/ingest-viz.test.ts rename to packages/cli/test/ingest-viz.test.ts index 17b35f75..a794043f 100644 --- a/packages/cli/src/ingest-viz.test.ts +++ b/packages/cli/test/ingest-viz.test.ts @@ -1,10 +1,10 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import type { LocalIngestResult, RunLocalIngestOptions } from './context/ingest/local-ingest.js'; -import type { MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; +import type { LocalIngestResult, RunLocalIngestOptions } from '../src/context/ingest/local-ingest.js'; +import type { MemoryFlowReplayInput } from '../src/context/ingest/memory-flow/types.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { runKtxIngest } from './ingest.js'; +import { runKtxIngest } from '../src/ingest.js'; import { completedLocalBundleRun, emitLiveLocalMemoryFlow, @@ -14,7 +14,7 @@ import { writeBundleReportFile, writeWarehouseConfig, } from './ingest.test-utils.js'; -import { resetVizFallbackWarningsForTest } from './viz-fallback.js'; +import { resetVizFallbackWarningsForTest } from '../src/viz-fallback.js'; describe('runKtxIngest viz and replay', () => { let tempDir: string; diff --git a/packages/cli/src/ingest.test-utils.ts b/packages/cli/test/ingest.test-utils.ts similarity index 94% rename from packages/cli/src/ingest.test-utils.ts rename to packages/cli/test/ingest.test-utils.ts index 9b3f16fa..7198ff5d 100644 --- a/packages/cli/src/ingest.test-utils.ts +++ b/packages/cli/test/ingest.test-utils.ts @@ -1,21 +1,21 @@ import { EventEmitter } from 'node:events'; import { mkdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; -import type { AgentRunnerPort, RunLoopParams } from './context/llm/runtime-port.js'; -import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './context/ingest/adapters/metabase/local-source-state-store.js'; -import { MetabaseSourceAdapter } from './context/ingest/adapters/metabase/metabase.adapter.js'; -import { getLocalIngestStatus, type LocalIngestResult, type RunLocalIngestOptions } from './context/ingest/local-ingest.js'; -import type { ChunkResult, FetchContext, SourceAdapter } from './context/ingest/types.js'; -import type { IngestReportSnapshot } from './context/ingest/reports.js'; -import type { LookerMappingClient, LookerTableIdentifierParser } from './context/ingest/adapters/looker/mapping.js'; -import type { LookerRuntimeClient } from './context/ingest/adapters/looker/fetch.js'; -import type { MemoryFlowEventSink } from './context/ingest/memory-flow/types.js'; -import type { MetabaseCard, MetabaseCardSummary, MetabaseClientFactory, MetabaseRuntimeClient } from './context/ingest/adapters/metabase/client-port.js'; -import type { SqliteBundleIngestStore } from './context/ingest/sqlite-bundle-ingest-store.js'; -import { ktxLocalStateDbPath } from './context/project/local-state-db.js'; -import { loadKtxProject } from './context/project/project.js'; +import type { AgentRunnerPort, RunLoopParams } from '../src/context/llm/runtime-port.js'; +import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from '../src/context/ingest/adapters/metabase/local-source-state-store.js'; +import { MetabaseSourceAdapter } from '../src/context/ingest/adapters/metabase/metabase.adapter.js'; +import { getLocalIngestStatus, type LocalIngestResult, type RunLocalIngestOptions } from '../src/context/ingest/local-ingest.js'; +import type { ChunkResult, FetchContext, SourceAdapter } from '../src/context/ingest/types.js'; +import type { IngestReportSnapshot } from '../src/context/ingest/reports.js'; +import type { LookerMappingClient, LookerTableIdentifierParser } from '../src/context/ingest/adapters/looker/mapping.js'; +import type { LookerRuntimeClient } from '../src/context/ingest/adapters/looker/fetch.js'; +import type { MemoryFlowEventSink } from '../src/context/ingest/memory-flow/types.js'; +import type { MetabaseCard, MetabaseCardSummary, MetabaseClientFactory, MetabaseRuntimeClient } from '../src/context/ingest/adapters/metabase/client-port.js'; +import type { SqliteBundleIngestStore } from '../src/context/ingest/sqlite-bundle-ingest-store.js'; +import { ktxLocalStateDbPath } from '../src/context/project/local-state-db.js'; +import { loadKtxProject } from '../src/context/project/project.js'; import { expect, vi } from 'vitest'; -import { runKtxIngest } from './ingest.js'; +import { runKtxIngest } from '../src/ingest.js'; export function makeIo( options: { @@ -533,7 +533,7 @@ export async function runPublicMetabaseSyncModeCase(tempDir: string, input: Sync ).resolves.toBe(0); expect(io.stderr()).toContain('Metabase ingest: prod-metabase'); - expect(io.stdout()).toContain('Metabase fan-out: all_succeeded'); + expect(io.stdout()).toContain('Metabase fanout: all_succeeded'); expect(io.stdout()).toContain(`target=warehouse_a database=1 status=done job=${jobId}`); const report = await getLocalIngestStatus(project, jobId); @@ -675,7 +675,7 @@ export function localFakeBundleReport( } export async function localBundleStore(projectDir: string, ids: [string, string]): Promise { - const { SqliteBundleIngestStore } = await import('./context/ingest/sqlite-bundle-ingest-store.js');; + const { SqliteBundleIngestStore } = await import('../src/context/ingest/sqlite-bundle-ingest-store.js');; const project = await loadKtxProject({ projectDir }); return new SqliteBundleIngestStore({ dbPath: ktxLocalStateDbPath(project), diff --git a/packages/cli/src/ingest.test.ts b/packages/cli/test/ingest.test.ts similarity index 90% rename from packages/cli/src/ingest.test.ts rename to packages/cli/test/ingest.test.ts index 15c71e00..4fc47d0c 100644 --- a/packages/cli/src/ingest.test.ts +++ b/packages/cli/test/ingest.test.ts @@ -1,15 +1,15 @@ import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { LocalLookerRuntimeStore } from './context/ingest/adapters/looker/local-runtime-store.js'; -import { LocalMetabaseDiscoveryCache } from './context/ingest/adapters/metabase/local-source-state-store.js'; -import type { LocalIngestResult, LocalMetabaseFanoutProgress, RunLocalIngestOptions } from './context/ingest/local-ingest.js'; -import type { SourceAdapter } from './context/ingest/types.js'; -import { initKtxProject, loadKtxProject } from './context/project/project.js'; -import { ktxLocalStateDbPath } from './context/project/local-state-db.js'; +import { LocalLookerRuntimeStore } from '../src/context/ingest/adapters/looker/local-runtime-store.js'; +import { LocalMetabaseDiscoveryCache } from '../src/context/ingest/adapters/metabase/local-source-state-store.js'; +import type { LocalIngestResult, LocalMetabaseFanoutProgress, RunLocalIngestOptions } from '../src/context/ingest/local-ingest.js'; +import type { SourceAdapter } from '../src/context/ingest/types.js'; +import { initKtxProject, loadKtxProject } from '../src/context/project/project.js'; +import { ktxLocalStateDbPath } from '../src/context/project/local-state-db.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { type KtxIngestArgs, type KtxIngestDeps, runKtxIngest } from './ingest.js'; -import type { KtxCliLocalIngestAdaptersOptions } from './local-adapters.js'; +import { type KtxIngestArgs, type KtxIngestDeps, runKtxIngest } from '../src/ingest.js'; +import type { KtxCliLocalIngestAdaptersOptions } from '../src/local-adapters.js'; import { CliLookerSlWritingAgentRunner, CliMetabaseAgentRunner, @@ -25,8 +25,8 @@ import { writeMetabaseConfig, writeWarehouseConfig, } from './ingest.test-utils.js'; -import { resetVizFallbackWarningsForTest } from './viz-fallback.js'; -import { runKtxSetup } from './setup.js'; +import { resetVizFallbackWarningsForTest } from '../src/viz-fallback.js'; +import { runKtxSetup } from '../src/setup.js'; describe('runKtxIngest', () => { let tempDir: string; @@ -284,7 +284,30 @@ describe('runKtxIngest', () => { return 0; }, scanConnection: async () => 0, - historicSqlProbe: async () => ({ ok: true, lines: ['PASS Historic SQL probe skipped in test'] }), + historicSqlReadinessProbe: async () => ({ + ok: true, + dialect: 'postgres', + runner: { + dialect: 'postgres', + catalogName: 'pg_stat_statements', + async run() { + return { warnings: [], info: [] }; + }, + formatSuccessDetail() { + return { + detail: 'pg_stat_statements ready (PostgreSQL 16.4)', + warnings: [], + }; + }, + fixAdvice() { + return { + failHeadline: 'pg_stat_statements unavailable', + remediation: 'Fix query-history grants.', + }; + }, + }, + result: { pgServerVersion: 'PostgreSQL 16.4', warnings: [], info: [] }, + }), }, context: async () => ({ status: 'skipped', projectDir }), runtime: async () => runtimeReady(projectDir), @@ -314,16 +337,19 @@ describe('runKtxIngest', () => { expect(runIo.stdout()).toBe(''); expect(runIo.stderr()).toContain( - 'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, or claude-code, or an injected agentRunner.', + 'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, claude-code, or codex, or an injected agentRunner.', ); - expect(runIo.stderr()).toContain('Configure a local Claude Code session or API-backed LLM, then rerun ingest:'); + expect(runIo.stderr()).toContain('Configure a local Claude Code/Codex session or API-backed LLM, then rerun ingest:'); expect(runIo.stderr()).toContain(`ktx setup --project-dir ${projectDir} --llm-backend claude-code --no-input`); + expect(runIo.stderr()).toContain( + `ktx setup --project-dir ${projectDir} --llm-backend codex --llm-model gpt-5.5 --no-input`, + ); expect(runIo.stderr()).toContain( `ktx setup --project-dir ${projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --llm-model claude-sonnet-4-6 --no-input`, ); }); - it('routes metabase scheduled pulls to the fan-out runner and prints child summaries', async () => { + it('routes metabase scheduled pulls to the fanout runner and prints child summaries', async () => { const projectDir = join(tempDir, 'project'); await writeMetabaseConfig(projectDir); const io = makeIo(); @@ -374,13 +400,13 @@ describe('runKtxIngest', () => { ), ).resolves.toBe(0); - expect(io.stdout()).toContain('Metabase fan-out: all_succeeded'); + expect(io.stdout()).toContain('Metabase fanout: all_succeeded'); expect(io.stdout()).toContain('warehouse_a'); expect(io.stdout()).toContain('metabase-child-1'); expect(io.stderr()).toContain('Metabase ingest: prod-metabase'); }); - it('returns a non-zero code when Metabase fan-out has failed children', async () => { + it('returns a non-zero code when a Metabase fanout child fully fails', async () => { const projectDir = join(tempDir, 'project'); await writeMetabaseConfig(projectDir); const io = makeIo(); @@ -418,7 +444,7 @@ describe('runKtxIngest', () => { { runLocalMetabaseIngest: async () => ({ metabaseConnectionId: 'prod-metabase', - status: 'partial_failure', + status: 'all_failed', totals: { workUnits: 1, failedWorkUnits: 1 }, children: [ { @@ -444,13 +470,87 @@ describe('runKtxIngest', () => { ), ).resolves.toBe(1); - expect(io.stdout()).toContain('Metabase fan-out: partial_failure'); - expect(io.stdout()).toContain('Failed tasks: 1'); + expect(io.stdout()).toContain('Metabase fanout: all_failed'); expect(io.stdout()).toContain('status=error'); + }); + + it('exits 0 and reports status=partial when a Metabase child saved memory despite a failure', async () => { + const projectDir = join(tempDir, 'project'); + await writeMetabaseConfig(projectDir); + const io = makeIo(); + const report = localFakeBundleReport('metabase-child-1', { + id: 'report-metabase-child-1', + runId: 'run-a', + jobId: 'metabase-child-1', + connectionId: 'warehouse_a', + sourceKey: 'metabase', + body: { + failedWorkUnits: ['metabase-db-2'], + workUnits: [ + { + unitKey: 'metabase-db-1', + rawFiles: ['cards/1.json'], + status: 'success', + actions: [{ target: 'sl', type: 'updated', key: 'warehouse.orders', detail: 'measure' }], + touchedSlSources: [], + }, + { + unitKey: 'metabase-db-2', + rawFiles: ['cards/2.json'], + status: 'failed', + reason: 'bad SQL', + actions: [], + touchedSlSources: [], + }, + ], + }, + }); + + await expect( + runKtxIngest( + { + command: 'run', + projectDir, + connectionId: 'prod-metabase', + adapter: 'metabase', + outputMode: 'plain', + }, + io.io, + { + runLocalMetabaseIngest: async () => ({ + metabaseConnectionId: 'prod-metabase', + status: 'partial_failure', + totals: { workUnits: 2, failedWorkUnits: 1 }, + children: [ + { + jobId: 'metabase-child-1', + metabaseConnectionId: 'prod-metabase', + metabaseDatabaseId: 1, + targetConnectionId: 'warehouse_a', + result: { + jobId: 'metabase-child-1', + runId: 'run-a', + syncId: 'sync-a', + diffSummary: { added: 1, modified: 0, deleted: 0, unchanged: 0 }, + workUnitCount: 2, + failedWorkUnits: ['metabase-db-2'], + artifactsWritten: 1, + commitSha: 'abc', + }, + report, + }, + ], + }), + }, + ), + ).resolves.toBe(0); + + expect(io.stdout()).toContain('Metabase fanout: partial_failure'); + expect(io.stdout()).toContain('status=partial'); expect(io.stderr()).toContain('Metabase ingest: prod-metabase'); }); - it('prints Metabase fan-out progress before the final summary', async () => { + it('prints Metabase fanout progress before the final summary', async () => { const projectDir = join(tempDir, 'project'); await writeMetabaseConfig(projectDir); const io = makeIo(); @@ -525,11 +625,11 @@ describe('runKtxIngest', () => { expect(io.stderr()).toContain('Targets: 1 mapped database'); expect(io.stderr()).toContain('- database=1 target=warehouse_a status=running job=metabase-child-1'); expect(io.stderr()).toContain('- database=1 target=warehouse_a status=done job=metabase-child-1'); - expect(io.stdout()).toContain('Metabase fan-out: all_succeeded'); + expect(io.stdout()).toContain('Metabase fanout: all_succeeded'); expect(io.stdout()).not.toContain('status=running job=metabase-child-1'); }); - it('writes metabase fan-out progress to stderr and final result to stdout', async () => { + it('writes metabase fanout progress to stderr and final result to stdout', async () => { const projectDir = join(tempDir, 'project'); await writeMetabaseConfig(projectDir); const io = makeIo({ isTTY: true }); @@ -569,11 +669,11 @@ describe('runKtxIngest', () => { expect(io.stderr()).toContain('Metabase ingest: prod-metabase'); expect(io.stderr()).toContain('status=running job=metabase-child-1'); - expect(io.stdout()).toContain('Metabase fan-out: all_succeeded'); + expect(io.stdout()).toContain('Metabase fanout: all_succeeded'); expect(io.stdout()).not.toContain('status=running job=metabase-child-1'); }); - it('emits structured progress for Metabase fan-out without writing progress to JSON output', async () => { + it('emits structured progress for Metabase fanout without writing progress to JSON output', async () => { const projectDir = join(tempDir, 'project'); await writeMetabaseConfig(projectDir); const io = makeIo(); @@ -632,7 +732,7 @@ describe('runKtxIngest', () => { expect(io.stderr()).not.toContain('Metabase ingest: prod-metabase'); }); - it('emits structured child ingest progress during Metabase fan-out', async () => { + it('emits structured child ingest progress during Metabase fanout', async () => { const projectDir = join(tempDir, 'project'); await writeMetabaseConfig(projectDir); const io = makeIo(); @@ -743,7 +843,7 @@ describe('runKtxIngest', () => { expect(io.stderr()).not.toContain('Metabase ingest: prod-metabase'); }); - it('runs Metabase scheduled ingest through the public CLI command path with real fan-out', async () => { + it('runs Metabase scheduled ingest through the public CLI command path with real fanout', async () => { const projectDir = join(tempDir, 'metabase-cli-project'); await writeWarehouseConfig(projectDir); await writeFile( @@ -815,7 +915,7 @@ describe('runKtxIngest', () => { expect(io.stderr()).toContain('Metabase ingest: prod-metabase'); expect(io.stderr()).toContain('Targets: 2 mapped databases'); - expect(io.stdout()).toContain('Metabase fan-out: all_succeeded'); + expect(io.stdout()).toContain('Metabase fanout: all_succeeded'); expect(io.stdout()).toContain('Source: prod-metabase'); expect(io.stdout()).toContain('Children: 2'); expect(io.stdout()).toContain('target=warehouse_a database=1 status=done job=metabase-child-1'); @@ -870,7 +970,7 @@ describe('runKtxIngest', () => { }); }); - it('prints metabase fan-out JSON results', async () => { + it('prints metabase fanout JSON results', async () => { const projectDir = join(tempDir, 'project'); await writeMetabaseConfig(projectDir); const io = makeIo(); @@ -944,7 +1044,7 @@ describe('runKtxIngest', () => { expect(io.stderr()).toBe(''); }); - it('rejects source-dir uploads through the metabase fan-out route', async () => { + it('rejects source-dir uploads through the metabase fanout route', async () => { const projectDir = join(tempDir, 'project'); await writeMetabaseConfig(projectDir); const io = makeIo(); @@ -962,13 +1062,13 @@ describe('runKtxIngest', () => { io.io, { runLocalMetabaseIngest: async () => { - throw new Error('fan-out should not be called'); + throw new Error('fanout should not be called'); }, }, ), ).resolves.toBe(1); - expect(io.stderr()).toContain('source-dir uploads are not supported for the Metabase fan-out adapter'); + expect(io.stderr()).toContain('source-dir uploads are not supported for the Metabase fanout adapter'); expect(io.stderr()).not.toContain('ktx ingest requires llm.provider.backend'); expect(io.stdout()).toBe(''); }); @@ -1117,6 +1217,63 @@ describe('runKtxIngest', () => { expect(io.stdout()).toContain('Status: error\n'); }); + it('exits 0 and reports Status: partial when a single-source ingest saved memory despite a failure', async () => { + const projectDir = join(tempDir, 'project'); + await writeWarehouseConfig(projectDir); + const sourceDir = join(tempDir, 'source'); + await mkdir(join(sourceDir, 'orders'), { recursive: true }); + await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8'); + + const partialReport = localFakeBundleReport('local-job-partial', { + connectionId: 'warehouse', + sourceKey: 'fake', + body: { + failedWorkUnits: ['orders-bad'], + workUnits: [ + { + unitKey: 'orders-ok', + rawFiles: ['orders/orders.json'], + status: 'success', + actions: [{ target: 'wiki', type: 'created', key: 'wiki/orders.md', detail: 'orders' }], + touchedSlSources: [], + }, + { + unitKey: 'orders-bad', + rawFiles: ['orders/bad.json'], + status: 'failed', + reason: 'writer tool failed', + actions: [], + touchedSlSources: [], + }, + ], + }, + }); + const runLocal = vi.fn(async (_input: RunLocalIngestOptions) => ({ + result: { + jobId: 'local-job-partial', + runId: partialReport.runId, + syncId: partialReport.body.syncId, + diffSummary: partialReport.body.diffSummary, + workUnitCount: partialReport.body.workUnits.length, + failedWorkUnits: partialReport.body.failedWorkUnits, + artifactsWritten: 1, + commitSha: partialReport.body.commitSha, + }, + report: partialReport, + })); + + const io = makeIo(); + await expect( + runKtxIngest( + { command: 'run', projectDir, connectionId: 'warehouse', adapter: 'fake', sourceDir, outputMode: 'plain' }, + io.io, + { runLocalIngest: runLocal, jobIdFactory: () => 'local-job-partial' }, + ), + ).resolves.toBe(0); + + expect(io.stdout()).toContain('Status: partial\n'); + }); + it('prints trace path and error status for stored failed ingest reports', async () => { const projectDir = join(tempDir, 'project'); await writeWarehouseConfig(projectDir); diff --git a/packages/cli/src/io/logger.test.ts b/packages/cli/test/io/logger.test.ts similarity index 97% rename from packages/cli/src/io/logger.test.ts rename to packages/cli/test/io/logger.test.ts index bf21a150..46248d02 100644 --- a/packages/cli/src/io/logger.test.ts +++ b/packages/cli/test/io/logger.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { createCliOperationalLogger, createNoopOperationalLogger } from './logger.js'; +import { createCliOperationalLogger, createNoopOperationalLogger } from '../../src/io/logger.js'; function makeIo() { let stdout = ''; diff --git a/packages/cli/src/io/mode.test.ts b/packages/cli/test/io/mode.test.ts similarity index 95% rename from packages/cli/src/io/mode.test.ts rename to packages/cli/test/io/mode.test.ts index cfc9a9fc..4d36c37a 100644 --- a/packages/cli/src/io/mode.test.ts +++ b/packages/cli/test/io/mode.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import type { KtxCliIo } from '../cli-runtime.js'; -import { resolveOutputMode } from './mode.js'; +import type { KtxCliIo } from '../../src/cli-runtime.js'; +import { resolveOutputMode } from '../../src/io/mode.js'; function ioWith(isTTY: boolean | undefined): KtxCliIo { return { diff --git a/packages/cli/src/io/print-list.test.ts b/packages/cli/test/io/print-list.test.ts similarity index 98% rename from packages/cli/src/io/print-list.test.ts rename to packages/cli/test/io/print-list.test.ts index f084e519..f065d067 100644 --- a/packages/cli/src/io/print-list.test.ts +++ b/packages/cli/test/io/print-list.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; -import type { KtxCliIo } from '../cli-runtime.js'; -import { createRankBadgeFormatter, printList, type PrintListColumn } from './print-list.js'; -import { SYMBOLS } from './symbols.js'; +import type { KtxCliIo } from '../../src/cli-runtime.js'; +import { createRankBadgeFormatter, printList, type PrintListColumn } from '../../src/io/print-list.js'; +import { SYMBOLS } from '../../src/io/symbols.js'; function recorder(): { io: KtxCliIo; out: () => string; err: () => string } { let stdout = ''; diff --git a/packages/cli/src/knowledge.test.ts b/packages/cli/test/knowledge.test.ts similarity index 83% rename from packages/cli/src/knowledge.test.ts rename to packages/cli/test/knowledge.test.ts index 69581f0f..94e4bb63 100644 --- a/packages/cli/src/knowledge.test.ts +++ b/packages/cli/test/knowledge.test.ts @@ -2,11 +2,11 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { stripVTControlCharacters } from 'node:util'; -import { initKtxProject, loadKtxProject } from './context/project/project.js'; -import type { KtxEmbeddingPort } from './context/core/embedding.js'; -import { writeLocalKnowledgePage } from './context/wiki/local-knowledge.js'; +import { initKtxProject, loadKtxProject } from '../src/context/project/project.js'; +import type { KtxEmbeddingPort } from '../src/context/core/embedding.js'; +import { writeLocalKnowledgePage } from '../src/context/wiki/local-knowledge.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { runKtxKnowledge } from './knowledge.js'; +import { runKtxKnowledge } from '../src/knowledge.js'; function makeIo(options: { isTTY?: boolean } = {}) { let stdout = ''; @@ -98,6 +98,46 @@ describe('runKtxKnowledge', () => { expect(searchIo.stdout()).toContain('metrics-revenue'); }); + it('reads a wiki page as raw markdown with frontmatter', async () => { + const projectDir = join(tempDir, 'read-project'); + await initKtxProject({ projectDir }); + await seedWikiPage(projectDir, { + key: 'metrics-revenue', + summary: 'Revenue', + content: 'Revenue is paid order value.', + tags: ['finance'], + slRefs: ['orders'], + }); + + const readIo = makeIo(); + await expect( + runKtxKnowledge({ command: 'read', projectDir, key: 'metrics-revenue', userId: 'local' }, readIo.io), + ).resolves.toBe(0); + + expect(readIo.stdout()).toContain('---\n'); + expect(readIo.stdout()).toContain('summary: Revenue'); + expect(readIo.stdout()).toContain('tags:'); + expect(readIo.stdout()).toContain('- finance'); + expect(readIo.stdout()).toContain('sl_refs:'); + expect(readIo.stdout()).toContain('- orders'); + expect(readIo.stdout()).toContain('usage_mode: auto'); + expect(readIo.stdout()).toContain('Revenue is paid order value.'); + expect(readIo.stderr()).toBe(''); + }); + + it('reports a clear error when a wiki page key is missing', async () => { + const projectDir = join(tempDir, 'missing-read-project'); + await initKtxProject({ projectDir }); + + const readIo = makeIo(); + await expect( + runKtxKnowledge({ command: 'read', projectDir, key: 'missing-page', userId: 'local' }, readIo.io), + ).resolves.toBe(1); + + expect(readIo.stdout()).toBe(''); + expect(readIo.stderr()).toBe("No wiki page found for key 'missing-page'\n"); + }); + it('emits debug telemetry for wiki search without query text', async () => { vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); vi.stubEnv('CI', ''); diff --git a/packages/cli/src/llm/embedding-health.test.ts b/packages/cli/test/llm/embedding-health.test.ts similarity index 97% rename from packages/cli/src/llm/embedding-health.test.ts rename to packages/cli/test/llm/embedding-health.test.ts index 65956311..f659b2d6 100644 --- a/packages/cli/src/llm/embedding-health.test.ts +++ b/packages/cli/test/llm/embedding-health.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { runKtxEmbeddingHealthCheck } from './embedding-health.js'; +import { runKtxEmbeddingHealthCheck } from '../../src/llm/embedding-health.js'; describe('KTX embedding health check', () => { it('runs a one-shot OpenAI embedding check through the configured provider', async () => { diff --git a/packages/cli/src/llm/embedding-provider.test.ts b/packages/cli/test/llm/embedding-provider.test.ts similarity index 96% rename from packages/cli/src/llm/embedding-provider.test.ts rename to packages/cli/test/llm/embedding-provider.test.ts index c649a948..071a17b0 100644 --- a/packages/cli/src/llm/embedding-provider.test.ts +++ b/packages/cli/test/llm/embedding-provider.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { createKtxEmbeddingProvider } from './embedding-provider.js'; -import type { KtxEmbeddingConfig } from './types.js'; +import { createKtxEmbeddingProvider } from '../../src/llm/embedding-provider.js'; +import type { KtxEmbeddingConfig } from '../../src/llm/types.js'; describe('createKtxEmbeddingProvider', () => { it('rejects deterministic embeddings', () => { diff --git a/packages/cli/src/llm/message-builder.test.ts b/packages/cli/test/llm/message-builder.test.ts similarity index 97% rename from packages/cli/src/llm/message-builder.test.ts rename to packages/cli/test/llm/message-builder.test.ts index 60f7d948..5ebbb590 100644 --- a/packages/cli/src/llm/message-builder.test.ts +++ b/packages/cli/test/llm/message-builder.test.ts @@ -1,7 +1,7 @@ import type { ModelMessage } from 'ai'; import { describe, expect, it } from 'vitest'; -import { KtxMessageBuilder, splitKtxSystemMessages } from './message-builder.js'; -import { createKtxLlmProvider } from './model-provider.js'; +import { KtxMessageBuilder, splitKtxSystemMessages } from '../../src/llm/message-builder.js'; +import { createKtxLlmProvider } from '../../src/llm/model-provider.js'; function makeBuilder(overrides: Parameters[0]['promptCaching'] = {}) { const provider = createKtxLlmProvider({ diff --git a/packages/cli/src/llm/model-health.test.ts b/packages/cli/test/llm/model-health.test.ts similarity index 97% rename from packages/cli/src/llm/model-health.test.ts rename to packages/cli/test/llm/model-health.test.ts index 8cf7a7ee..0af6f8e8 100644 --- a/packages/cli/src/llm/model-health.test.ts +++ b/packages/cli/test/llm/model-health.test.ts @@ -1,6 +1,6 @@ import { wrapLanguageModel as defaultWrapLanguageModel } from 'ai'; import { describe, expect, it, vi } from 'vitest'; -import { runKtxLlmHealthCheck } from './model-health.js'; +import { runKtxLlmHealthCheck } from '../../src/llm/model-health.js'; const anthropicModel = { modelId: 'claude-sonnet-4-6' } as never; diff --git a/packages/cli/src/llm/model-provider.test.ts b/packages/cli/test/llm/model-provider.test.ts similarity index 97% rename from packages/cli/src/llm/model-provider.test.ts rename to packages/cli/test/llm/model-provider.test.ts index 1a61d0a1..17d47c6a 100644 --- a/packages/cli/src/llm/model-provider.test.ts +++ b/packages/cli/test/llm/model-provider.test.ts @@ -1,7 +1,7 @@ import { devToolsMiddleware as defaultDevToolsMiddleware } from '@ai-sdk/devtools'; import { wrapLanguageModel as defaultWrapLanguageModel, type LanguageModel } from 'ai'; import { describe, expect, it, vi } from 'vitest'; -import { createKtxLlmProvider, type KtxLlmProviderFactoryDeps } from './model-provider.js'; +import { createKtxLlmProvider, type KtxLlmProviderFactoryDeps } from '../../src/llm/model-provider.js'; const languageModel = (modelId: string, provider = 'test'): LanguageModel => ({ modelId, provider }) as LanguageModel; const devtoolsMiddleware = (): ReturnType => ({ specificationVersion: 'v3' }); @@ -312,4 +312,13 @@ describe('createKtxLlmProvider', () => { }), ).toThrow('claude-code is not an AI SDK LanguageModel backend'); }); + + it('rejects codex as an AI SDK LanguageModel backend', () => { + expect(() => + createKtxLlmProvider({ + backend: 'codex', + modelSlots: { default: 'gpt-5.3-codex' }, + }), + ).toThrow('codex is not an AI SDK LanguageModel backend'); + }); }); diff --git a/packages/cli/src/llm/repair.test.ts b/packages/cli/test/llm/repair.test.ts similarity index 97% rename from packages/cli/src/llm/repair.test.ts rename to packages/cli/test/llm/repair.test.ts index bef53a46..743039a9 100644 --- a/packages/cli/src/llm/repair.test.ts +++ b/packages/cli/test/llm/repair.test.ts @@ -1,6 +1,6 @@ import { NoSuchToolError, type LanguageModel } from 'ai'; import { describe, expect, it, vi } from 'vitest'; -import { createKtxToolCallRepairHandler } from './repair.js'; +import { createKtxToolCallRepairHandler } from '../../src/llm/repair.js'; const repairModel = { modelId: 'claude-repair', provider: 'anthropic' } as LanguageModel; diff --git a/packages/cli/src/local-adapters.test.ts b/packages/cli/test/local-adapters.test.ts similarity index 56% rename from packages/cli/src/local-adapters.test.ts rename to packages/cli/test/local-adapters.test.ts index a4e856b4..467c9a56 100644 --- a/packages/cli/src/local-adapters.test.ts +++ b/packages/cli/test/local-adapters.test.ts @@ -1,9 +1,9 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { loadKtxProject } from './context/project/project.js'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { createKtxCliLocalIngestAdapters } from './local-adapters.js'; +import { loadKtxProject } from '../src/context/project/project.js'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { createKtxCliHistoricSqlRuntime, createKtxCliLocalIngestAdapters } from '../src/local-adapters.js'; function sqlAnalysisStub() { return { @@ -70,6 +70,147 @@ describe('CLI local ingest adapters', () => { ]); }); + it('creates reusable query-history runtime dependencies for setup', async () => { + await writeProject( + tempDir, + [ + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:WAREHOUSE_DATABASE_URL', + ' readonly: true', + ' context:', + ' queryHistory:', + ' enabled: true', + '', + ].join('\n'), + ); + const project = await loadKtxProject({ projectDir: tempDir }); + const sqlAnalysis = sqlAnalysisStub(); + + const runtime = createKtxCliHistoricSqlRuntime(project, 'warehouse', { sqlAnalysis }); + + expect(runtime).toMatchObject({ + dialect: 'postgres', + sqlAnalysis, + }); + expect(runtime?.reader).toBeDefined(); + expect(runtime?.queryClient).toBeDefined(); + }); + + it('uses managed daemon SQL analysis when query-history runtime gets managed daemon options', async () => { + await writeProject( + tempDir, + [ + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:WAREHOUSE_DATABASE_URL', + ' readonly: true', + ' context:', + ' queryHistory:', + ' enabled: true', + '', + ].join('\n'), + ); + const project = await loadKtxProject({ projectDir: tempDir }); + const testIo = { + stdout: { write: vi.fn() }, + stderr: { write: vi.fn() }, + }; + const ensureRuntime = vi.fn(async () => ({ + layout: {} as never, + manifest: {} as never, + })); + const startDaemon = vi.fn(async () => ({ + status: 'started' as const, + layout: {} as never, + state: { pid: 1234 } as never, + baseUrl: 'http://127.0.0.1:61234', + })); + const postJson = vi.fn(async () => ({ + results: { + probe: { + tables_touched: [], + columns_by_clause: {}, + error: null, + }, + }, + })); + + const runtime = createKtxCliHistoricSqlRuntime(project, 'warehouse', { + managedDaemon: { + cliVersion: '0.2.0', + projectDir: tempDir, + installPolicy: 'auto', + io: testIo, + ensureRuntime, + startDaemon, + postJson, + }, + }); + + await expect(runtime?.sqlAnalysis.analyzeBatch([{ id: 'probe', sql: 'select 1' }], 'postgres')).resolves.toEqual( + new Map([ + [ + 'probe', + { + tablesTouched: [], + columnsByClause: {}, + error: null, + }, + ], + ]), + ); + expect(ensureRuntime).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + installPolicy: 'auto', + io: testIo, + feature: 'core', + }); + expect(startDaemon).toHaveBeenCalledWith({ + cliVersion: '0.2.0', + projectDir: tempDir, + features: ['core'], + force: false, + }); + expect(postJson).toHaveBeenCalledWith('http://127.0.0.1:61234', '/sql/analyze-batch', { + dialect: 'postgres', + items: [{ id: 'probe', sql: 'select 1' }], + }); + }); + + it('registers historic SQL when explicitly requested even if connection query history is disabled', async () => { + await writeProject( + tempDir, + [ + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:WAREHOUSE_DATABASE_URL', + ' readonly: true', + ' context:', + ' queryHistory:', + ' enabled: false', + 'ingest:', + ' adapters:', + ' - historic-sql', + '', + ].join('\n'), + ); + const project = await loadKtxProject({ projectDir: tempDir }); + + // `--query-history` sets historicSqlConnectionId for the run; that explicit + // request is the opt-in, so the persisted context.queryHistory.enabled flag + // must not gate adapter registration. + const adapters = createKtxCliLocalIngestAdapters(project, { + historicSqlConnectionId: 'warehouse', + sqlAnalysis: sqlAnalysisStub(), + }); + + expect(adapters.some((adapter) => adapter.source === 'historic-sql')).toBe(true); + }); + it('registers BigQuery historic SQL from the requested connection', async () => { await writeProject( tempDir, diff --git a/packages/cli/src/local-scan-connectors.test.ts b/packages/cli/test/local-scan-connectors.test.ts similarity index 94% rename from packages/cli/src/local-scan-connectors.test.ts rename to packages/cli/test/local-scan-connectors.test.ts index a993faa2..1dadb6c4 100644 --- a/packages/cli/src/local-scan-connectors.test.ts +++ b/packages/cli/test/local-scan-connectors.test.ts @@ -1,9 +1,9 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject, loadKtxProject } from './context/project/project.js'; +import { initKtxProject, loadKtxProject } from '../src/context/project/project.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { createKtxCliScanConnector } from './local-scan-connectors.js'; +import { createKtxCliScanConnector } from '../src/local-scan-connectors.js'; const bigQueryMock = vi.hoisted(() => ({ constructorInputs: [] as Array<{ @@ -12,7 +12,7 @@ const bigQueryMock = vi.hoisted(() => ({ }>, })); -vi.mock('./connectors/bigquery/connector.js', () => ({ +vi.mock('../src/connectors/bigquery/connector.js', () => ({ isKtxBigQueryConnectionConfig: (connection: { driver?: unknown } | undefined) => String(connection?.driver ?? '').toLowerCase() === 'bigquery', KtxBigQueryScanConnector: class { diff --git a/packages/cli/src/managed-local-embeddings.test.ts b/packages/cli/test/managed-local-embeddings.test.ts similarity index 96% rename from packages/cli/src/managed-local-embeddings.test.ts rename to packages/cli/test/managed-local-embeddings.test.ts index 9ee938fb..9c78e177 100644 --- a/packages/cli/src/managed-local-embeddings.test.ts +++ b/packages/cli/test/managed-local-embeddings.test.ts @@ -3,10 +3,10 @@ import { ensureManagedLocalEmbeddingsDaemon, managedLocalEmbeddingHealthConfig, tryUseManagedLocalEmbeddingsDaemon, -} from './managed-local-embeddings.js'; -import type { ManagedPythonCommandRuntime } from './managed-python-command.js'; -import type { ManagedPythonDaemonStartResult } from './managed-python-daemon.js'; -import type { ManagedPythonDaemonLayout } from './managed-python-runtime.js'; +} from '../src/managed-local-embeddings.js'; +import type { ManagedPythonCommandRuntime } from '../src/managed-python-command.js'; +import type { ManagedPythonDaemonStartResult } from '../src/managed-python-daemon.js'; +import type { ManagedPythonDaemonLayout } from '../src/managed-python-runtime.js'; function makeIo() { let stdout = ''; diff --git a/packages/cli/src/managed-mcp-daemon.test.ts b/packages/cli/test/managed-mcp-daemon.test.ts similarity index 99% rename from packages/cli/src/managed-mcp-daemon.test.ts rename to packages/cli/test/managed-mcp-daemon.test.ts index d72bb6a4..81566d40 100644 --- a/packages/cli/src/managed-mcp-daemon.test.ts +++ b/packages/cli/test/managed-mcp-daemon.test.ts @@ -9,7 +9,7 @@ import { stopKtxMcpDaemon, type KtxMcpDaemonChild, type KtxMcpDaemonState, -} from './managed-mcp-daemon.js'; +} from '../src/managed-mcp-daemon.js'; type KtxMcpDaemonStartOptions = Parameters[0]; diff --git a/packages/cli/src/managed-python-command.test.ts b/packages/cli/test/managed-python-command.test.ts similarity index 99% rename from packages/cli/src/managed-python-command.test.ts rename to packages/cli/test/managed-python-command.test.ts index 717accf4..f08589f1 100644 --- a/packages/cli/src/managed-python-command.test.ts +++ b/packages/cli/test/managed-python-command.test.ts @@ -4,14 +4,14 @@ import { ensureManagedPythonCommandRuntime, managedRuntimeInstallCommand, runtimeInstallPolicyFromFlags, -} from './managed-python-command.js'; +} from '../src/managed-python-command.js'; import type { InstalledKtxRuntimeManifest, KtxRuntimeFeature, ManagedPythonRuntimeInstallResult, ManagedPythonRuntimeLayout, ManagedPythonRuntimeStatus, -} from './managed-python-runtime.js'; +} from '../src/managed-python-runtime.js'; function makeIo() { let stdout = ''; diff --git a/packages/cli/src/managed-python-daemon.test.ts b/packages/cli/test/managed-python-daemon.test.ts similarity index 83% rename from packages/cli/src/managed-python-daemon.test.ts rename to packages/cli/test/managed-python-daemon.test.ts index d56a27b1..4684c731 100644 --- a/packages/cli/src/managed-python-daemon.test.ts +++ b/packages/cli/test/managed-python-daemon.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { + ManagedPythonDaemonStartError, readManagedPythonDaemonStatus, startManagedPythonDaemon, stopAllManagedPythonDaemons, @@ -12,13 +13,13 @@ import { type ManagedPythonDaemonProcessInfo, type ManagedPythonDaemonSpawn, type ManagedPythonDaemonState, -} from './managed-python-daemon.js'; +} from '../src/managed-python-daemon.js'; import type { InstalledKtxRuntimeManifest, ManagedPythonDaemonLayout, ManagedPythonRuntimeInstallResult, ManagedPythonRuntimeLayout, -} from './managed-python-runtime.js'; +} from '../src/managed-python-runtime.js'; function layout(root: string): ManagedPythonDaemonLayout { const projectDir = join(root, 'project'); @@ -244,6 +245,76 @@ describe('KTX daemon lifecycle', () => { }); }); + it('kills the spawned daemon when the startup health check times out', async () => { + const spawnDaemon = makeSpawn(7777); + const killProcess = vi.fn(); + const fetch = vi.fn().mockRejectedValue(new Error('fetch failed')); + + await expect( + startManagedPythonDaemon({ + ...daemonOptionsBase(tempDir), + features: ['core'], + installRuntime: vi.fn(async () => installResult(tempDir)), + spawnDaemon, + fetch, + processAlive: vi.fn(() => true), + killProcess, + allocatePort: vi.fn(async () => 61234), + now: () => new Date('2026-05-11T00:00:00.000Z'), + startupTimeoutMs: 5, + pollIntervalMs: 1, + }), + ).rejects.toBeInstanceOf(ManagedPythonDaemonStartError); + + expect(killProcess).toHaveBeenCalledWith(7777); + await expect(readFile(layout(tempDir).daemonStatePath, 'utf8')).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('surfaces the underlying fetch cause in the startup failure message', async () => { + const cause = new Error('connect ECONNREFUSED 127.0.0.1:61234'); + const fetchError = new Error('fetch failed'); + (fetchError as Error & { cause?: unknown }).cause = cause; + + const error = await startManagedPythonDaemon({ + ...daemonOptionsBase(tempDir), + features: ['core'], + installRuntime: vi.fn(async () => installResult(tempDir)), + spawnDaemon: makeSpawn(7778), + fetch: vi.fn().mockRejectedValue(fetchError), + processAlive: vi.fn(() => false), + killProcess: vi.fn(), + allocatePort: vi.fn(async () => 61234), + now: () => new Date('2026-05-11T00:00:00.000Z'), + startupTimeoutMs: 5, + pollIntervalMs: 1, + }).catch((value: unknown) => value); + + expect(error).toBeInstanceOf(ManagedPythonDaemonStartError); + const startError = error as ManagedPythonDaemonStartError; + expect(startError.detail).toContain('fetch failed'); + expect(startError.detail).toContain('ECONNREFUSED'); + expect(startError.message).toContain('ECONNREFUSED'); + }); + + it('exposes the daemon stderr log path on startup failure', async () => { + const error = await startManagedPythonDaemon({ + ...daemonOptionsBase(tempDir), + features: ['core'], + installRuntime: vi.fn(async () => installResult(tempDir)), + spawnDaemon: makeSpawn(7779), + fetch: vi.fn().mockRejectedValue(new Error('fetch failed')), + processAlive: vi.fn(() => false), + killProcess: vi.fn(), + allocatePort: vi.fn(async () => 61234), + now: () => new Date('2026-05-11T00:00:00.000Z'), + startupTimeoutMs: 5, + pollIntervalMs: 1, + }).catch((value: unknown) => value); + + expect(error).toBeInstanceOf(ManagedPythonDaemonStartError); + expect((error as ManagedPythonDaemonStartError).stderrLog).toBe(layout(tempDir).daemonStderrPath); + }); + it('reuses a healthy daemon with the requested feature set', async () => { await mkdir(layout(tempDir).daemonStateDir, { recursive: true }); await writeFile(layout(tempDir).daemonStatePath, `${JSON.stringify(runningState(tempDir), null, 2)}\n`); diff --git a/packages/cli/src/managed-python-http.test.ts b/packages/cli/test/managed-python-http.test.ts similarity index 97% rename from packages/cli/src/managed-python-http.test.ts rename to packages/cli/test/managed-python-http.test.ts index 73cae844..74334042 100644 --- a/packages/cli/src/managed-python-http.test.ts +++ b/packages/cli/test/managed-python-http.test.ts @@ -5,7 +5,7 @@ import { createManagedDaemonSqlAnalysisPort, createManagedPythonDaemonBaseUrlResolver, managedDaemonDatabaseIntrospectionOptions, -} from './managed-python-http.js'; +} from '../src/managed-python-http.js'; function io() { let stderr = ''; @@ -161,7 +161,7 @@ describe('KTX daemon ingest ports', () => { const requestJson = vi.fn(async () => ({ results: { orders: { - tables_touched: ['public.orders'], + tables_touched: [{ catalog: null, db: 'public', name: 'orders' }], columns_by_clause: { select: ['status'] }, error: null, }, @@ -175,7 +175,7 @@ describe('KTX daemon ingest ports', () => { [ 'orders', { - tablesTouched: ['public.orders'], + tablesTouched: [{ catalog: null, db: 'public', name: 'orders' }], columnsByClause: { select: ['status'] }, error: null, }, diff --git a/packages/cli/src/managed-python-runtime.test.ts b/packages/cli/test/managed-python-runtime.test.ts similarity index 99% rename from packages/cli/src/managed-python-runtime.test.ts rename to packages/cli/test/managed-python-runtime.test.ts index 92e34e35..143802ad 100644 --- a/packages/cli/src/managed-python-runtime.test.ts +++ b/packages/cli/test/managed-python-runtime.test.ts @@ -13,7 +13,7 @@ import { readManagedPythonRuntimeStatus, verifyRuntimeAsset, type ManagedPythonRuntimeExec, -} from './managed-python-runtime.js'; +} from '../src/managed-python-runtime.js'; function runtimeWheelContents(input: { label?: string; requiresPython?: string | null } = {}): Buffer { const label = input.label ?? 'runtime-wheel'; diff --git a/packages/cli/src/mcp-http-server.test.ts b/packages/cli/test/mcp-http-server.test.ts similarity index 99% rename from packages/cli/src/mcp-http-server.test.ts rename to packages/cli/test/mcp-http-server.test.ts index d34f0c0c..ddd0bf0f 100644 --- a/packages/cli/src/mcp-http-server.test.ts +++ b/packages/cli/test/mcp-http-server.test.ts @@ -7,7 +7,7 @@ import { isMcpRequestAuthorized, normalizeHostHeader, runKtxMcpHttpServer, -} from './mcp-http-server.js'; +} from '../src/mcp-http-server.js'; describe('normalizeHostHeader', () => { it('normalizes host headers before allow-list comparison', () => { diff --git a/packages/cli/src/mcp-server-factory.test.ts b/packages/cli/test/mcp-server-factory.test.ts similarity index 85% rename from packages/cli/src/mcp-server-factory.test.ts rename to packages/cli/test/mcp-server-factory.test.ts index 64c7275d..05ac5aac 100644 --- a/packages/cli/src/mcp-server-factory.test.ts +++ b/packages/cli/test/mcp-server-factory.test.ts @@ -1,10 +1,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { createDefaultKtxMcpServer } from './context/mcp/server.js'; -import { createLocalProjectMcpContextPorts } from './context/mcp/local-project-ports.js'; -import { createLocalProjectMemoryIngest } from './context/memory/local-memory.js'; -import { resolveProjectEmbeddingProvider } from './embedding-resolution.js'; -import { createKtxCliScanConnector } from './local-scan-connectors.js'; -import { createKtxMcpServerFactory } from './mcp-server-factory.js'; +import { createDefaultKtxMcpServer } from '../src/context/mcp/server.js'; +import { createLocalProjectMcpContextPorts } from '../src/context/mcp/local-project-ports.js'; +import { createLocalProjectMemoryIngest } from '../src/context/memory/local-memory.js'; +import { resolveProjectEmbeddingProvider } from '../src/embedding-resolution.js'; +import { createKtxCliScanConnector } from '../src/local-scan-connectors.js'; +import { createKtxMcpServerFactory } from '../src/mcp-server-factory.js'; type FakeEmbeddingProvider = { maxBatchSize: number; @@ -19,7 +19,7 @@ const mocks = vi.hoisted(() => ({ memoryIngest: { ingest: vi.fn(), status: vi.fn(), waitForRun: vi.fn() }, })); -vi.mock('./context/llm/embedding-port.js', () => ({ +vi.mock('../src/context/llm/embedding-port.js', () => ({ KtxIngestEmbeddingPortAdapter: class { readonly maxBatchSize: number; @@ -37,35 +37,35 @@ vi.mock('./context/llm/embedding-port.js', () => ({ }, })); -vi.mock('./context/mcp/server.js', () => ({ +vi.mock('../src/context/mcp/server.js', () => ({ createDefaultKtxMcpServer: vi.fn(() => ({ kind: 'mcp-server' })), })); -vi.mock('./context/mcp/local-project-ports.js', () => ({ +vi.mock('../src/context/mcp/local-project-ports.js', () => ({ createLocalProjectMcpContextPorts: vi.fn(() => ({ context_tool: { name: 'context_tool' } })), })); -vi.mock('./context/memory/local-memory.js', () => ({ +vi.mock('../src/context/memory/local-memory.js', () => ({ createLocalProjectMemoryIngest: vi.fn(() => mocks.memoryIngest), })); -vi.mock('./embedding-resolution.js', () => ({ +vi.mock('../src/embedding-resolution.js', () => ({ resolveProjectEmbeddingProvider: vi.fn(), })); -vi.mock('./ingest-query-executor.js', () => ({ +vi.mock('../src/ingest-query-executor.js', () => ({ createKtxCliIngestQueryExecutor: vi.fn(() => mocks.queryExecutor), })); -vi.mock('./local-scan-connectors.js', () => ({ +vi.mock('../src/local-scan-connectors.js', () => ({ createKtxCliScanConnector: vi.fn(() => ({ source: 'fake-scan-connector' })), })); -vi.mock('./managed-python-command.js', () => ({ +vi.mock('../src/managed-python-command.js', () => ({ createManagedPythonSemanticLayerComputePort: vi.fn(async () => mocks.semanticLayerCompute), })); -vi.mock('./managed-python-http.js', () => ({ +vi.mock('../src/managed-python-http.js', () => ({ createManagedDaemonSqlAnalysisPort: vi.fn(() => mocks.sqlAnalysis), })); diff --git a/packages/cli/src/memory-flow-interactive.test.ts b/packages/cli/test/memory-flow-interactive.test.ts similarity index 97% rename from packages/cli/src/memory-flow-interactive.test.ts rename to packages/cli/test/memory-flow-interactive.test.ts index d6976a03..befc7f01 100644 --- a/packages/cli/src/memory-flow-interactive.test.ts +++ b/packages/cli/test/memory-flow-interactive.test.ts @@ -1,7 +1,7 @@ import { EventEmitter } from 'node:events'; -import type { MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; +import type { MemoryFlowReplayInput } from '../src/context/ingest/memory-flow/types.js'; import { describe, expect, it, vi } from 'vitest'; -import { memoryFlowCommandForKey, renderMemoryFlowInteractively } from './memory-flow-interactive.js'; +import { memoryFlowCommandForKey, renderMemoryFlowInteractively } from '../src/memory-flow-interactive.js'; class FakeStdin extends EventEmitter { isTTY = true; diff --git a/packages/cli/src/memory-flow-tui.test.tsx b/packages/cli/test/memory-flow-tui.test.tsx similarity index 99% rename from packages/cli/src/memory-flow-tui.test.tsx rename to packages/cli/test/memory-flow-tui.test.tsx index 09d50125..1bb38b72 100644 --- a/packages/cli/src/memory-flow-tui.test.tsx +++ b/packages/cli/test/memory-flow-tui.test.tsx @@ -1,5 +1,5 @@ /* @jsxImportSource react */ -import type { MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; +import type { MemoryFlowReplayInput } from '../src/context/ingest/memory-flow/types.js'; import { render as renderInkTest } from 'ink-testing-library'; import React, { type ReactNode } from 'react'; import { describe, expect, it, vi } from 'vitest'; @@ -11,7 +11,7 @@ import { startLiveMemoryFlowTui, type KtxMemoryFlowTuiIo, type MemoryFlowInkInstance, -} from './memory-flow-tui.js'; +} from '../src/memory-flow-tui.js'; function replayInput(): MemoryFlowReplayInput { return { diff --git a/packages/cli/src/next-steps.test.ts b/packages/cli/test/next-steps.test.ts similarity index 92% rename from packages/cli/src/next-steps.test.ts rename to packages/cli/test/next-steps.test.ts index 8a3e5e2a..eed0f3bf 100644 --- a/packages/cli/src/next-steps.test.ts +++ b/packages/cli/test/next-steps.test.ts @@ -4,7 +4,7 @@ import { KTX_NEXT_STEP_COMMANDS, formatNextStepLines, formatSetupNextStepLines, -} from './next-steps.js'; +} from '../src/next-steps.js'; describe('KTX demo next steps', () => { it('uses supported context-build commands before agent usage', () => { @@ -65,8 +65,7 @@ describe('KTX demo next steps', () => { agentIntegrationReady: true, }).join('\n'); - expect(rendered).toContain('Build KTX context next.'); - expect(rendered).toContain('Run ingest to build database schema context before context-source ingest.'); + expect(rendered).toContain('Setup is complete. The only step left is to build context for your agents.'); expect(rendered).toContain('ktx ingest'); expect(rendered).not.toContain('resume'); expect(rendered).not.toContain('scan'); @@ -87,6 +86,6 @@ describe('KTX demo next steps', () => { expect(rendered).toContain('ktx status --json'); expect(rendered).not.toContain('ktx agent'); expect(rendered).not.toContain('ktx serve --mcp stdio --user-id local'); - expect(rendered).not.toContain('Build KTX context next.'); + expect(rendered).not.toContain('Setup is complete.'); }); }); diff --git a/packages/cli/src/notion-page-picker.test.ts b/packages/cli/test/notion-page-picker.test.ts similarity index 98% rename from packages/cli/src/notion-page-picker.test.ts rename to packages/cli/test/notion-page-picker.test.ts index 29f5a352..dda695cb 100644 --- a/packages/cli/src/notion-page-picker.test.ts +++ b/packages/cli/test/notion-page-picker.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import type { PickerState } from './tree-picker-state.js'; -import type { TreePickerChrome, TreePickerResult, TreePickerTuiIo } from './tree-picker-tui.js'; +import type { PickerState } from '../src/tree-picker-state.js'; +import type { TreePickerChrome, TreePickerResult, TreePickerTuiIo } from '../src/tree-picker-tui.js'; import { discoverNotionPickerPages, notionPickerPageFromSearchResult, @@ -8,7 +8,7 @@ import { pickNotionRootPages, resolveNotionWorkspaceLabel, type NotionPickerApi, -} from './notion-page-picker.js'; +} from '../src/notion-page-picker.js'; function makeIo() { let stdout = ''; diff --git a/packages/cli/src/print-command-tree.test.ts b/packages/cli/test/print-command-tree.test.ts similarity index 72% rename from packages/cli/src/print-command-tree.test.ts rename to packages/cli/test/print-command-tree.test.ts index edd0b69a..387874b3 100644 --- a/packages/cli/src/print-command-tree.test.ts +++ b/packages/cli/test/print-command-tree.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { renderKtxCommandTree } from './print-command-tree.js'; +import { renderKtxCommandTree } from '../src/print-command-tree.js'; describe('renderKtxCommandTree', () => { it('renders an indented tree rooted at "ktx" with known top-level commands', () => { @@ -12,10 +12,14 @@ describe('renderKtxCommandTree', () => { .filter((line) => /^ {2}[├└]── \S/.test(line)) .map((line) => line.replace(/^ {2}[├└]── /, '').trim().split(' ')[0]); - for (const expected of ['setup', 'connection', 'ingest', 'sl', 'mcp', 'admin']) { + for (const expected of ['setup', 'connection', 'ingest', 'sl', 'mcp', 'admin', 'completion']) { expect(topLevel).toContain(expected); } + // The internal completion helper is hidden and must not appear in the tree. + expect(topLevel).not.toContain('__complete'); + expect(output).not.toContain('__complete'); + expect(output).toContain('│ └── test [connectionId]'); expect(output).toContain('│ ├── status Show KTX MCP daemon status'); expect(output).not.toContain('│ ├── add'); @@ -27,10 +31,14 @@ describe('renderKtxCommandTree', () => { expect(output).not.toContain('scan '); expect(output).not.toContain('│ ├── replay'); expect(output).not.toContain('│ └── replay'); - expect(output).not.toContain('│ ├── run'); + // Match `run` as a whole command name, not the `run` prefix of `runtime`. + expect(output).not.toMatch(/[├└]── run(\s|$)/m); expect(output).not.toContain('│ ├── watch'); expect(output).not.toContain('│ └── watch'); - expect(output).not.toContain('│ ├── read'); + expect(output).toContain('│ └── read Read a wiki page file by key'); + expect(output).toContain( + '│ ├── read Read a semantic-layer source YAML file', + ); expect(output).not.toContain('│ ├── write'); expect(output).not.toContain('│ └── write'); }); diff --git a/packages/cli/test/progress-port-adapter.test.ts b/packages/cli/test/progress-port-adapter.test.ts new file mode 100644 index 00000000..336883aa --- /dev/null +++ b/packages/cli/test/progress-port-adapter.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest'; +import { createAggregateProgressPort } from '../src/progress-port-adapter.js'; + +describe('createAggregateProgressPort', () => { + it('flattens nested weighted progress into absolute percent updates', async () => { + const updates: Array<{ percent: number; message: string; transient?: boolean }> = []; + const progress = createAggregateProgressPort((update) => updates.push(update)); + + await progress.update(0.1, 'Preparing scan'); + const nested = progress.startPhase(0.5); + await nested.update(0.5, 'Generating descriptions 2/4 tables', { transient: true }); + await progress.update(0.95, 'Writing schema artifacts'); + + expect(updates).toEqual([ + { percent: 10, message: 'Preparing scan' }, + { percent: 35, message: 'Generating descriptions 2/4 tables', transient: true }, + { percent: 95, message: 'Writing schema artifacts' }, + ]); + }); + + it('clamps updates and never moves the shared progress state backward', async () => { + const updates: Array<{ percent: number; message: string }> = []; + const progress = createAggregateProgressPort((update) => updates.push(update)); + + await progress.update(0.8, 'Building enriched schema context'); + await progress.update(0.2, 'Older scan callback'); + await progress.update(1.4, 'Scan completed'); + + expect(updates).toEqual([ + { percent: 80, message: 'Building enriched schema context' }, + { percent: 80, message: 'Older scan callback' }, + { percent: 100, message: 'Scan completed' }, + ]); + }); +}); diff --git a/packages/cli/src/project-dir.test.ts b/packages/cli/test/project-dir.test.ts similarity index 98% rename from packages/cli/src/project-dir.test.ts rename to packages/cli/test/project-dir.test.ts index 25a9b585..14c3ceb7 100644 --- a/packages/cli/src/project-dir.test.ts +++ b/packages/cli/test/project-dir.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { runKtxCli, type KtxCliDeps } from './index.js'; +import { runKtxCli, type KtxCliDeps } from '../src/index.js'; async function makeFixtureProject(prefix: string): Promise { const dir = await mkdtemp(join(tmpdir(), prefix)); diff --git a/packages/cli/src/project-resolver.test.ts b/packages/cli/test/project-resolver.test.ts similarity index 98% rename from packages/cli/src/project-resolver.test.ts rename to packages/cli/test/project-resolver.test.ts index 39dab27b..9680fd78 100644 --- a/packages/cli/src/project-resolver.test.ts +++ b/packages/cli/test/project-resolver.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { findNearestKtxProjectDir, resolveKtxProjectDir } from './project-resolver.js'; +import { findNearestKtxProjectDir, resolveKtxProjectDir } from '../src/project-resolver.js'; describe('resolveKtxProjectDir', () => { let tempDir: string; diff --git a/packages/cli/src/prompt-navigation.test.ts b/packages/cli/test/prompt-navigation.test.ts similarity index 97% rename from packages/cli/src/prompt-navigation.test.ts rename to packages/cli/test/prompt-navigation.test.ts index 9338b56e..e0b4cf8b 100644 --- a/packages/cli/src/prompt-navigation.test.ts +++ b/packages/cli/test/prompt-navigation.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { withMenuOptionSpacing, withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js'; +import { withMenuOptionSpacing, withMultiselectNavigation, withTextInputNavigation } from '../src/prompt-navigation.js'; describe('prompt navigation helpers', () => { it('leaves compact single-line menu prompts unchanged', () => { diff --git a/packages/cli/src/proxy-env.test.ts b/packages/cli/test/proxy-env.test.ts similarity index 92% rename from packages/cli/src/proxy-env.test.ts rename to packages/cli/test/proxy-env.test.ts index 1da7bc91..38279a04 100644 --- a/packages/cli/src/proxy-env.test.ts +++ b/packages/cli/test/proxy-env.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { sanitizeChildProxyEnv } from './proxy-env.js'; +import { sanitizeChildProxyEnv } from '../src/proxy-env.js'; describe('sanitizeChildProxyEnv', () => { it('drops IPv6 CIDR no-proxy entries and normalizes both env keys', () => { diff --git a/packages/cli/src/public-ingest-copy.test.ts b/packages/cli/test/public-ingest-copy.test.ts similarity index 98% rename from packages/cli/src/public-ingest-copy.test.ts rename to packages/cli/test/public-ingest-copy.test.ts index d13696df..539d76ce 100644 --- a/packages/cli/src/public-ingest-copy.test.ts +++ b/packages/cli/test/public-ingest-copy.test.ts @@ -3,7 +3,7 @@ import { publicDatabaseIngestMessage, publicIngestOutputLine, publicQueryHistoryMessage, -} from './public-ingest-copy.js'; +} from '../src/public-ingest-copy.js'; describe('public ingest copy sanitizers', () => { it('maps database scan progress into schema-context wording', () => { diff --git a/packages/cli/src/public-ingest.test.ts b/packages/cli/test/public-ingest.test.ts similarity index 58% rename from packages/cli/src/public-ingest.test.ts rename to packages/cli/test/public-ingest.test.ts index 7c400752..6dea8834 100644 --- a/packages/cli/src/public-ingest.test.ts +++ b/packages/cli/test/public-ingest.test.ts @@ -1,16 +1,29 @@ -import { mkdtemp, rm } from 'node:fs/promises'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; -import { initKtxProject } from './context/project/project.js'; -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from '../src/context/project/config.js'; +import { initKtxProject } from '../src/context/project/project.js'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { buildPublicIngestPlan, + executePublicIngestTarget, type KtxPublicIngestDeps, type KtxPublicIngestProject, + publicProgressMessage, runKtxPublicIngest, -} from './public-ingest.js'; -import type { ManagedPythonCommandRuntime } from './managed-python-command.js'; +} from '../src/public-ingest.js'; + +const reportExceptionMock = vi.hoisted(() => vi.fn(async () => {})); + +vi.mock('../src/telemetry/exception.js', () => ({ + reportException: reportExceptionMock, +})); + +/** Count non-overlapping occurrences of `needle` in `haystack`. */ +function occurrences(haystack: string, needle: string): number { + return haystack.split(needle).length - 1; +} +import type { ManagedPythonCommandRuntime } from '../src/managed-python-command.js'; function makeIo(options: { isTTY?: boolean; interactive?: boolean } = {}) { let stdout = ''; @@ -88,7 +101,7 @@ function deepReadyProject( describe('buildPublicIngestPlan', () => { it('plans warehouse connections as scan targets and source connections as source ingest targets', () => { - const project = projectWithConnections({ + const project = deepReadyProject({ warehouse: { driver: 'postgres' }, prod_metabase: { driver: 'metabase', api_url: 'https://metabase.example.com' }, docs: { driver: 'notion' }, @@ -103,8 +116,7 @@ describe('buildPublicIngestPlan', () => { operation: 'database-ingest', debugCommand: 'ktx ingest warehouse --debug', steps: ['database-schema'], - databaseDepth: 'fast', - detectRelationships: false, + detectRelationships: true, queryHistory: { enabled: false }, }, { @@ -139,44 +151,18 @@ describe('buildPublicIngestPlan', () => { expect(plan.targets.map((target) => target.connectionId).sort()).toEqual(['docs', 'warehouse']); }); - it('resolves database depth from flags, stored context, and defaults', () => { + it('rejects stale local Looker source driver aliases', () => { const project = projectWithConnections({ - fast_default: { driver: 'postgres' }, - deep_default: { driver: 'postgres', context: { depth: 'deep' } }, - docs: { driver: 'notion' }, + local_looker: { driver: 'local_looker' } as never, }); - expect( - buildPublicIngestPlan(project, { - projectDir: '/tmp/project', - targetConnectionId: 'fast_default', - all: false, - queryHistory: 'default', - }).targets[0], - ).toMatchObject({ connectionId: 'fast_default', databaseDepth: 'fast', queryHistory: { enabled: false } }); - - expect( - buildPublicIngestPlan(project, { - projectDir: '/tmp/project', - targetConnectionId: 'deep_default', - all: false, - queryHistory: 'default', - }).targets[0], - ).toMatchObject({ connectionId: 'deep_default', databaseDepth: 'deep' }); - - expect( - buildPublicIngestPlan(project, { - projectDir: '/tmp/project', - targetConnectionId: 'docs', - all: false, - depth: 'deep', - queryHistory: 'default', - }).warnings, - ).toEqual(['--deep affects database ingest only; ignoring it for docs.']); + expect(() => buildPublicIngestPlan(project, { projectDir: '/tmp/project', all: true })).toThrow( + 'unsupported public ingest driver "local_looker"', + ); }); - it('upgrades effective depth when query history is explicitly enabled', () => { - const project = projectWithConnections({ + it('enables query history when explicitly requested even if stored config disables it', () => { + const project = deepReadyProject({ warehouse: { driver: 'postgres', context: { queryHistory: { enabled: false } } }, }); @@ -184,17 +170,16 @@ describe('buildPublicIngestPlan', () => { projectDir: '/tmp/project', targetConnectionId: 'warehouse', all: false, - depth: 'fast', queryHistory: 'enabled', queryHistoryWindowDays: 30, }); expect(plan.targets[0]).toMatchObject({ connectionId: 'warehouse', - databaseDepth: 'deep', queryHistory: { enabled: true, windowDays: 30, dialect: 'postgres' }, + steps: ['database-schema', 'query-history'], }); - expect(plan.warnings).toEqual(['--query-history requires deep ingest; running warehouse with --deep.']); + expect(plan.warnings).toEqual([]); }); it('warns and skips query history for unsupported database drivers', () => { @@ -209,7 +194,6 @@ describe('buildPublicIngestPlan', () => { expect(plan.targets[0]).toMatchObject({ connectionId: 'local', - databaseDepth: 'fast', queryHistory: { enabled: false, unsupported: true }, }); expect(plan.warnings).toEqual(['--query-history is not supported for sqlite; running schema ingest for local.']); @@ -220,12 +204,11 @@ describe('buildPublicIngestPlan', () => { deepReadyProject({ local: { driver: 'sqlite' }, mysql_warehouse: { driver: 'mysql' }, - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, + warehouse: { driver: 'postgres' }, }), { projectDir: '/tmp/project', all: true, - depth: 'deep', queryHistory: 'enabled', }, ); @@ -297,7 +280,6 @@ describe('buildPublicIngestPlan', () => { expect(plan.targets[0]).toMatchObject({ connectionId: 'warehouse', - databaseDepth: 'deep', queryHistory: { enabled: true, dialect: 'postgres', windowDays: 30 }, steps: ['database-schema', 'query-history'], }); @@ -305,7 +287,7 @@ describe('buildPublicIngestPlan', () => { it('adds a schema-first notice when query history is explicitly enabled', () => { const project = deepReadyProject({ - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, + warehouse: { driver: 'postgres' }, }); expect( @@ -334,34 +316,15 @@ describe('buildPublicIngestPlan', () => { expect(plan.targets[0]).toMatchObject({ connectionId: 'local', - databaseDepth: 'fast', queryHistory: { enabled: false, windowDays: 30, unsupported: true }, steps: ['database-schema'], }); expect(plan.warnings).toEqual(['--query-history is not supported for sqlite; running schema ingest for local.']); }); - it('aggregates ignored database-depth warnings for all source targets', () => { - const plan = buildPublicIngestPlan( - projectWithConnections({ - warehouse: { driver: 'postgres' }, - docs: { driver: 'notion' }, - dbt: { driver: 'dbt' }, - }), - { - projectDir: '/tmp/project', - all: true, - depth: 'deep', - queryHistory: 'default', - }, - ); - - expect(plan.warnings).toEqual(['--deep ignored for 2 non-database sources.']); - }); - - it('records a preflight failure for deep database ingest when readiness config is missing', () => { + it('records a preflight failure for database ingest when enrichment readiness config is missing', () => { const project = projectWithConnections({ - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, + warehouse: { driver: 'postgres' }, }); const plan = buildPublicIngestPlan(project, { @@ -373,15 +336,14 @@ describe('buildPublicIngestPlan', () => { expect(plan.targets[0]).toMatchObject({ connectionId: 'warehouse', - databaseDepth: 'deep', preflightFailure: - 'warehouse requires deep ingest readiness: model configuration, scan enrichment mode, scan embeddings. Run ktx setup or rerun with --fast.', + 'warehouse cannot be ingested: enrichment is not configured (model configuration, scan enrichment mode, scan embeddings). Run ktx setup to configure a model and embeddings.', }); }); - it('honors scan.relationships.enabled when planning deep database ingest', () => { + it('honors scan.relationships.enabled when planning database ingest', () => { const plan = buildPublicIngestPlan( - deepReadyProject({ warehouse: { driver: 'postgres', context: { depth: 'deep' } } }, false), + deepReadyProject({ warehouse: { driver: 'postgres' } }, false), { projectDir: '/tmp/project', targetConnectionId: 'warehouse', @@ -392,22 +354,48 @@ describe('buildPublicIngestPlan', () => { expect(plan.targets[0]).toMatchObject({ connectionId: 'warehouse', - databaseDepth: 'deep', detectRelationships: false, }); }); }); +describe('publicProgressMessage', () => { + it('rewrites internal scan and historic-sql phrasing for public ingest progress', () => { + const databaseProject = deepReadyProject({ + warehouse: { driver: 'postgres', context: { queryHistory: { enabled: true, dialect: 'postgres' } } }, + }); + const databaseTarget = buildPublicIngestPlan(databaseProject, { + projectDir: '/tmp/project', + all: false, + targetConnectionId: 'warehouse', + queryHistory: 'default', + }).targets[0]; + + expect(databaseTarget).toBeDefined(); + expect(publicProgressMessage('Inspecting database schema', databaseTarget)).toBe('Reading database schema'); + expect(publicProgressMessage('Enriching schema metadata', databaseTarget)).toBe( + 'Building enriched schema context', + ); + expect(publicProgressMessage('Fetching source files for warehouse/historic-sql', databaseTarget)).toBe( + 'Fetching query history for warehouse', + ); + }); +}); + describe('runKtxPublicIngest', () => { + beforeEach(() => { + reportExceptionMock.mockClear(); + }); + afterEach(() => { vi.unstubAllEnvs(); }); - it('maps fast and deep database targets to scan internals', async () => { + it('maps database targets to enriched scan internals', async () => { const io = makeIo(); const project = deepReadyProject({ - fast: { driver: 'postgres' }, - deep: { driver: 'postgres', context: { depth: 'deep' } }, + first: { driver: 'postgres' }, + second: { driver: 'postgres' }, }); const runScan = vi.fn(async () => 0); @@ -421,13 +409,15 @@ describe('runKtxPublicIngest', () => { expect(runScan).toHaveBeenNthCalledWith( 1, - expect.objectContaining({ connectionId: 'deep', mode: 'enriched', detectRelationships: true }), + expect.objectContaining({ connectionId: 'first', mode: 'enriched', detectRelationships: true }), expect.anything(), + expect.objectContaining({ progress: expect.any(Object) }), ); expect(runScan).toHaveBeenNthCalledWith( 2, - expect.objectContaining({ connectionId: 'fast', mode: 'structural', detectRelationships: false }), + expect.objectContaining({ connectionId: 'second', mode: 'enriched', detectRelationships: true }), expect.anything(), + expect.objectContaining({ progress: expect.any(Object) }), ); }); @@ -438,7 +428,7 @@ describe('runKtxPublicIngest', () => { try { await initKtxProject({ projectDir }); const io = makeIo({ isTTY: true }); - const project = projectWithConnections({ + const project = deepReadyProject({ warehouse: { driver: 'sqlite', path: join(projectDir, 'warehouse.sqlite') }, }); @@ -457,6 +447,118 @@ describe('runKtxPublicIngest', () => { } }); + it('records errorDetail in ingest_completed telemetry when a target fails', async () => { + vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); + vi.stubEnv('CI', ''); + const projectDir = await mkdtemp(join(tmpdir(), 'ktx-public-ingest-telemetry-fail-')); + try { + await initKtxProject({ projectDir }); + const io = makeIo({ isTTY: true }); + const project = deepReadyProject({ + warehouse: { driver: 'sqlite', path: join(projectDir, 'warehouse.sqlite') }, + }); + + const code = await runKtxPublicIngest( + { command: 'run', projectDir, targetConnectionId: 'warehouse', all: false, json: false, inputMode: 'disabled' }, + io.io, + { loadProject: vi.fn(async () => project), runScan: vi.fn(async () => 1) }, + ); + + expect(code).toBe(1); + expect(io.stderr()).toContain('"event":"ingest_completed"'); + expect(io.stderr()).toContain('"outcome":"error"'); + expect(io.stderr()).toContain('"errorDetail"'); + } finally { + await rm(projectDir, { recursive: true, force: true }); + } + }); + + it('emits exactly one ingest_completed from the shared executePublicIngestTarget chokepoint', async () => { + // executePublicIngestTarget is the single per-target path reached by every + // entrypoint (plain/json ingest, foreground ingest via runContextBuild, and + // setup). Emitting here is what makes ingest_completed fire on every path. + vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); + vi.stubEnv('CI', ''); + const io = makeIo({ isTTY: true }); + const project = deepReadyProject({ warehouse: { driver: 'postgres' } }); + const [target] = buildPublicIngestPlan(project, { + projectDir: '/tmp/project', + targetConnectionId: 'warehouse', + all: false, + }).targets; + + const runScan = vi.fn(async () => 0); + const result = await executePublicIngestTarget( + target, + { command: 'run', projectDir: '/tmp/project', targetConnectionId: 'warehouse', all: false, json: false, inputMode: 'disabled' }, + io.io, + { runScan }, + project, + ); + + expect(result.steps.some((step) => step.status === 'failed')).toBe(false); + expect(occurrences(io.stderr(), '"event":"ingest_completed"')).toBe(1); + expect(io.stderr()).toContain('"outcome":"ok"'); + // A database-ingest target must run a scan — runKtxScan is what emits + // scan_completed, so this guards against the 0.7.0-style regression where a + // path stopped triggering the scan and the event silently went to zero. + expect(runScan).toHaveBeenCalledTimes(1); + }); + + it('still emits ingest_completed when a target fails preflight (early-return branch)', async () => { + // The chokepoint must emit on every internal branch, including the early + // preflight-failure return — otherwise failed-setup installs vanish. + vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); + vi.stubEnv('CI', ''); + const io = makeIo({ isTTY: true }); + // projectWithConnections leaves enrichment unconfigured → preflight failure. + const project = projectWithConnections({ warehouse: { driver: 'postgres' } }); + const [target] = buildPublicIngestPlan(project, { + projectDir: '/tmp/project', + targetConnectionId: 'warehouse', + all: false, + }).targets; + expect(target.preflightFailure).toBeTruthy(); + + const runScan = vi.fn(async () => 0); + await executePublicIngestTarget( + target, + { command: 'run', projectDir: '/tmp/project', targetConnectionId: 'warehouse', all: false, json: false, inputMode: 'disabled' }, + io.io, + { runScan }, + project, + ); + + expect(occurrences(io.stderr(), '"event":"ingest_completed"')).toBe(1); + expect(io.stderr()).toContain('"outcome":"error"'); + expect(runScan).not.toHaveBeenCalled(); + }); + + it('emits one ingest_completed per target and never double-emits across a multi-target run', async () => { + vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); + vi.stubEnv('CI', ''); + const projectDir = await mkdtemp(join(tmpdir(), 'ktx-public-ingest-no-double-')); + try { + await initKtxProject({ projectDir }); + const io = makeIo({ isTTY: true }); + const project = deepReadyProject({ + first: { driver: 'sqlite', path: join(projectDir, 'first.sqlite') }, + second: { driver: 'sqlite', path: join(projectDir, 'second.sqlite') }, + }); + + const code = await runKtxPublicIngest( + { command: 'run', projectDir, all: true, json: false, inputMode: 'disabled' }, + io.io, + { loadProject: vi.fn(async () => project), runScan: vi.fn(async () => 0) }, + ); + + expect(code).toBe(0); + expect(occurrences(io.stderr(), '"event":"ingest_completed"')).toBe(2); + } finally { + await rm(projectDir, { recursive: true, force: true }); + } + }); + it('runs query history after schema ingest with current-run window override', async () => { const io = makeIo(); const runtimeIo = makeIo({ isTTY: true }); @@ -576,16 +678,138 @@ describe('runKtxPublicIngest', () => { dropFailedBelow: { errorRate: 0.5, executions: 3 }, }, redactionPatterns: ['(?i)secret'], - enabledTables: ['orbit_analytics.int_active_contract_arr'], + enabledTables: [{ catalog: null, db: 'orbit_analytics', name: 'int_active_contract_arr' }], }, }); expect(ingestArgs?.historicSqlPullConfigOverride).not.toHaveProperty('enabled'); }); + it('resolves query-history scope after the schema scan writes artifacts', async () => { + const io = makeIo(); + const projectDir = await mkdtemp(join(tmpdir(), 'ktx-public-qh-scope-')); + const project = deepReadyProject({ + warehouse: { + driver: 'postgres', + schemas: ['orbit_raw'], + context: { queryHistory: { enabled: true } }, + }, + }); + const runScan = vi.fn(async () => { + await mkdir(join(projectDir, 'semantic-layer/warehouse'), { recursive: true }); + await writeFile( + join(projectDir, 'semantic-layer/warehouse/revenue.yaml'), + [ + 'name: revenue', + 'table: orbit_analytics.mart_revenue', + 'grain: [id]', + 'columns:', + ' - name: id', + ' type: string', + '', + ].join('\n'), + 'utf-8', + ); + const rawRoot = join(projectDir, 'raw-sources/warehouse/live-database/sync-1'); + await mkdir(join(rawRoot, 'tables'), { recursive: true }); + await writeFile( + join(rawRoot, 'connection.json'), + `${JSON.stringify({ connectionId: 'warehouse', driver: 'postgres' }, null, 2)}\n`, + 'utf-8', + ); + await writeFile( + join(rawRoot, 'tables/accounts.json'), + `${JSON.stringify( + { + catalog: null, + db: 'orbit_raw', + name: 'accounts', + kind: 'table', + comment: null, + estimatedRows: null, + columns: [ + { + name: 'id', + nativeType: 'integer', + normalizedType: 'integer', + dimensionType: 'number', + nullable: false, + primaryKey: true, + comment: null, + }, + ], + foreignKeys: [], + }, + null, + 2, + )}\n`, + 'utf-8', + ); + await writeFile( + join(rawRoot, 'scan-report.json'), + `${JSON.stringify( + { + connectionId: 'warehouse', + driver: 'postgres', + syncId: 'sync-1', + runId: 'scan-sync-1', + trigger: 'cli', + mode: 'enriched', + dryRun: false, + artifactPaths: { + rawSourcesDir: 'raw-sources/warehouse/live-database/sync-1', + reportPath: 'raw-sources/warehouse/live-database/sync-1/scan-report.json', + manifestShards: [], + enrichmentArtifacts: [], + }, + counts: {}, + warnings: [], + enrichment: {}, + enrichmentState: {}, + }, + null, + 2, + )}\n`, + 'utf-8', + ); + return 0; + }); + const runIngest = vi.fn>(async () => 0); + + await expect( + runKtxPublicIngest( + { + command: 'run', + projectDir, + targetConnectionId: 'warehouse', + all: false, + json: false, + inputMode: 'disabled', + queryHistory: 'enabled', + }, + io.io, + { loadProject: vi.fn(async () => ({ ...project, projectDir })), runScan, runIngest }, + ), + ).resolves.toBe(0); + + const ingestArgs = runIngest.mock.calls[0]?.[0] as + | Extract>[0], { command: 'run' }> + | undefined; + expect(ingestArgs?.historicSqlPullConfigOverride).toMatchObject({ + dialect: 'postgres', + enabledSchemas: ['orbit_analytics', 'orbit_raw'], + modeledTableCatalog: [ + { catalog: null, db: 'orbit_analytics', name: 'mart_revenue' }, + { catalog: null, db: 'orbit_raw', name: 'accounts' }, + ], + }); + + await rm(projectDir, { recursive: true, force: true }); + }); + it('prints the schema-first notice for explicit query-history runs', async () => { const io = makeIo(); const project = deepReadyProject({ - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, + warehouse: { driver: 'postgres' }, }); const runScan = vi.fn(async () => 0); const runIngest = vi.fn(async () => 0); @@ -611,7 +835,7 @@ describe('runKtxPublicIngest', () => { it('suppresses internal scan output for public database ingest summaries', async () => { const io = makeIo(); - const project = projectWithConnections({ warehouse: { driver: 'postgres' } }); + const project = deepReadyProject({ warehouse: { driver: 'postgres' } }); const runScan = vi.fn(async (_args, scanIo) => { scanIo.stdout.write('KTX scan completed\n'); scanIo.stdout.write('Mode: structural\n'); @@ -645,7 +869,7 @@ describe('runKtxPublicIngest', () => { it('sanitizes captured database scan failure details in direct public output', async () => { const io = makeIo(); - const project = deepReadyProject({ warehouse: { driver: 'postgres', context: { depth: 'deep' } } }); + const project = deepReadyProject({ warehouse: { driver: 'postgres' } }); const runScan = vi.fn(async (_args, scanIo) => { scanIo.stdout.write('KTX scan enrichment failed after structural scan completed: embedding service timed out\n'); return 1; @@ -660,7 +884,6 @@ describe('runKtxPublicIngest', () => { all: false, json: false, inputMode: 'disabled', - depth: 'deep', }, io.io, { loadProject: vi.fn(async () => project), runScan }, @@ -670,7 +893,7 @@ describe('runKtxPublicIngest', () => { expect(io.stdout()).toContain( 'warehouse failed: Database enrichment failed after schema context completed: embedding service timed out.', ); - expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --deep'); + expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project'); expect(io.stdout()).not.toContain('KTX scan enrichment failed'); expect(io.stdout()).not.toContain('structural scan'); }); @@ -708,13 +931,16 @@ describe('runKtxPublicIngest', () => { expect(io.stdout()).not.toContain('Report: report-docs-1'); expect(io.stdout()).not.toContain('Adapter:'); expect(io.stdout()).not.toContain('notion\n'); - expect(io.stderr()).toBe(''); + expect(io.stderr()).toContain('docs · source ingest\n'); + expect(io.stderr()).toContain(' done\n'); + expect(io.stderr()).not.toContain('Report: report-docs-1'); + expect(io.stderr()).not.toContain('Adapter:'); }); it('suppresses historic-sql report output during direct public query-history ingest', async () => { const io = makeIo(); const project = deepReadyProject({ - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, + warehouse: { driver: 'postgres' }, }); const runScan = vi.fn(async () => 0); const runIngest = vi.fn(async (_args, ingestIo) => { @@ -747,9 +973,168 @@ describe('runKtxPublicIngest', () => { expect(io.stdout()).not.toContain('Report: report-query-history-1'); expect(io.stdout()).not.toContain('Adapter:'); expect(io.stdout()).not.toContain('historic-sql'); + expect(io.stderr()).toContain('warehouse · database schema\n'); + expect(io.stderr()).toContain('warehouse · query history\n'); + expect(io.stderr()).toContain(' done\n'); + expect(io.stderr()).not.toContain('Report: report-query-history-1'); + expect(io.stderr()).not.toContain('Adapter:'); + expect(io.stderr()).not.toContain('historic-sql'); + }); + + it('streams plain non-json progress to stderr while keeping final results on stdout', async () => { + const io = makeIo(); + const project = deepReadyProject({ + warehouse: { driver: 'postgres', context: { queryHistory: { enabled: true, dialect: 'postgres' } } }, + docs: { driver: 'notion' }, + }); + const runScan = vi.fn>(async (_args, scanIo, deps) => { + scanIo.stdout.write('KTX scan completed\n'); + scanIo.stdout.write('Report: raw-sources/warehouse/live-database/sync-1/scan-report.json\n'); + await deps?.progress?.update(0.12, 'Inspecting database schema'); + const enrichmentProgress = deps?.progress?.startPhase(0.5); + await enrichmentProgress?.update(0.75, 'Enriching schema metadata', { transient: true }); + await deps?.progress?.update(1, 'Writing schema artifacts'); + return 0; + }); + const runIngest = vi.fn>(async (ingestArgs, ingestIo, deps) => { + if (ingestArgs.command !== 'run') { + throw new Error(`Unexpected ingest command: ${ingestArgs.command}`); + } + ingestIo.stdout.write(`Adapter: ${ingestArgs.adapter}\n`); + ingestIo.stdout.write('Report: report-progress-1\n'); + if (ingestArgs.adapter === 'historic-sql') { + deps?.progress?.({ percent: 15, message: 'Fetching source files for warehouse/historic-sql' }); + deps?.progress?.({ percent: 90, message: 'Saved memory: 1 wiki, 1 SL' }); + return 0; + } + deps?.progress?.({ percent: 55, message: 'Processing 3/8 tasks' }); + deps?.progress?.({ percent: 90, message: 'Saved memory: 6 wiki, 2 SL' }); + return 0; + }); + + await expect( + runKtxPublicIngest( + { + command: 'run', + projectDir: '/tmp/project', + all: true, + json: false, + inputMode: 'disabled', + queryHistory: 'default', + }, + io.io, + { loadProject: vi.fn(async () => project), runScan, runIngest }, + ), + ).resolves.toBe(0); + + expect(io.stdout()).toContain('Ingest finished'); + expect(io.stdout()).toContain('warehouse'); + expect(io.stdout()).toContain('docs'); + expect(io.stdout()).not.toContain('KTX scan completed'); + expect(io.stdout()).not.toContain('Report:'); + expect(io.stdout()).not.toContain('Adapter:'); + expect(io.stderr()).toContain('[1/2] warehouse · database schema\n'); + expect(io.stderr()).toContain(' [12%] Reading database schema\n'); + expect(io.stderr()).toContain(' [50%] Building enriched schema context\n'); + expect(io.stderr()).toContain('[1/2] warehouse · query history\n'); + expect(io.stderr()).toContain(' [15%] Fetching query history for warehouse\n'); + expect(io.stderr()).toContain('[2/2] docs · source ingest\n'); + expect(io.stderr()).toContain(' [55%] Processing 3/8 tasks\n'); + expect(io.stderr()).not.toContain('\r'); + }); + + it('does not emit plain progress for json public ingest output', async () => { + const io = makeIo(); + const project = deepReadyProject({ + warehouse: { driver: 'postgres' }, + }); + const runScan = vi.fn>(async (_args, _scanIo, deps) => { + expect(deps?.progress).toBeUndefined(); + return 0; + }); + + await expect( + runKtxPublicIngest( + { + command: 'run', + projectDir: '/tmp/project', + targetConnectionId: 'warehouse', + all: false, + json: true, + inputMode: 'disabled', + }, + io.io, + { loadProject: vi.fn(async () => project), runScan }, + ), + ).resolves.toBe(0); + + expect(JSON.parse(io.stdout())).toMatchObject({ + plan: { projectDir: '/tmp/project' }, + results: [{ connectionId: 'warehouse', driver: 'postgres' }], + }); expect(io.stderr()).toBe(''); }); + it('keeps captured failure details when plain progress ports are active', async () => { + const io = makeIo(); + const project = deepReadyProject({ warehouse: { driver: 'postgres' } }); + const runScan = vi.fn>(async (_args, scanIo, deps) => { + await deps?.progress?.update(0.42, 'Enriching schema metadata'); + scanIo.stdout.write('KTX scan enrichment failed after structural scan completed: embedding service timed out\n'); + return 1; + }); + + await expect( + runKtxPublicIngest( + { + command: 'run', + projectDir: '/tmp/project', + targetConnectionId: 'warehouse', + all: false, + json: false, + inputMode: 'disabled', + }, + io.io, + { loadProject: vi.fn(async () => project), runScan }, + ), + ).resolves.toBe(1); + + expect(io.stderr()).toContain('warehouse · database schema\n'); + expect(io.stderr()).toContain(' [42%] Building enriched schema context\n'); + expect(io.stderr()).toContain(' failed\n'); + expect(io.stdout()).toContain( + 'warehouse failed: Database enrichment failed after schema context completed: embedding service timed out.', + ); + expect(io.stdout()).not.toContain('KTX scan enrichment failed'); + expect(io.stdout()).not.toContain('structural scan'); + }); + + it('prints a failed plain phase when preflight fails before phase start', async () => { + const io = makeIo(); + const project = projectWithConnections({ + warehouse: { driver: 'postgres' }, + }); + + await expect( + runKtxPublicIngest( + { + command: 'run', + projectDir: '/tmp/project', + targetConnectionId: 'warehouse', + all: false, + json: false, + inputMode: 'disabled', + }, + io.io, + { loadProject: vi.fn(async () => project) }, + ), + ).resolves.toBe(1); + + expect(io.stderr()).toContain('warehouse · database schema\n'); + expect(io.stderr()).toContain(' failed · warehouse cannot be ingested: enrichment is not configured'); + expect(io.stdout()).toContain('warehouse failed: warehouse cannot be ingested: enrichment is not configured'); + }); + it('delegates interactive TTY public ingest to the foreground context-build view', async () => { const io = makeIo({ isTTY: true, interactive: true }); const project = projectWithConnections({ warehouse: { driver: 'postgres' } }); @@ -765,7 +1150,6 @@ describe('runKtxPublicIngest', () => { all: false, json: false, inputMode: 'auto', - depth: 'fast', queryHistory: 'default', }, io.io, @@ -780,7 +1164,6 @@ describe('runKtxPublicIngest', () => { targetConnectionId: 'warehouse', all: false, entrypoint: 'ingest', - depth: 'fast', queryHistory: 'default', }), io.io, @@ -792,7 +1175,7 @@ describe('runKtxPublicIngest', () => { const io = makeIo({ isTTY: true, interactive: true }); const calls: string[] = []; const project = projectWithConnections({ - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, + warehouse: { driver: 'postgres' }, }); const ensureRuntime = vi.fn(async (): Promise => { calls.push('runtime'); @@ -835,6 +1218,104 @@ describe('runKtxPublicIngest', () => { ); }); + it('reports foreground runtime preflight exceptions', async () => { + const io = makeIo({ isTTY: true, interactive: true }); + const project = projectWithConnections({ + warehouse: { driver: 'postgres' }, + }); + const ensureRuntime = vi.fn(async (): Promise => { + throw new Error('runtime unavailable'); + }); + const runContextBuild = vi.fn(async () => ({ exitCode: 0 })); + + await expect( + runKtxPublicIngest( + { + command: 'run', + projectDir: '/tmp/project', + targetConnectionId: 'warehouse', + all: false, + json: false, + inputMode: 'auto', + queryHistory: 'enabled', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'prompt', + }, + io.io, + { + loadProject: vi.fn(async () => project), + ensureRuntime, + runContextBuild, + }, + ), + ).resolves.toBe(1); + + expect(runContextBuild).not.toHaveBeenCalled(); + expect(io.stderr()).toContain('runtime unavailable'); + expect(reportExceptionMock).toHaveBeenCalledWith( + expect.objectContaining({ + context: expect.objectContaining({ source: 'ingest runtime', handled: true, fatal: false }), + projectDir: '/tmp/project', + }), + ); + }); + + it('reports foreground context-build exceptions', async () => { + const io = makeIo({ isTTY: true, interactive: true }); + const config = buildDefaultKtxProjectConfig(); + const project: KtxPublicIngestProject = { + projectDir: '/tmp/project', + config: { + ...config, + connections: { warehouse: { driver: 'postgres', password: 'env:INGEST_DB_PASSWORD' } }, // pragma: allowlist secret + llm: { + ...config.llm, + provider: { + backend: 'anthropic', + anthropic: { api_key: 'env:ANTHROPIC_API_KEY' }, // pragma: allowlist secret + }, + models: { default: 'claude-sonnet-4-6' }, + }, + }, + }; + const runContextBuild = vi.fn(async () => { + throw new Error('context build failed'); + }); + + await expect( + runKtxPublicIngest( + { + command: 'run', + projectDir: '/tmp/project', + targetConnectionId: 'warehouse', + all: false, + json: false, + inputMode: 'auto', + queryHistory: 'default', + }, + io.io, + { + loadProject: vi.fn(async () => project), + runContextBuild, + env: { + ...process.env, + ANTHROPIC_API_KEY: 'ingest-anthropic-secret', // pragma: allowlist secret + INGEST_DB_PASSWORD: 'ingest-db-password', // pragma: allowlist secret + }, + }, + ), + ).resolves.toBe(1); + + expect(io.stderr()).toContain('context build failed'); + expect(reportExceptionMock).toHaveBeenCalledWith( + expect.objectContaining({ + context: expect.objectContaining({ source: 'ingest context-build', handled: true, fatal: false }), + projectDir: '/tmp/project', + redactionSecrets: expect.arrayContaining(['ingest-anthropic-secret', 'ingest-db-password']), + }), + ); + }); + it('preflights foreground managed embeddings runtime before starting the context-build view', async () => { const io = makeIo({ isTTY: true, interactive: true }); const config = buildDefaultKtxProjectConfig(); @@ -894,10 +1375,13 @@ describe('runKtxPublicIngest', () => { it('runs all independent targets and reports partial failures', async () => { const io = makeIo(); - const project = projectWithConnections({ - warehouse: { driver: 'postgres' }, - prod_metabase: { driver: 'metabase', api_url: 'https://metabase.example.com' }, - }); + const project = deepReadyProject( + { + warehouse: { driver: 'postgres' }, + prod_metabase: { driver: 'metabase', api_url: 'https://metabase.example.com' }, + }, + false, + ); const runScan = vi.fn(async () => 1); const runIngest = vi.fn(async () => 0); @@ -924,28 +1408,30 @@ describe('runKtxPublicIngest', () => { inputMode: 'disabled', }), expect.anything(), + expect.objectContaining({ progress: expect.any(Function) }), ); expect(runScan).toHaveBeenCalledWith( { command: 'run', projectDir: '/tmp/project', connectionId: 'warehouse', - mode: 'structural', + mode: 'enriched', detectRelationships: false, dryRun: false, }, expect.anything(), + expect.objectContaining({ progress: expect.any(Object) }), ); expect(io.stdout()).toContain('Ingest finished with partial failures'); expect(io.stdout()).toContain('warehouse failed at database-schema.'); - expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --fast'); + expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project'); expect(io.stdout()).not.toContain('Debug:'); }); it('skips the query-history facet but keeps the target green when query-history fails', async () => { const io = makeIo(); const project = deepReadyProject({ - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, + warehouse: { driver: 'postgres' }, }); const runScan = vi.fn(async () => 0); const runIngest = vi.fn(async (_args, ingestIo) => { @@ -978,14 +1464,53 @@ describe('runKtxPublicIngest', () => { 'Query history failed for 60 tasks. First failure: Google Cloud authentication failed while analyzing query history', ); expect(io.stdout()).not.toContain('warehouse failed: Error:'); - expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --deep --query-history'); + expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --query-history'); expect(io.stdout()).not.toContain('historic-sql'); }); + it('reports the query-history failure without leaking earlier scan report output', async () => { + const io = makeIo(); + const project = deepReadyProject({ + warehouse: { driver: 'postgres' }, + }); + const runScan = vi.fn(async (_args, scanIo) => { + scanIo.stdout.write('Run: scan-run-1\n'); + scanIo.stdout.write('Mode: enriched\n'); + scanIo.stdout.write('Dry run: no\n'); + scanIo.stdout.write('KTX scan completed\n'); + return 0; + }); + const runIngest = vi.fn(async (_args, ingestIo) => { + ingestIo.stderr.write('Stopped query history before persisting any results\n'); + return 1; + }); + + await expect( + runKtxPublicIngest( + { + command: 'run', + projectDir: '/tmp/project', + targetConnectionId: 'warehouse', + all: false, + json: false, + inputMode: 'disabled', + queryHistory: 'enabled', + }, + io.io, + { loadProject: vi.fn(async () => project), runScan, runIngest }, + ), + ).resolves.toBe(0); + + expect(io.stdout()).toContain('Skipped query history:'); + expect(io.stdout()).toContain('Stopped query history before persisting any results'); + expect(io.stdout()).not.toContain('Dry run: no'); + expect(io.stdout()).not.toContain('Mode: enriched'); + }); + it('prints the runtime artifact build hint for missing query-history runtime assets', async () => { const io = makeIo(); const project = deepReadyProject({ - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, + warehouse: { driver: 'postgres' }, }); const runScan = vi.fn(async () => 0); const runIngest = vi.fn(async (_args, ingestIo) => { @@ -1016,14 +1541,14 @@ describe('runKtxPublicIngest', () => { expect(io.stdout()).toContain( 'In a source checkout, build the local runtime assets with: pnpm run artifacts:build', ); - expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --deep --query-history'); + expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --query-history'); expect(io.stdout()).not.toContain('Then retry the runtime-backed KTX command'); }); - it('fails deep-readiness targets before work starts while continuing independent --all targets', async () => { + it('fails enrichment-readiness targets before work starts while continuing independent --all targets', async () => { const io = makeIo(); const project = projectWithConnections({ - warehouse: { driver: 'postgres', context: { depth: 'deep' } }, + warehouse: { driver: 'postgres' }, docs: { driver: 'notion' }, }); const runScan = vi.fn(async () => 0); @@ -1041,13 +1566,14 @@ describe('runKtxPublicIngest', () => { expect(runIngest).toHaveBeenCalledWith( expect.objectContaining({ command: 'run', connectionId: 'docs', adapter: 'notion' }), expect.anything(), + expect.objectContaining({ progress: expect.any(Function) }), ); - expect(io.stdout()).toContain('warehouse requires deep ingest readiness'); + expect(io.stdout()).toContain('warehouse cannot be ingested: enrichment is not configured'); }); - it('can request enriched relationship scans for setup-managed context builds', async () => { + it('drives scan relationship detection from project config, not from legacy args', async () => { const io = makeIo(); - const project = deepReadyProject({ warehouse: { driver: 'postgres' } }); + const project = deepReadyProject({ warehouse: { driver: 'postgres' } }, false); const runScan = vi.fn(async () => 0); await expect( @@ -1075,16 +1601,17 @@ describe('runKtxPublicIngest', () => { projectDir: '/tmp/project', connectionId: 'warehouse', mode: 'enriched', - detectRelationships: true, + detectRelationships: false, dryRun: false, }, expect.objectContaining({ capturedOutput: expect.any(Function) }), + expect.objectContaining({ progress: expect.any(Object) }), ); }); it('prints stable JSON results', async () => { const io = makeIo(); - const project = projectWithConnections({ warehouse: { driver: 'postgres' } }); + const project = deepReadyProject({ warehouse: { driver: 'postgres' } }); await expect( runKtxPublicIngest( @@ -1151,6 +1678,7 @@ describe('runKtxPublicIngest', () => { sourceDir: '/repo/dbt', }), expect.objectContaining({ capturedOutput: expect.any(Function) }), + expect.objectContaining({ progress: expect.any(Function) }), ); }); @@ -1187,6 +1715,7 @@ describe('runKtxPublicIngest', () => { allowImplicitAdapter: true, }), expect.objectContaining({ capturedOutput: expect.any(Function) }), + expect.objectContaining({ progress: expect.any(Function) }), ); }); diff --git a/packages/cli/test/reveal-password-prompt.test.ts b/packages/cli/test/reveal-password-prompt.test.ts new file mode 100644 index 00000000..7bb8cc10 --- /dev/null +++ b/packages/cli/test/reveal-password-prompt.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest'; +import { maskRevealingTail } from '../src/reveal-password-prompt.js'; + +const MASK = '▪'; + +describe('maskRevealingTail', () => { + it('reveals the last `tail` characters of a long value', () => { + const value = 'example-token-value-abcd'; + const masked = maskRevealingTail(value, MASK, 4); + expect(masked).toBe(`${MASK.repeat(value.length - 4)}abcd`); + expect(masked.endsWith('abcd')).toBe(true); + }); + + it('keeps the same length as the input so cursor slicing stays aligned', () => { + for (const secret of ['', 'a', 'abcdefgh', 'abcdefghijklmnop']) { + expect(maskRevealingTail(secret, MASK, 4)).toHaveLength(secret.length); + } + }); + + it('fully masks secrets that are not longer than tail * 2', () => { + expect(maskRevealingTail('abcdefgh', MASK, 4)).toBe(MASK.repeat(8)); + expect(maskRevealingTail('abcd', MASK, 4)).toBe(MASK.repeat(4)); + expect(maskRevealingTail('ab', MASK, 4)).toBe(MASK.repeat(2)); + }); + + it('reveals the tail once the secret crosses the tail * 2 boundary', () => { + // length 9 > 8 → reveal last 4, hide the first 5 + expect(maskRevealingTail('abcdefghi', MASK, 4)).toBe(`${MASK.repeat(5)}fghi`); + }); + + it('fully masks an empty value', () => { + expect(maskRevealingTail('', MASK, 4)).toBe(''); + }); + + it('honors a custom tail count', () => { + // tail 2 reveals only when length > 4 + expect(maskRevealingTail('abcde', MASK, 2)).toBe(`${MASK.repeat(3)}de`); + expect(maskRevealingTail('abcd', MASK, 2)).toBe(MASK.repeat(4)); + }); +}); diff --git a/packages/cli/src/runtime-requirements.test.ts b/packages/cli/test/runtime-requirements.test.ts similarity index 78% rename from packages/cli/src/runtime-requirements.test.ts rename to packages/cli/test/runtime-requirements.test.ts index 5f8831cf..0a951056 100644 --- a/packages/cli/src/runtime-requirements.test.ts +++ b/packages/cli/test/runtime-requirements.test.ts @@ -1,9 +1,9 @@ -import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from '../src/context/project/config.js'; import { describe, expect, it } from 'vitest'; import { resolveProjectRuntimeRequirements, resolvePublicIngestRuntimeRequirements, -} from './runtime-requirements.js'; +} from '../src/runtime-requirements.js'; describe('runtime requirement detection', () => { it('does not require runtime for agent/MCP setup alone', () => { @@ -26,6 +26,33 @@ describe('runtime requirement detection', () => { ); }); + it('does not treat stale local Looker driver aliases as Looker sources', () => { + const config: KtxProjectConfig = { + ...buildDefaultKtxProjectConfig(), + connections: { + stale: { driver: 'local_looker' } as never, + }, + }; + + expect(resolveProjectRuntimeRequirements(config).features).toEqual([]); + expect( + resolvePublicIngestRuntimeRequirements({ + projectDir: '/tmp/project', + warnings: [], + targets: [ + { + connectionId: 'stale', + driver: 'local_looker', + operation: 'source-ingest', + adapter: 'local_looker', + debugCommand: 'ktx ingest stale --debug', + steps: ['source-ingest'], + }, + ], + }).features, + ).toEqual([]); + }); + it('requires core for query-history ingest unless SQL analysis is externally configured', () => { const config: KtxProjectConfig = { ...buildDefaultKtxProjectConfig(), diff --git a/packages/cli/src/runtime.test.ts b/packages/cli/test/runtime.test.ts similarity index 99% rename from packages/cli/src/runtime.test.ts rename to packages/cli/test/runtime.test.ts index 266359e4..4525bf83 100644 --- a/packages/cli/src/runtime.test.ts +++ b/packages/cli/test/runtime.test.ts @@ -3,13 +3,13 @@ import type { ManagedPythonDaemonStopAllResult, ManagedPythonDaemonStartResult, ManagedPythonDaemonStopResult, -} from './managed-python-daemon.js'; +} from '../src/managed-python-daemon.js'; import type { ManagedPythonRuntimeDoctorCheck, ManagedPythonRuntimeInstallResult, ManagedPythonRuntimeStatus, -} from './managed-python-runtime.js'; -import { runKtxRuntime, type KtxRuntimeDeps } from './runtime.js'; +} from '../src/managed-python-runtime.js'; +import { runKtxRuntime, type KtxRuntimeDeps } from '../src/runtime.js'; function makeIo() { let stdout = ''; diff --git a/packages/cli/src/scan.test.ts b/packages/cli/test/scan.test.ts similarity index 92% rename from packages/cli/src/scan.test.ts rename to packages/cli/test/scan.test.ts index 5ec745e6..51c55498 100644 --- a/packages/cli/src/scan.test.ts +++ b/packages/cli/test/scan.test.ts @@ -1,12 +1,19 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import type { SourceAdapter } from './context/ingest/types.js'; -import { initKtxProject } from './context/project/project.js'; -import type { KtxScanReport } from './context/scan/types.js'; -import type { LocalScanRunResult, RunLocalScanOptions } from './context/scan/local-scan.js'; +import type { SourceAdapter } from '../src/context/ingest/types.js'; +import { parseKtxProjectConfig, serializeKtxProjectConfig } from '../src/context/project/config.js'; +import { initKtxProject } from '../src/context/project/project.js'; +import type { KtxScanReport } from '../src/context/scan/types.js'; +import type { LocalScanRunResult, RunLocalScanOptions } from '../src/context/scan/local-scan.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { createCliScanProgress, runKtxScan, type KtxScanDeps } from './scan.js'; +import { createCliScanProgress, runKtxScan, type KtxScanDeps } from '../src/scan.js'; + +const reportExceptionMock = vi.hoisted(() => vi.fn(async () => {})); + +vi.mock('../src/telemetry/exception.js', () => ({ + reportException: reportExceptionMock, +})); const sqlServerExtractSchema = vi.hoisted(() => vi.fn(async (connectionId: string) => ({ @@ -123,7 +130,7 @@ const createPostgresLiveDatabaseIntrospection = vi.hoisted(() => ); const isKtxPostgresConnectionConfig = vi.hoisted(() => vi.fn((connection: { driver?: string } | undefined) => - ['postgres', 'postgresql'].includes(String(connection?.driver ?? '').toLowerCase()), + String(connection?.driver ?? '').toLowerCase() === 'postgres', ), ); const KtxPostgresScanConnector = vi.hoisted( @@ -138,35 +145,35 @@ const KtxPostgresScanConnector = vi.hoisted( }, ); -vi.mock('./connectors/sqlserver/connector.js', () => ({ +vi.mock('../src/connectors/sqlserver/connector.js', () => ({ isKtxSqlServerConnectionConfig, KtxSqlServerScanConnector, })); -vi.mock('./connectors/sqlserver/live-database-introspection.js', () => ({ +vi.mock('../src/connectors/sqlserver/live-database-introspection.js', () => ({ createSqlServerLiveDatabaseIntrospection, })); -vi.mock('./connectors/bigquery/connector.js', () => ({ +vi.mock('../src/connectors/bigquery/connector.js', () => ({ isKtxBigQueryConnectionConfig, KtxBigQueryScanConnector, })); -vi.mock('./connectors/bigquery/live-database-introspection.js', () => ({ +vi.mock('../src/connectors/bigquery/live-database-introspection.js', () => ({ createBigQueryLiveDatabaseIntrospection, })); -vi.mock('./connectors/snowflake/connector.js', () => ({ +vi.mock('../src/connectors/snowflake/connector.js', () => ({ isKtxSnowflakeConnectionConfig, KtxSnowflakeScanConnector, })); -vi.mock('./connectors/snowflake/live-database-introspection.js', () => ({ +vi.mock('../src/connectors/snowflake/live-database-introspection.js', () => ({ createSnowflakeLiveDatabaseIntrospection, })); -vi.mock('./connectors/postgres/connector.js', () => ({ +vi.mock('../src/connectors/postgres/connector.js', () => ({ isKtxPostgresConnectionConfig, KtxPostgresScanConnector, })); -vi.mock('./connectors/postgres/live-database-introspection.js', () => ({ +vi.mock('../src/connectors/postgres/live-database-introspection.js', () => ({ createPostgresLiveDatabaseIntrospection, })); @@ -317,6 +324,7 @@ describe('runKtxScan', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-scan-')); + reportExceptionMock.mockClear(); }); afterEach(async () => { @@ -423,6 +431,69 @@ describe('runKtxScan', () => { expect(io.stderr()).not.toContain(tempDir); }); + it('records the raw errorDetail in scan_completed telemetry when the scan throws', async () => { + vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); + vi.stubEnv('CI', ''); + vi.stubEnv('ANTHROPIC_API_KEY', 'anthropic-callsite-secret'); // pragma: allowlist secret + vi.stubEnv('DATABASE_URL', 'postgres://svc:scan-db-password@db.example.test/analytics'); // pragma: allowlist secret + await initKtxProject({ projectDir: tempDir }); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + await writeFile( + join(tempDir, 'ktx.yaml'), + serializeKtxProjectConfig({ + ...config, + connections: { + warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' }, + }, + llm: { + ...config.llm, + provider: { + backend: 'anthropic', + anthropic: { api_key: 'env:ANTHROPIC_API_KEY' }, // pragma: allowlist secret + }, + models: { default: 'claude-sonnet-4-6' }, + }, + }), + 'utf-8', + ); + const runLocalScan = vi.fn(async (): Promise => { + const error = new Error('introspection timed out'); + (error as { code?: unknown }).code = 'ETIMEDOUT'; + throw error; + }); + const io = makeIo({ isTTY: true }); + + const code = await runKtxScan( + { + command: 'run', + projectDir: tempDir, + connectionId: 'warehouse', + mode: 'structural', + detectRelationships: false, + dryRun: false, + databaseIntrospectionUrl: 'http://127.0.0.1:8765', + }, + io.io, + { runLocalScan, createLocalIngestAdapters: noLocalIngestAdapters }, + ); + + expect(code).toBe(1); + expect(io.stderr()).toContain('"event":"scan_completed"'); + expect(io.stderr()).toContain('"outcome":"error"'); + expect(io.stderr()).toContain('"errorDetail":"ETIMEDOUT: introspection timed out"'); + expect(reportExceptionMock).toHaveBeenCalledWith( + expect.objectContaining({ + context: expect.objectContaining({ source: 'scan run', handled: true, fatal: false }), + projectDir: tempDir, + redactionSecrets: expect.arrayContaining([ + 'anthropic-callsite-secret', + 'postgres://svc:scan-db-password@db.example.test/analytics', // pragma: allowlist secret + 'scan-db-password', + ]), + }), + ); + }); + it('passes KTX daemon options to local ingest adapters when no explicit daemon URL is set', async () => { await initKtxProject({ projectDir: tempDir }); const createLocalIngestAdapters = vi.fn(() => []); diff --git a/packages/cli/src/setup-agents.test.ts b/packages/cli/test/setup-agents.test.ts similarity index 95% rename from packages/cli/src/setup-agents.test.ts rename to packages/cli/test/setup-agents.test.ts index a5f488d5..c6c2d7c4 100644 --- a/packages/cli/src/setup-agents.test.ts +++ b/packages/cli/test/setup-agents.test.ts @@ -1,7 +1,7 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { readKtxSetupState } from './context/project/setup-config.js'; +import { readKtxSetupState } from '../src/context/project/setup-config.js'; import { strFromU8, unzipSync } from 'fflate'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { @@ -11,7 +11,7 @@ import { readKtxAgentInstallManifest, removeKtxAgentInstall, runKtxSetupAgentsStep, -} from './setup-agents.js'; +} from '../src/setup-agents.js'; function makeIo() { let stdout = ''; @@ -82,7 +82,6 @@ describe('setup agents', () => { { kind: 'file', path: join(tempDir, '.claude/skills/ktx-analytics/SKILL.md'), role: 'analytics-skill' }, ]); expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'claude-desktop', scope: 'global', mode: 'mcp' })).toEqual([ - { kind: 'file', path: join(tempDir, '.ktx/agents/claude/ktx-plugin-runner.sh'), role: 'launcher' }, { kind: 'file', path: join(tempDir, '.ktx/agents/claude/ktx-analytics.zip'), @@ -127,7 +126,6 @@ describe('setup agents', () => { { kind: 'file', path: join(tempDir, '.agents/skills/ktx/SKILL.md') }, ]); expect(plannedKtxAgentFiles({ projectDir: tempDir, target: 'claude-desktop', scope: 'global', mode: 'mcp-cli' })).toEqual([ - { kind: 'file', path: join(tempDir, '.ktx/agents/claude/ktx-plugin-runner.sh'), role: 'launcher' }, { kind: 'file', path: join(tempDir, '.ktx/agents/claude/ktx-analytics.zip'), @@ -420,6 +418,11 @@ describe('setup agents', () => { label: 'Ask data questions + manage KTX with CLI commands', hint: 'Adds an admin CLI skill so agents can run ktx status, sl, wiki, and setup commands.', }, + { + value: 'skip', + label: 'Skip agent setup for now', + hint: 'Leaves agent integration incomplete. You can run ktx setup --agents later.', + }, ], }); expect(prompts.multiselect).toHaveBeenCalledWith( @@ -429,6 +432,58 @@ describe('setup agents', () => { ); }); + it('lets interactive setup skip agent integration from the connection mode prompt', async () => { + const io = makeIo(); + const prompts = { + select: vi.fn(async () => 'skip'), + multiselect: vi.fn(async () => { + throw new Error('target selection should not run'); + }), + cancel: vi.fn(), + }; + + await expect( + runKtxSetupAgentsStep( + { + projectDir: tempDir, + inputMode: 'auto', + yes: false, + agents: true, + scope: 'project', + mode: 'mcp', + skipAgents: false, + }, + io.io, + { prompts }, + ), + ).resolves.toMatchObject({ status: 'skipped', projectDir: tempDir }); + + expect(prompts.select).toHaveBeenCalledWith({ + message: 'What should agents be allowed to do with this KTX project?', + options: [ + { + value: 'mcp', + label: 'Ask data questions with KTX MCP', + hint: 'Installs the MCP connection and analytics workflow skill. Best for normal use.', + }, + { + value: 'mcp-cli', + label: 'Ask data questions + manage KTX with CLI commands', + hint: 'Adds an admin CLI skill so agents can run ktx status, sl, wiki, and setup commands.', + }, + { + value: 'skip', + label: 'Skip agent setup for now', + hint: 'Leaves agent integration incomplete. You can run ktx setup --agents later.', + }, + ], + }); + expect(prompts.multiselect).not.toHaveBeenCalled(); + expect(io.stdout()).toContain('Agent integration skipped.'); + await expect(stat(join(tempDir, '.ktx/agents/install-manifest.json'))).rejects.toThrow(); + expect(await readKtxSetupState(tempDir)).toEqual({ completed_steps: [] }); + }); + it('prompts for global scope when every selected target supports it', async () => { const home = await mkdtemp(join(tmpdir(), 'ktx-setup-agents-home-')); const previousHome = process.env.HOME; @@ -518,19 +573,15 @@ describe('setup agents', () => { const launcherPath = join(tempDir, '.ktx/agents/claude/ktx-plugin-runner.sh'); await expect(stat(analyticsSkillPath)).resolves.toBeDefined(); await expect(stat(adminSkillPath)).rejects.toThrow(); - const launcherStat = await stat(launcherPath); - expect(launcherStat.mode & 0o111).not.toBe(0); - const launcher = await readFile(launcherPath, 'utf-8'); - expect(launcher).toContain('KTX_CLI_BIN='); - expect(launcher).toContain('.nvm/versions/node'); + await expect(stat(launcherPath)).rejects.toThrow(); const configPath = join(home, 'Library/Application Support/Claude/claude_desktop_config.json'); const config = JSON.parse(await readFile(configPath, 'utf-8')) as { mcpServers: { ktx: { command: string; args: string[]; env?: Record } }; }; expect(config.mcpServers.ktx).toEqual({ - command: launcherPath, - args: ['--project-dir', tempDir, 'mcp', 'stdio'], + command: process.execPath, + args: [expect.stringContaining('bin.js'), '--project-dir', tempDir, 'mcp', 'stdio'], }); expect(await readZipText(analyticsSkillPath, 'ktx-analytics/SKILL.md')).toContain('KTX Analytics Workflow'); @@ -901,7 +952,7 @@ describe('setup agents', () => { const configPath = join(home, 'Library/Application Support/Claude/claude_desktop_config.json'); await expect(stat(analyticsSkillPath)).resolves.toBeDefined(); await expect(stat(adminSkillPath)).resolves.toBeDefined(); - await expect(stat(launcherPath)).resolves.toBeDefined(); + await expect(stat(launcherPath)).rejects.toThrow(); const beforeConfig = JSON.parse(await readFile(configPath, 'utf-8')) as { mcpServers: Record; }; @@ -911,7 +962,6 @@ describe('setup agents', () => { await expect(stat(analyticsSkillPath)).rejects.toThrow(); await expect(stat(adminSkillPath)).rejects.toThrow(); - await expect(stat(launcherPath)).rejects.toThrow(); const afterConfig = JSON.parse(await readFile(configPath, 'utf-8')) as { mcpServers: Record; }; diff --git a/packages/cli/src/setup-context.test.ts b/packages/cli/test/setup-context.test.ts similarity index 82% rename from packages/cli/src/setup-context.test.ts rename to packages/cli/test/setup-context.test.ts index 7bcad93e..743cfee9 100644 --- a/packages/cli/src/setup-context.test.ts +++ b/packages/cli/test/setup-context.test.ts @@ -1,8 +1,8 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { buildDefaultKtxProjectConfig, parseKtxProjectConfig, serializeKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; -import { readKtxSetupState, writeKtxSetupState } from './context/project/setup-config.js'; +import { buildDefaultKtxProjectConfig, serializeKtxProjectConfig, type KtxProjectConfig } from '../src/context/project/config.js'; +import { readKtxSetupState, writeKtxSetupState } from '../src/context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { @@ -11,7 +11,7 @@ import { runKtxSetupContextStep, type KtxSetupContextDeps, writeKtxSetupContextState, -} from './setup-context.js'; +} from '../src/setup-context.js'; function makeIo() { let stdout = ''; @@ -49,7 +49,7 @@ async function writeReadyProject(projectDir: string, overrides: ReadyProjectOver ...defaults, setup: { database_connection_ids: ['warehouse'] }, connections: { - warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', context: { depth: 'deep' } }, + warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' }, docs: { driver: 'notion', auth_token_ref: 'env:NOTION_TOKEN', crawl_mode: 'all_accessible' }, }, llm: { @@ -264,6 +264,7 @@ describe('setup context build state', () => { now: () => new Date('2026-05-09T10:00:00.000Z'), runContextBuild: runContextBuildMock, verifyContextReady, + testConnection: async () => 0, }, ), ).resolves.toEqual({ status: 'ready', projectDir: tempDir, runId: 'setup-context-local-abc123' }); @@ -315,6 +316,7 @@ describe('setup context build state', () => { runIdFactory: () => 'setup-context-local-failed', now: () => new Date('2026-05-09T10:00:00.000Z'), runContextBuild: runContextBuildMock, + testConnection: async () => 0, }, ), ).resolves.toEqual({ status: 'failed', projectDir: tempDir }); @@ -332,6 +334,31 @@ describe('setup context build state', () => { }); }); + it('captures the raw errorDetail on the result when the context build throws', async () => { + await writeReadyProject(tempDir); + const io = makeIo(); + const runContextBuildMock = vi.fn>(async () => { + throw new Error('managed runtime exited with code 1'); + }); + + await expect( + runKtxSetupContextStep( + { projectDir: tempDir, inputMode: 'disabled' }, + io.io, + { + runIdFactory: () => 'setup-context-local-throw', + now: () => new Date('2026-05-09T10:00:00.000Z'), + runContextBuild: runContextBuildMock, + testConnection: async () => 0, + }, + ), + ).resolves.toEqual({ + status: 'failed', + projectDir: tempDir, + errorDetail: 'managed runtime exited with code 1', + }); + }); + it('marks context complete without prompting when initial source ingest already made agent context', async () => { await writeReadyProject(tempDir); await mkdir(join(tempDir, 'semantic-layer', 'dbt-main'), { recursive: true }); @@ -399,6 +426,7 @@ describe('setup context build state', () => { runIdFactory: () => 'setup-context-local-enriched-scan', now: () => new Date('2026-05-09T10:00:00.000Z'), runContextBuild: runContextBuildMock, + testConnection: async () => 0, }, ), ).resolves.toEqual({ status: 'ready', projectDir: tempDir, runId: 'setup-context-local-enriched-scan' }); @@ -407,130 +435,10 @@ describe('setup context build state', () => { expect(io.stdout()).not.toContain('Existing context artifacts were found from setup ingest.'); }); - it('treats fast database context as ready from schema manifest shards without AI artifacts', async () => { + it('requires completed relationships for database context when relationship discovery is enabled', async () => { await writeReadyProject(tempDir, { connections: { - warehouse: { driver: 'postgres', readonly: true, context: { depth: 'fast' } }, - }, - llm: { provider: { backend: 'none' }, models: {} }, - scan: { enrichment: { mode: 'none' } }, - }); - await mkdir(join(tempDir, 'semantic-layer', 'warehouse', '_schema'), { recursive: true }); - await writeFile(join(tempDir, 'semantic-layer', 'warehouse', '_schema', 'public.yaml'), 'tables: {}\n'); - await writeScanReport(tempDir, '2026-05-09T10:00:00.000Z', { - mode: 'structural', - tableDescriptions: 'skipped', - columnDescriptions: 'skipped', - embeddings: 'skipped', - manifestShards: ['semantic-layer/warehouse/_schema/public.yaml'], - }); - const io = makeIo(); - const runContextBuildMock = vi.fn>(async () => ({ - exitCode: 0, - })); - - await expect( - runKtxSetupContextStep( - { projectDir: tempDir, inputMode: 'disabled' }, - io.io, - { - runContextBuild: runContextBuildMock, - }, - ), - ).resolves.toMatchObject({ status: 'ready' }); - - expect(runContextBuildMock).not.toHaveBeenCalled(); - expect(io.stdout()).toContain('Existing context artifacts were found from setup ingest.'); - }); - - it('stores fast context depth non-interactively when deep readiness is missing', async () => { - await writeReadyProject(tempDir, { - connections: { warehouse: { driver: 'postgres', readonly: true } }, - llm: { provider: { backend: 'none' }, models: {} }, - scan: { enrichment: { mode: 'none' } }, - }); - const io = makeIo(); - const runContextBuildMock = vi.fn>(async () => ({ - exitCode: 0, - })); - const verifyContextReady = vi.fn(async () => ({ - ready: true, - agentContextReady: true, - semanticSearchReady: true, - details: ['ready'], - })); - - await expect( - runKtxSetupContextStep( - { projectDir: tempDir, inputMode: 'disabled' }, - io.io, - { runContextBuild: runContextBuildMock, verifyContextReady }, - ), - ).resolves.toMatchObject({ status: 'ready' }); - - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.connections.warehouse.context).toMatchObject({ depth: 'fast' }); - expect(runContextBuildMock).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ projectDir: tempDir, inputMode: 'disabled' }), - expect.anything(), - expect.anything(), - ); - expect(runContextBuildMock.mock.calls[0]?.[1]).not.toMatchObject({ - scanMode: 'enriched', - detectRelationships: true, - }); - }); - - it('prompts for database context depth after final readiness is known', async () => { - await writeReadyProject(tempDir, { - connections: { warehouse: { driver: 'postgres', readonly: true } }, - llm: { - provider: { backend: 'gateway', gateway: { api_key: 'env:KTX_GATEWAY_API_KEY' } }, // pragma: allowlist secret - models: { default: 'gpt-test' }, - }, - scan: { - enrichment: { - mode: 'llm', - embeddings: { backend: 'openai', model: 'text-embedding-3-small', dimensions: 1536 }, - }, - }, - }); - const io = makeIo(); - const select = vi.fn(async () => 'deep'); - const runContextBuildMock = vi.fn(async () => ({ exitCode: 0 })); - const verifyContextReady = vi.fn(async () => ({ - ready: true, - agentContextReady: true, - semanticSearchReady: true, - details: ['ready'], - })); - - await expect( - runKtxSetupContextStep( - { projectDir: tempDir, inputMode: 'auto' }, - io.io, - { - prompts: { select, cancel: vi.fn() }, - runContextBuild: runContextBuildMock, - verifyContextReady, - }, - ), - ).resolves.toMatchObject({ status: 'ready' }); - - expect(select).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('How much database context should KTX build?'), - }), - ); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.connections.warehouse.context).toMatchObject({ depth: 'deep' }); - }); - - it('requires completed relationships for deep context when relationship discovery is enabled', async () => { - await writeReadyProject(tempDir, { - connections: { - warehouse: { driver: 'postgres', readonly: true, context: { depth: 'deep' } }, + warehouse: { driver: 'postgres', readonly: true }, }, scan: { relationships: { enabled: true } }, }); @@ -553,17 +461,17 @@ describe('setup context build state', () => { runKtxSetupContextStep( { projectDir: tempDir, inputMode: 'disabled' }, io.io, - { runContextBuild: runContextBuildMock }, + { runContextBuild: runContextBuildMock, testConnection: async () => 0 }, ), ).resolves.toMatchObject({ status: 'ready' }); expect(runContextBuildMock).toHaveBeenCalledOnce(); }); - it('does not require relationships for deep context when relationship discovery is disabled', async () => { + it('does not require relationships for database context when relationship discovery is disabled', async () => { await writeReadyProject(tempDir, { connections: { - warehouse: { driver: 'postgres', readonly: true, context: { depth: 'deep' } }, + warehouse: { driver: 'postgres', readonly: true }, }, scan: { relationships: { enabled: false } }, }); @@ -620,7 +528,7 @@ describe('setup context build state', () => { it('starts a fresh foreground build when stale state is found', async () => { await writeReadyProject(tempDir, { - connections: { warehouse: { driver: 'postgres', readonly: true, context: { depth: 'fast' } } }, + connections: { warehouse: { driver: 'postgres', readonly: true } }, }); await writeKtxSetupContextState(tempDir, { runId: 'setup-context-local-stale', @@ -648,10 +556,119 @@ describe('setup context build state', () => { runKtxSetupContextStep( { projectDir: tempDir, inputMode: 'disabled' }, io.io, - { runContextBuild: runContextBuildMock, verifyContextReady }, + { runContextBuild: runContextBuildMock, verifyContextReady, testConnection: async () => 0 }, ), ).resolves.toMatchObject({ status: 'ready' }); expect(runContextBuildMock).toHaveBeenCalledOnce(); }); + + it('blocks the build and names the failing connection without leaking raw error text', async () => { + const missingDbPath = join(tempDir, 'missing-warehouse.sqlite'); + await writeReadyProject(tempDir, { + connections: { warehouse: { driver: 'sqlite', path: missingDbPath } }, + }); + const io = makeIo(); + const runContextBuildMock = vi.fn(async () => ({ exitCode: 0 })); + + await expect( + runKtxSetupContextStep( + { projectDir: tempDir, inputMode: 'disabled' }, + io.io, + { + runIdFactory: () => 'setup-context-local-gate', + now: () => new Date('2026-05-09T10:00:00.000Z'), + runContextBuild: runContextBuildMock, + }, + ), + ).resolves.toEqual({ status: 'failed', projectDir: tempDir }); + + expect(runContextBuildMock).not.toHaveBeenCalled(); + // Names the failing connection by id + connector type, with remediation. + expect(io.stderr()).toContain('warehouse (sqlite)'); + expect(io.stderr()).toContain('ktx connection test'); + // The remediation command targets the project that just failed, not cwd. + expect(io.stderr()).toContain(`ktx connection test --project-dir ${tempDir}`); + // Never surfaces raw connection error text (or the database path) to the user. + expect(io.stderr()).not.toContain('File not found'); + expect(io.stderr()).not.toContain(missingDbPath); + // The failed context state forces context.ready=false so setup cannot read as ready. + await expect(readKtxSetupContextState(tempDir)).resolves.toMatchObject({ + status: 'failed', + failureReason: 'Required connections failed their live test: warehouse (sqlite).', + }); + expect((await readKtxSetupState(tempDir)).completed_steps).not.toContain('context'); + }); + + it('retries connection tests after a fix and then builds in interactive mode', async () => { + await writeReadyProject(tempDir, { + connections: { warehouse: { driver: 'postgres', readonly: true } }, + }); + const io = makeIo(); + const runContextBuildMock = vi.fn(async () => ({ exitCode: 0 })); + const verifyContextReady = vi.fn(async () => ({ + ready: true, + agentContextReady: true, + semanticSearchReady: true, + details: ['ready'], + })); + let gateRounds = 0; + const testConnection = vi.fn(async () => (++gateRounds === 1 ? 1 : 0)); + let selectCalls = 0; + const select = vi.fn(async () => { + selectCalls += 1; + return selectCalls === 1 ? 'build' : 'retry'; + }); + + await expect( + runKtxSetupContextStep( + { projectDir: tempDir, inputMode: 'auto' }, + io.io, + { + prompts: { select, cancel: vi.fn() }, + runContextBuild: runContextBuildMock, + verifyContextReady, + testConnection, + }, + ), + ).resolves.toMatchObject({ status: 'ready' }); + + expect(testConnection).toHaveBeenCalledTimes(2); + expect(runContextBuildMock).toHaveBeenCalledOnce(); + expect(io.stderr()).toContain('warehouse (postgres)'); + }); + + it('returns to setup when the user backs out of a failing connection in interactive mode', async () => { + await writeReadyProject(tempDir, { + connections: { warehouse: { driver: 'postgres', readonly: true } }, + }); + const io = makeIo(); + const runContextBuildMock = vi.fn(async () => ({ exitCode: 0 })); + const verifyContextReady = vi.fn(async () => ({ + ready: true, + agentContextReady: true, + semanticSearchReady: true, + details: ['ready'], + })); + let selectCalls = 0; + const select = vi.fn(async () => { + selectCalls += 1; + return selectCalls === 1 ? 'build' : 'back'; + }); + + await expect( + runKtxSetupContextStep( + { projectDir: tempDir, inputMode: 'auto' }, + io.io, + { + prompts: { select, cancel: vi.fn() }, + runContextBuild: runContextBuildMock, + verifyContextReady, + testConnection: async () => 1, + }, + ), + ).resolves.toEqual({ status: 'back', projectDir: tempDir }); + + expect(runContextBuildMock).not.toHaveBeenCalled(); + }); }); diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/test/setup-databases.test.ts similarity index 71% rename from packages/cli/src/setup-databases.test.ts rename to packages/cli/test/setup-databases.test.ts index a9da0f51..957dfdb2 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/test/setup-databases.test.ts @@ -1,21 +1,23 @@ import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; -import { initKtxProject, loadKtxProject } from './context/project/project.js'; -import { parseKtxProjectConfig } from './context/project/config.js'; -import { readKtxSetupState, writeKtxSetupState } from './context/project/setup-config.js'; +import { initKtxProject, loadKtxProject } from '../src/context/project/project.js'; +import { parseKtxProjectConfig } from '../src/context/project/config.js'; +import { readKtxSetupState, writeKtxSetupState } from '../src/context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { + managedDaemonOptionsForSetupQueryHistoryPicker, type KtxSetupDatabaseDriver, type KtxSetupDatabasesDeps, type KtxSetupDatabasesPromptAdapter, runKtxSetupDatabasesStep, -} from './setup-databases.js'; -import type { KtxCliIo } from './cli-runtime.js'; +} from '../src/setup-databases.js'; +import type { KtxCliIo } from '../src/cli-runtime.js'; import type { DatabaseScopePickResult, PickDatabaseScopeArgs, -} from './database-tree-picker.js'; +} from '../src/database-tree-picker.js'; +import type { KtxSetupPromptOption } from '../src/setup-prompts.js'; function makeIo() { let stdout = ''; @@ -136,6 +138,22 @@ function textInputPrompt(message: string): string { return `${title}\n│\n│ ${bodyLines.join('\n│ ')}\n│ Press Escape to go back.\n│`; } +function queryHistoryFromConfig(connection: unknown): { + filters?: { serviceAccounts?: unknown; dropTrivialProbes?: boolean }; +} | undefined { + if (!connection || typeof connection !== 'object' || Array.isArray(connection)) { + return undefined; + } + const context = (connection as { context?: unknown }).context; + if (!context || typeof context !== 'object' || Array.isArray(context)) { + return undefined; + } + const queryHistory = (context as { queryHistory?: unknown }).queryHistory; + return queryHistory && typeof queryHistory === 'object' && !Array.isArray(queryHistory) + ? (queryHistory as { filters?: { serviceAccounts?: unknown; dropTrivialProbes?: boolean } }) + : undefined; +} + describe('setup databases step', () => { let tempDir: string; @@ -149,6 +167,61 @@ describe('setup databases step', () => { await rm(tempDir, { recursive: true, force: true }); }); + it('builds managed daemon options for setup query-history SQL analysis', () => { + const io = makeIo(); + + expect( + managedDaemonOptionsForSetupQueryHistoryPicker({ + projectDir: tempDir, + args: { + inputMode: 'disabled', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + }, + io: io.io, + }), + ).toEqual({ + cliVersion: '0.2.0', + projectDir: tempDir, + installPolicy: 'auto', + io: io.io, + }); + }); + + it('defaults managed daemon setup options when the database step is called directly', () => { + const io = makeIo(); + + expect( + managedDaemonOptionsForSetupQueryHistoryPicker({ + projectDir: tempDir, + args: { + inputMode: 'disabled', + }, + io: io.io, + }), + ).toMatchObject({ + cliVersion: expect.any(String), + projectDir: tempDir, + installPolicy: 'never', + io: io.io, + }); + + expect( + managedDaemonOptionsForSetupQueryHistoryPicker({ + projectDir: tempDir, + args: { + inputMode: 'auto', + }, + io: io.io, + }), + ).toMatchObject({ + cliVersion: expect.any(String), + projectDir: tempDir, + installPolicy: 'prompt', + io: io.io, + }); + }); + it('shows every supported database in the interactive checklist', async () => { const prompts = makePromptAdapter({ multiselectValues: [['back']] }); @@ -164,13 +237,13 @@ describe('setup databases step', () => { 'Which databases should KTX connect to?\n' + 'Use Up/Down to move, Space to select or unselect, Enter to confirm, Escape to go back, or Ctrl+C to exit.', options: [ - { value: 'sqlite', label: 'SQLite' }, { value: 'postgres', label: 'PostgreSQL' }, + { value: 'bigquery', label: 'BigQuery' }, + { value: 'snowflake', label: 'Snowflake' }, { value: 'mysql', label: 'MySQL' }, { value: 'clickhouse', label: 'ClickHouse' }, { value: 'sqlserver', label: 'SQL Server' }, - { value: 'bigquery', label: 'BigQuery' }, - { value: 'snowflake', label: 'Snowflake' }, + { value: 'sqlite', label: 'SQLite' }, ], required: true, }); @@ -261,48 +334,6 @@ describe('setup databases step', () => { expect(prompts.select).toHaveBeenCalledTimes(1); }); - it('preserves context.depth when editing an existing database connection', async () => { - await writeFile( - join(tempDir, 'ktx.yaml'), - [ - 'connections:', - ' warehouse:', - ' driver: sqlite', - ' path: ./warehouse.sqlite', - ' context:', - ' depth: deep', - '', - ].join('\n'), - 'utf-8', - ); - const prompts = makePromptAdapter({ - selectValues: ['edit', 'warehouse', 'continue'], - textValues: ['./warehouse.sqlite'], - }); - const testConnection = vi.fn(async () => 0); - const scanConnection = vi.fn(async () => 0); - const io = makeIo(); - const result = await runKtxSetupDatabasesStep( - { - projectDir: tempDir, - inputMode: 'auto', - skipDatabases: false, - databaseSchemas: [], - disableQueryHistory: true, - }, - io.io, - { prompts, testConnection, scanConnection }, - ); - - expect(result.status, io.stderr()).toBe('ready'); - const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); - expect(config.connections.warehouse).toMatchObject({ - driver: 'sqlite', - path: './warehouse.sqlite', - context: { depth: 'deep' }, - }); - }); - it('labels existing database connections with the database type', async () => { await writeFile( join(tempDir, 'ktx.yaml'), @@ -375,18 +406,21 @@ describe('setup databases step', () => { expect(config.connections['postgres-warehouse']).toEqual({ driver: 'postgres', url: 'env:DATABASE_URL', - context: { depth: 'fast' }, }); }); it('emits debug telemetry when setup writes a database connection', async () => { vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); + vi.stubEnv('KTX_TELEMETRY_DISABLED', ''); + vi.stubEnv('DO_NOT_TRACK', ''); vi.stubEnv('CI', ''); const io = makeIo(); const prompts = makePromptAdapter({ selectValues: ['url'], textValues: ['', 'env:DATABASE_URL'], }); + const listSchemas = vi.fn(async () => []); + const listTables = vi.fn(async () => []); const result = await runKtxSetupDatabasesStep( { @@ -397,7 +431,13 @@ describe('setup databases step', () => { skipDatabases: false, }, io.io, - { prompts, testConnection: vi.fn(async () => 0), scanConnection: vi.fn(async () => 0) }, + { + prompts, + testConnection: vi.fn(async () => 0), + scanConnection: vi.fn(async () => 0), + listSchemas, + listTables, + }, ); expect(result.status).toBe('ready'); @@ -695,6 +735,7 @@ describe('setup databases step', () => { message: 'Databases configured: warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to context sources' }, + { value: 'skip-sources', label: 'Skip context sources' }, { value: 'edit', label: 'Edit an existing database' }, { value: 'add', label: 'Add another database' }, ], @@ -703,6 +744,48 @@ describe('setup databases step', () => { expect(scanConnection).not.toHaveBeenCalled(); }); + it('can skip context sources from the configured database menu', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ selectValues: ['skip-sources'] }); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'auto', + skipDatabases: false, + databaseSchemas: [], + disableQueryHistory: true, + }, + makeIo().io, + { prompts, testConnection, scanConnection }, + ); + + expect(result).toEqual({ + status: 'ready', + projectDir: tempDir, + connectionIds: ['warehouse'], + skipSources: true, + }); + expect(testConnection).not.toHaveBeenCalled(); + expect(scanConnection).not.toHaveBeenCalled(); + }); + it('preserves existing database ids when adding another database from the configured menu', async () => { await writeFile( join(tempDir, 'ktx.yaml'), @@ -753,6 +836,7 @@ describe('setup databases step', () => { message: 'Databases configured: warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to context sources' }, + { value: 'skip-sources', label: 'Skip context sources' }, { value: 'edit', label: 'Edit an existing database' }, { value: 'add', label: 'Add another database' }, ], @@ -801,6 +885,7 @@ describe('setup databases step', () => { message: 'Databases configured: postgres-warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to context sources' }, + { value: 'skip-sources', label: 'Skip context sources' }, { value: 'edit', label: 'Edit an existing database' }, { value: 'add', label: 'Add another database' }, ], @@ -846,6 +931,7 @@ describe('setup databases step', () => { message: 'Databases configured: postgres-warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to context sources' }, + { value: 'skip-sources', label: 'Skip context sources' }, { value: 'edit', label: 'Edit an existing database' }, { value: 'add', label: 'Add another database' }, ], @@ -890,6 +976,7 @@ describe('setup databases step', () => { message: 'Databases configured: warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to context sources' }, + { value: 'skip-sources', label: 'Skip context sources' }, { value: 'edit', label: 'Edit an existing database' }, { value: 'add', label: 'Add another database' }, ], @@ -936,6 +1023,7 @@ describe('setup databases step', () => { message: 'Databases configured: warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to context sources' }, + { value: 'skip-sources', label: 'Skip context sources' }, { value: 'edit', label: 'Edit an existing database' }, { value: 'add', label: 'Add another database' }, ], @@ -981,7 +1069,7 @@ describe('setup databases step', () => { const testConnection = vi.fn(async () => 0); const scanConnection = vi.fn(async () => 0); const listSchemas = vi.fn(async () => ['analytics', 'public']); - const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'customers', kind: 'table' as const }]); + const listTables = vi.fn(async () => [{ catalog: null, schema: 'analytics', name: 'customers', kind: 'table' as const }]); const pickers = makePickerStubs({ scopes: [{ schemas: ['analytics'], tables: ['analytics.customers'] }], }); @@ -1052,9 +1140,9 @@ describe('setup databases step', () => { }); const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']); const listTables = vi.fn(async () => [ - { schema: 'public', name: 'customers', kind: 'table' as const }, - { schema: 'public', name: 'orders', kind: 'table' as const }, - { schema: 'public', name: 'products', kind: 'table' as const }, + { catalog: null, schema: 'public', name: 'customers', kind: 'table' as const }, + { catalog: null, schema: 'public', name: 'orders', kind: 'table' as const }, + { catalog: null, schema: 'public', name: 'products', kind: 'table' as const }, ]); const pickers = makePickerStubs({ scopes: [{ schemas: ['public'], tables: ['public.customers', 'public.orders'] }], @@ -1126,8 +1214,8 @@ describe('setup databases step', () => { const scanConnection = vi.fn(async () => 0); const listSchemas = vi.fn(async () => ['analytics', 'public']); const listTables = vi.fn(async () => [ - { schema: 'analytics', name: 'customers', kind: 'table' as const }, - { schema: 'public', name: 'orders', kind: 'table' as const }, + { catalog: null, schema: 'analytics', name: 'customers', kind: 'table' as const }, + { catalog: null, schema: 'public', name: 'orders', kind: 'table' as const }, ]); const pickers = makePickerStubs({ scopes: ['back'] }); @@ -1192,8 +1280,8 @@ describe('setup databases step', () => { const scanConnection = vi.fn(async () => 0); const listSchemas = vi.fn(async () => ['public']); const listTables = vi.fn(async () => [ - { schema: 'public', name: 'customers', kind: 'table' as const }, - { schema: 'public', name: 'orders', kind: 'table' as const }, + { catalog: null, schema: 'public', name: 'customers', kind: 'table' as const }, + { catalog: null, schema: 'public', name: 'orders', kind: 'table' as const }, ]); const pickers = makePickerStubs({ scopes: [{ schemas: ['public'], tables: 'back' }] }); @@ -1245,16 +1333,21 @@ describe('setup databases step', () => { const prompts = makePromptAdapter({ textValues: ['env:DATABASE_URL'], }); + let primaryMenuCount = 0; vi.mocked(prompts.select).mockImplementation(async (options) => { - if (options.message === 'Databases configured: warehouse\nWhat would you like to do?') return 'edit'; + if (options.message === 'Databases configured: warehouse\nWhat would you like to do?') { + primaryMenuCount += 1; + return primaryMenuCount === 1 ? 'edit' : 'continue'; + } if (options.message === 'Database to edit') return 'warehouse'; if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; if (options.message.startsWith('Enable query-history ingest')) return 'no'; + if (options.message === 'Connection setup failed for warehouse') return 'back'; return 'back'; }); const listTables = vi.fn(async () => [ - { schema: 'public', name: 'customers', kind: 'table' as const }, - { schema: 'public', name: 'orders', kind: 'table' as const }, + { catalog: null, schema: 'public', name: 'customers', kind: 'table' as const }, + { catalog: null, schema: 'public', name: 'orders', kind: 'table' as const }, ]); const pickers = makePickerStubs({ scopes: ['enable-all'] }); @@ -1270,13 +1363,283 @@ describe('setup databases step', () => { }, ); - expect(result).toEqual({ status: 'failed', projectDir: tempDir }); + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.connections.warehouse).toMatchObject({ enabled_tables: ['public.orders'], }); }); + it('recovers from an interactive database edit failure by re-entering details', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'connections:', + ' analytics:', + ' driver: postgres', + ' url: env:OLD_DATABASE_URL', + 'setup:', + ' database_connection_ids:', + ' - analytics', + '', + ].join('\n'), + 'utf-8', + ); + const io = makeIo(); + const prompts = makePromptAdapter({ + selectValues: ['edit', 'analytics', 'url', 'no', 're-enter', 'url', 'no', 'continue'], + textValues: ['env:BAD_DATABASE_URL', 'env:FIXED_DATABASE_URL'], + }); + let attempts = 0; + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'auto', + databaseSchemas: [], + skipDatabases: false, + }, + io.io, + { + prompts, + testConnection: vi.fn(async () => { + attempts += 1; + return attempts === 1 ? 1 : 0; + }), + scanConnection: vi.fn(async () => 0), + listSchemas: vi.fn(async () => ['public']), + listTables: vi.fn(async () => [{ catalog: null, schema: 'public', name: 'orders', kind: 'table' as const }]), + }, + ); + + expect(result.status).toBe('ready'); + expect(vi.mocked(prompts.select)).toHaveBeenCalledWith( + expect.objectContaining({ + message: 'Connection setup failed for analytics', + options: expect.arrayContaining([ + { value: 'retry', label: 'Retry connection test' }, + { value: 're-enter', label: 'Re-enter connection details' }, + { value: 'back', label: 'Back' }, + ]), + }), + ); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.analytics).toMatchObject({ + driver: 'postgres', + url: 'env:FIXED_DATABASE_URL', + }); + }); + + it('re-enters details after an interactive existing database validation failure', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:OLD_DATABASE_URL', + '', + ].join('\n'), + 'utf-8', + ); + const io = makeIo(); + const prompts = makePromptAdapter({ + selectValues: ['existing:warehouse', 'no', 're-enter', 'url', 'no'], + textValues: ['env:FIXED_DATABASE_URL'], + }); + let attempts = 0; + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'auto', + databaseDrivers: ['postgres'], + databaseSchemas: [], + skipDatabases: false, + }, + io.io, + { + prompts, + testConnection: vi.fn(async () => { + attempts += 1; + return attempts === 1 ? 1 : 0; + }), + scanConnection: vi.fn(async () => 0), + listSchemas: vi.fn(async () => ['public']), + listTables: vi.fn(async () => [ + { catalog: null, schema: 'public', name: 'orders', kind: 'table' as const }, + ]), + }, + ); + + expect(result.status).toBe('ready'); + expect(vi.mocked(prompts.select)).toHaveBeenCalledWith({ + message: 'How do you want to connect to PostgreSQL?', + options: [ + { value: 'url', label: 'Paste a connection URL' }, + { value: 'fields', label: 'Enter connection details (host, port, database, user)' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(vi.mocked(prompts.select)).toHaveBeenCalledWith( + expect.objectContaining({ + message: 'Connection setup failed for warehouse', + options: expect.arrayContaining([ + { value: 'retry', label: 'Retry connection test' }, + { value: 're-enter', label: 'Re-enter connection details' }, + { value: 'skip', label: 'Skip this connection' }, + { value: 'back', label: 'Back' }, + ]), + }), + ); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + driver: 'postgres', + url: 'env:FIXED_DATABASE_URL', + }); + }); + + it('restores the previous database config when backing out of a failed edit', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'connections:', + ' analytics:', + ' driver: postgres', + ' url: env:OLD_DATABASE_URL', + 'setup:', + ' database_connection_ids:', + ' - analytics', + '', + ].join('\n'), + 'utf-8', + ); + const io = makeIo(); + const prompts = makePromptAdapter({ + selectValues: ['edit', 'analytics', 'url', 'no', 'back', 'continue'], + textValues: ['env:BAD_DATABASE_URL'], + }); + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'auto', + databaseSchemas: [], + skipDatabases: false, + }, + io.io, + { + prompts, + testConnection: vi.fn(async () => 1), + scanConnection: vi.fn(async () => 0), + }, + ); + + expect(result.status).toBe('ready'); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.analytics).toMatchObject({ + driver: 'postgres', + url: 'env:OLD_DATABASE_URL', + }); + }); + + it('keeps scripted database setup fail-fast without rolling back attempted config', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'connections:', + ' analytics:', + ' driver: postgres', + ' url: env:OLD_DATABASE_URL', + '', + ].join('\n'), + 'utf-8', + ); + const io = makeIo(); + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'disabled', + databaseConnectionIds: ['analytics'], + databaseSchemas: [], + enableQueryHistory: true, + skipDatabases: false, + }, + io.io, + { + testConnection: vi.fn(async () => 1), + scanConnection: vi.fn(async () => 0), + }, + ); + + expect(result.status).toBe('failed'); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.analytics).toMatchObject({ + driver: 'postgres', + url: 'env:OLD_DATABASE_URL', + context: { + queryHistory: { + enabled: true, + }, + }, + }); + }); + + it('keeps scripted database ids fail-fast even when input mode is auto', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'connections:', + ' analytics:', + ' driver: postgres', + ' url: env:OLD_DATABASE_URL', + '', + ].join('\n'), + 'utf-8', + ); + const io = makeIo(); + const prompts = makePromptAdapter({}); + vi.mocked(prompts.select).mockImplementation(async ({ message }) => { + if (message === 'Connection setup failed for analytics') { + throw new Error('scripted selected-id setup opened the recovery menu'); + } + return 'finish'; + }); + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'auto', + databaseConnectionIds: ['analytics'], + databaseSchemas: [], + enableQueryHistory: true, + skipDatabases: false, + }, + io.io, + { + prompts, + testConnection: vi.fn(async () => 1), + scanConnection: vi.fn(async () => 0), + }, + ); + + expect(result.status).toBe('failed'); + expect(prompts.select).not.toHaveBeenCalledWith( + expect.objectContaining({ message: 'Connection setup failed for analytics' }), + ); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.analytics).toMatchObject({ + driver: 'postgres', + url: 'env:OLD_DATABASE_URL', + context: { + queryHistory: { + enabled: true, + }, + }, + }); + }); + it('lets Escape from connection fields return to connection method selection', async () => { const prompts = makePromptAdapter({ selectValues: ['fields', 'url'], @@ -1499,7 +1862,7 @@ describe('setup databases step', () => { ); expect(io.stdout()).not.toContain('Tables: 2'); expect(io.stdout()).toContain('◇ Building schema context for postgres-warehouse'); - expect(io.stdout()).toContain('│ Running fast database ingest…'); + expect(io.stdout()).toContain('│ Running database scan…'); expect(io.stdout()).toContain('◇ Schema context complete for postgres-warehouse'); expect(io.stdout()).toContain('│ Changes: 2 new tables'); expect(io.stdout()).toContain('◇ Database ready'); @@ -1552,7 +1915,7 @@ describe('setup databases step', () => { }); const listSchemas = vi.fn(async () => ['analytics', 'mart']); const listTables = vi.fn(async (_projectDir: string, _connectionId: string, schemas?: string[]) => - (schemas ?? []).map((schema) => ({ schema, name: 'orders', kind: 'table' as const })), + (schemas ?? []).map((schema) => ({ catalog: null, schema, name: 'orders', kind: 'table' as const })), ); const pickDatabaseScope = vi.fn(async (args: PickDatabaseScopeArgs) => { const scopedArgs = args as PickDatabaseScopeArgs & { @@ -1609,7 +1972,7 @@ describe('setup databases step', () => { textValues: ['bigquery-warehouse', '/tmp/service-account.json', 'US'], }); const listSchemas = vi.fn(async () => ['analytics']); - const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'orders', kind: 'table' as const }]); + const listTables = vi.fn(async () => [{ catalog: 'project-1', schema: 'analytics', name: 'orders', kind: 'table' as const }]); const pickDatabaseScope = vi.fn(async () => ({ kind: 'selected' as const, activeSchemas: ['analytics'], @@ -1642,9 +2005,9 @@ describe('setup databases step', () => { }); const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']); const listTables = vi.fn(async () => [ - { schema: 'orbit_analytics', name: 'events', kind: 'table' as const }, - { schema: 'orbit_raw', name: 'inputs', kind: 'table' as const }, - { schema: 'public', name: 'misc', kind: 'table' as const }, + { catalog: null, schema: 'orbit_analytics', name: 'events', kind: 'table' as const }, + { catalog: null, schema: 'orbit_raw', name: 'inputs', kind: 'table' as const }, + { catalog: null, schema: 'public', name: 'misc', kind: 'table' as const }, ]); const pickers = makePickerStubs({ scopes: [ @@ -1703,7 +2066,7 @@ describe('setup databases step', () => { throw new Error('permission denied to list schemas'); }); const listTables = vi.fn(async (_projectDir: string, _connectionId: string, schemas?: string[]) => - (schemas ?? []).map((schema) => ({ schema, name: 'events', kind: 'table' as const })), + (schemas ?? []).map((schema) => ({ catalog: null, schema, name: 'events', kind: 'table' as const })), ); const pickers = makePickerStubs({ scopes: [ @@ -1750,18 +2113,18 @@ describe('setup databases step', () => { it('passes schemas and a lazy table callback to the scope picker instead of eager table discovery', async () => { const listSchemas = vi.fn(async () => ['analytics', 'raw']); const listTables = vi.fn(async (_projectDir: string, _connectionId: string, schemas?: string[]) => - (schemas ?? []).map((schema) => ({ schema, name: 'orders', kind: 'table' as const })), + (schemas ?? []).map((schema) => ({ catalog: null, schema, name: 'orders', kind: 'table' as const })), ); const pickDatabaseScope = vi.fn(async (args: PickDatabaseScopeArgs) => { const lazyArgs = args as PickDatabaseScopeArgs & { schemas: string[]; - listTablesForSchemas: (schemas: string[]) => Promise>; + listTablesForSchemas: (schemas: string[]) => Promise>; }; expect(lazyArgs.schemas).toEqual(['analytics', 'raw']); expect(args).not.toHaveProperty('discovered'); expect(listTables).not.toHaveBeenCalled(); const tables = await lazyArgs.listTablesForSchemas(['analytics']); - expect(tables).toEqual([{ schema: 'analytics', name: 'orders', kind: 'table' }]); + expect(tables).toEqual([{ catalog: null, schema: 'analytics', name: 'orders', kind: 'table' }]); return { kind: 'selected' as const, activeSchemas: ['analytics'], enabledTables: ['analytics.orders'] }; }); @@ -1848,7 +2211,7 @@ describe('setup databases step', () => { driver: 'postgres', url: 'env:DATABASE_URL', schemas: ['public'], - context: { queryHistory: { enabled: false }, depth: 'fast' }, + context: { queryHistory: { enabled: false } }, }); expect(config.setup).toEqual({ database_connection_ids: ['warehouse'], @@ -1887,7 +2250,6 @@ describe('setup databases step', () => { expect(config.connections.warehouse).toEqual({ driver: 'sqlite', path: './warehouse.sqlite', - context: { depth: 'fast' }, }); expect(config.setup).toEqual({ database_connection_ids: ['warehouse'], @@ -1964,11 +2326,11 @@ describe('setup databases step', () => { const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.connections.warehouse).toMatchObject({ driver: 'postgres', url: 'env:DATABASE_URL' }); expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); - expect(io.stderr()).toContain('Fast database ingest failed for warehouse.'); - expect(io.stderr()).toContain('│ Fast database ingest failed for warehouse.'); - expect(io.stderr()).toContain(`Debug command: ktx ingest warehouse --project-dir ${tempDir} --fast --debug`); + expect(io.stderr()).toContain('Database scan failed for warehouse.'); + expect(io.stderr()).toContain('│ Database scan failed for warehouse.'); + expect(io.stderr()).toContain(`Debug command: ktx ingest warehouse --project-dir ${tempDir} --debug`); expect(io.stderr()).not.toContain('Structural scan failed for warehouse.'); - expect(io.stderr()).not.toMatch(/^Fast database ingest failed for warehouse\./m); + expect(io.stderr()).not.toMatch(/^Database scan failed for warehouse\./m); }); it('prints the native SQLite rebuild command when scanning hits a Node ABI mismatch', async () => { @@ -2007,7 +2369,7 @@ describe('setup databases step', () => { expect(io.stderr()).toContain('Native SQLite is built for a different Node.js ABI.'); expect(io.stderr()).toContain('│ Native SQLite is built for a different Node.js ABI.'); expect(io.stderr()).toContain('Fix: pnpm run native:rebuild'); - expect(io.stderr()).toContain(`Retry: ktx ingest warehouse --project-dir ${tempDir} --fast`); + expect(io.stderr()).toContain(`Retry: ktx ingest warehouse --project-dir ${tempDir}`); expect(io.stderr()).not.toContain('ktx scan'); expect(io.stderr()).not.toContain('npm rebuild'); expect(io.stderr()).not.toMatch(/^Native SQLite is built for a different Node.js ABI\./m); @@ -2068,9 +2430,40 @@ describe('setup databases step', () => { expect(io.stdout()).toContain('│ Changes: 0 changes across 56 tables'); }); + function fakeHistoricSqlRunner( + dialect: 'postgres' | 'snowflake' | 'bigquery', + catalogName: string, + ) { + return { + dialect, + catalogName, + async run() { + return { warnings: [], info: [] }; + }, + formatSuccessDetail() { + return { detail: `${catalogName} ready`, warnings: [] }; + }, + fixAdvice() { + return { + failHeadline: `${catalogName} unavailable`, + remediation: 'Fix query-history grants.', + }; + }, + }; + } + it('writes query history config for supported Snowflake databases after validation succeeds', async () => { const io = makeIo(); - const historicSqlProbe = vi.fn(async () => ({ ok: true, lines: [] })); + const runner = fakeHistoricSqlRunner( + 'snowflake', + 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY', + ); + const historicSqlReadinessProbe = vi.fn(async () => ({ + ok: true as const, + dialect: 'snowflake' as const, + runner, + result: { warnings: [], info: [] }, + })); const result = await runKtxSetupDatabasesStep( { projectDir: tempDir, @@ -2088,7 +2481,7 @@ describe('setup databases step', () => { { testConnection: vi.fn(async () => 0), scanConnection: vi.fn(async () => 0), - historicSqlProbe, + historicSqlReadinessProbe, prompts: makePromptAdapter({ selectValues: ['password'], textValues: ['env:SNOWFLAKE_ACCOUNT', 'WH', 'ANALYTICS', 'reader', ''], @@ -2096,11 +2489,11 @@ describe('setup databases step', () => { }), }, ); - expect(historicSqlProbe).toHaveBeenCalledWith( + expect(historicSqlReadinessProbe).toHaveBeenCalledWith( expect.objectContaining({ projectDir: tempDir, connectionId: 'snowflake', - dialect: 'snowflake', + connection: expect.objectContaining({ driver: 'snowflake' }), }), ); @@ -2197,7 +2590,15 @@ describe('setup databases step', () => { { testConnection: vi.fn(async () => 0), scanConnection: vi.fn(async () => 0), - historicSqlProbe: vi.fn(async () => ({ ok: true, lines: [' OK pg_stat_statements ready (PostgreSQL 16.4)'] })), + historicSqlReadinessProbe: vi.fn(async () => { + const runner = fakeHistoricSqlRunner('postgres', 'pg_stat_statements'); + return { + ok: true as const, + dialect: 'postgres' as const, + runner, + result: { pgServerVersion: 'PostgreSQL 16.4', warnings: [], info: [] }, + }; + }), }, ); @@ -2240,6 +2641,241 @@ describe('setup databases step', () => { expect(io.stdout()).toContain('pg_stat_statements ready'); }); + it('auto-applies derived query-history service-account filters in non-interactive setup', async () => { + const io = makeIo(); + const queryHistoryFilterPicker = vi.fn(async () => ({ + excludedRoles: [ + { + role: 'svc_loader', + pattern: '^svc_loader$', + reason: 'Runs recurring loader traffic against modeled tables.', + }, + ], + consideredRoleCount: 2, + skipped: null, + warnings: [], + parseFailedTemplateIds: [], + })); + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'disabled', + yes: true, + databaseDrivers: ['postgres'], + databaseConnectionId: 'warehouse', + databaseUrl: 'env:DATABASE_URL', + databaseSchemas: ['public'], + enableQueryHistory: true, + skipDatabases: false, + }, + io.io, + { + testConnection: vi.fn(async () => 0), + scanConnection: vi.fn(async () => 0), + historicSqlReadinessProbe: vi.fn(async () => { + const runner = fakeHistoricSqlRunner('postgres', 'pg_stat_statements'); + return { + ok: true as const, + dialect: 'postgres' as const, + runner, + result: { pgServerVersion: 'PostgreSQL 16.4', warnings: [], info: [] }, + }; + }), + queryHistoryFilterPicker, + createQueryHistoryLlmRuntime: vi.fn(() => null), + }, + ); + + expect(result.status).toBe('ready'); + expect(queryHistoryFilterPicker).toHaveBeenCalledTimes(1); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + context: { + queryHistory: { + filters: { + dropTrivialProbes: true, + serviceAccounts: { + mode: 'exclude', + patterns: ['^svc_loader$'], + }, + }, + }, + }, + }); + expect(io.stdout()).toContain('Proposed query-history service-account filters'); + expect(io.stdout()).toContain('svc_loader'); + }); + + it('collapses query-history parse failures to a count and lists ids only with --debug', async () => { + const io = makeIo(); + const queryHistoryFilterPicker = vi.fn(async () => ({ + excludedRoles: [], + consideredRoleCount: 1, + skipped: { reason: 'no-in-scope-history' as const }, + warnings: [], + parseFailedTemplateIds: ['111', '222'], + })); + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'disabled', + debug: true, + yes: true, + databaseDrivers: ['postgres'], + databaseConnectionId: 'warehouse', + databaseUrl: 'env:DATABASE_URL', + databaseSchemas: ['public'], + enableQueryHistory: true, + skipDatabases: false, + }, + io.io, + { + testConnection: vi.fn(async () => 0), + scanConnection: vi.fn(async () => 0), + historicSqlReadinessProbe: vi.fn(async () => { + const runner = fakeHistoricSqlRunner('postgres', 'pg_stat_statements'); + return { + ok: true as const, + dialect: 'postgres' as const, + runner, + result: { pgServerVersion: 'PostgreSQL 16.4', warnings: [], info: [] }, + }; + }), + queryHistoryFilterPicker, + createQueryHistoryLlmRuntime: vi.fn(() => null), + }, + ); + + expect(result.status).toBe('ready'); + expect(io.stdout()).toContain('Skipped 2 query templates ktx could not parse'); + expect(io.stdout()).not.toContain('111'); + expect(io.stdout()).not.toContain('222'); + expect(io.stderr()).toContain('could not parse 2 template(s): 111, 222'); + }); + + it('lets interactive setup skip applying derived filters', async () => { + const io = makeIo(); + const prompts = makePromptAdapter({ + selectValues: ['skip'], + }); + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'auto', + yes: false, + databaseDrivers: ['postgres'], + databaseConnectionId: 'warehouse', + databaseUrl: 'env:DATABASE_URL', + databaseSchemas: ['public'], + enableQueryHistory: true, + skipDatabases: false, + }, + io.io, + { + prompts, + testConnection: vi.fn(async () => 0), + scanConnection: vi.fn(async () => 0), + historicSqlReadinessProbe: vi.fn(async () => { + const runner = fakeHistoricSqlRunner('postgres', 'pg_stat_statements'); + return { + ok: true as const, + dialect: 'postgres' as const, + runner, + result: { pgServerVersion: 'PostgreSQL 16.4', warnings: [], info: [] }, + }; + }), + queryHistoryFilterPicker: vi.fn(async () => ({ + excludedRoles: [{ role: 'svc_loader', pattern: '^svc_loader$', reason: 'Loader traffic.' }], + consideredRoleCount: 2, + skipped: null, + warnings: [], + parseFailedTemplateIds: [], + })), + createQueryHistoryLlmRuntime: vi.fn(() => null), + }, + ); + + expect(result.status).toBe('ready'); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(queryHistoryFromConfig(config.connections.warehouse)?.filters).toEqual({ dropTrivialProbes: true }); + expect(prompts.select).toHaveBeenCalledWith({ + message: 'Apply 1 derived query-history service-account exclusion?', + options: [ + { value: 'apply', label: 'Apply derived filters (recommended)' }, + { value: 'skip', label: 'Leave query history filters unchanged' }, + ], + }); + }); + + it('does not overwrite an existing serviceAccounts block', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' context:', + ' queryHistory:', + ' enabled: true', + ' filters:', + ' dropTrivialProbes: true', + ' serviceAccounts:', + ' mode: exclude', + ' patterns:', + " - '^existing$'", + '', + ].join('\n'), + 'utf-8', + ); + + const io = makeIo(); + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'disabled', + yes: true, + databaseConnectionIds: ['warehouse'], + databaseSchemas: [], + enableQueryHistory: true, + skipDatabases: false, + }, + io.io, + { + testConnection: vi.fn(async () => 0), + scanConnection: vi.fn(async () => 0), + historicSqlReadinessProbe: vi.fn(async () => { + const runner = fakeHistoricSqlRunner('postgres', 'pg_stat_statements'); + return { + ok: true as const, + dialect: 'postgres' as const, + runner, + result: { pgServerVersion: 'PostgreSQL 16.4', warnings: [], info: [] }, + }; + }), + queryHistoryFilterPicker: vi.fn(async () => ({ + excludedRoles: [{ role: 'svc_loader', pattern: '^svc_loader$', reason: 'Loader traffic.' }], + consideredRoleCount: 2, + skipped: { reason: 'user-block-present' as const }, + warnings: [], + parseFailedTemplateIds: [], + })), + createQueryHistoryLlmRuntime: vi.fn(() => null), + }, + ); + + expect(result.status).toBe('ready'); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(queryHistoryFromConfig(config.connections.warehouse)?.filters?.serviceAccounts).toEqual({ + mode: 'exclude', + patterns: ['^existing$'], + }); + expect(io.stdout()).toContain('Existing query-history service-account filters left unchanged'); + }); + it('asks interactive Postgres setup whether to enable query history', async () => { await writeFile( join(tempDir, 'ktx.yaml'), @@ -2266,8 +2902,14 @@ describe('setup databases step', () => { 'utf-8', ); const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['yes', 'deep'] }); - const historicSqlProbe = vi.fn(async () => ({ ok: true, lines: [] })); + const prompts = makePromptAdapter({ selectValues: ['yes'] }); + const runner = fakeHistoricSqlRunner('postgres', 'pg_stat_statements'); + const historicSqlReadinessProbe = vi.fn(async () => ({ + ok: true as const, + dialect: 'postgres' as const, + runner, + result: { pgServerVersion: 'PostgreSQL 16.4', warnings: [], info: [] }, + })); const result = await runKtxSetupDatabasesStep( { @@ -2282,7 +2924,7 @@ describe('setup databases step', () => { prompts, testConnection: vi.fn(async () => 0), scanConnection: vi.fn(async () => 0), - historicSqlProbe, + historicSqlReadinessProbe, }, ); @@ -2295,17 +2937,13 @@ describe('setup databases step', () => { { value: 'back', label: 'Back' }, ], }); - expect(prompts.select).toHaveBeenNthCalledWith( - 2, + expect(historicSqlReadinessProbe).toHaveBeenCalledWith( expect.objectContaining({ - message: expect.stringContaining('How much database context should KTX build?'), + projectDir: tempDir, + connectionId: 'warehouse', + connection: expect.objectContaining({ driver: 'postgres' }), }), ); - expect(historicSqlProbe).toHaveBeenCalledWith({ - projectDir: tempDir, - connectionId: 'warehouse', - dialect: 'postgres', - }); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.connections.warehouse).toMatchObject({ context: { @@ -2314,7 +2952,6 @@ describe('setup databases step', () => { minExecutions: 5, filters: { dropTrivialProbes: true }, }, - depth: 'deep', }, }); }); @@ -2333,6 +2970,13 @@ describe('setup databases step', () => { 'utf-8', ); const io = makeIo(); + const runner = fakeHistoricSqlRunner('bigquery', 'INFORMATION_SCHEMA.JOBS_BY_PROJECT'); + const historicSqlReadinessProbe = vi.fn(async () => ({ + ok: true as const, + dialect: 'bigquery' as const, + runner, + result: { warnings: [], info: [] }, + })); const result = await runKtxSetupDatabasesStep( { @@ -2348,10 +2992,18 @@ describe('setup databases step', () => { { testConnection: vi.fn(async () => 0), scanConnection: vi.fn(async () => 0), + historicSqlReadinessProbe, }, ); expect(result.status).toBe('ready'); + expect(historicSqlReadinessProbe).toHaveBeenCalledWith( + expect.objectContaining({ + projectDir: tempDir, + connectionId: 'analytics', + connection: expect.objectContaining({ driver: 'bigquery' }), + }), + ); const configText = await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'); const config = parseKtxProjectConfig(configText); expect(config.connections.analytics).toMatchObject({ @@ -2372,6 +3024,146 @@ describe('setup databases step', () => { expect(config.ingest.adapters).toEqual([]); }); + it('prints a non-blocking BigQuery query history probe failure with the grants remediation', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'connections:', + ' analytics:', + ' driver: bigquery', + ' dataset_id: analytics', + ' credentials_json: env:BIGQUERY_CREDENTIALS_JSON', + '', + ].join('\n'), + 'utf-8', + ); + const io = makeIo(); + const runner = { + ...fakeHistoricSqlRunner('bigquery', 'INFORMATION_SCHEMA.JOBS_BY_PROJECT'), + fixAdvice: () => ({ + failHeadline: 'BigQuery principal cannot read INFORMATION_SCHEMA.JOBS_BY_PROJECT', + remediation: + 'Grant roles/bigquery.resourceViewer on the BigQuery project, or grant a custom role containing bigquery.jobs.listAll.', + }), + }; + const error = new Error('access denied'); + const historicSqlReadinessProbe = vi.fn(async () => ({ + ok: false as const, + dialect: 'bigquery' as const, + runner, + error, + })); + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'disabled', + databaseConnectionIds: ['analytics'], + databaseSchemas: [], + enableQueryHistory: true, + queryHistoryWindowDays: 45, + skipDatabases: false, + }, + io.io, + { + testConnection: vi.fn(async () => 0), + scanConnection: vi.fn(async () => 0), + historicSqlReadinessProbe, + }, + ); + + expect(result.status).toBe('ready'); + expect(historicSqlReadinessProbe).toHaveBeenCalledWith( + expect.objectContaining({ + projectDir: tempDir, + connectionId: 'analytics', + connection: expect.objectContaining({ driver: 'bigquery' }), + }), + ); + expect(io.stdout()).toContain('Query history probe...'); + expect(io.stdout()).toContain( + 'BigQuery principal cannot read INFORMATION_SCHEMA.JOBS_BY_PROJECT', + ); + expect(io.stdout()).toContain('roles/bigquery.resourceViewer'); + expect(io.stdout()).toContain('bigquery.jobs.listAll'); + expect(io.stdout()).toContain('Setup written; query history will be skipped until fixed.'); + }); + + it('lets interactive BigQuery setup disable unavailable query history and retry after scan failure', async () => { + const io = makeIo(); + const failurePromptOptions: KtxSetupPromptOption[][] = []; + let failurePromptCount = 0; + const prompts = makePromptAdapter({ + textValues: ['/tmp/service-account.json', 'US'], + }); + vi.mocked(prompts.select).mockImplementation(async ({ message, options }) => { + if (message.startsWith('Enable query-history ingest')) return 'yes'; + if (message.includes('How much database context should KTX build?')) return 'fast'; + if (message.startsWith('Connection setup failed for analytics')) { + failurePromptCount += 1; + failurePromptOptions.push(options); + if (failurePromptCount === 1) return 'disable-query-history'; + throw new Error('setup did not disable query history before retrying'); + } + throw new Error(`unexpected select prompt: ${message}`); + }); + const runner = { + ...fakeHistoricSqlRunner('bigquery', 'INFORMATION_SCHEMA.JOBS_BY_PROJECT'), + fixAdvice: () => ({ + failHeadline: 'BigQuery principal cannot read INFORMATION_SCHEMA.JOBS_BY_PROJECT', + remediation: + 'Grant roles/bigquery.resourceViewer on the BigQuery project, or grant a custom role containing bigquery.jobs.listAll.', + }), + }; + const historicSqlReadinessProbe = vi.fn(async () => ({ + ok: false as const, + dialect: 'bigquery' as const, + runner, + error: new Error('access denied'), + })); + let scanAttempts = 0; + const scanConnection = vi.fn(async () => { + scanAttempts += 1; + return scanAttempts === 1 ? 1 : 0; + }); + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'auto', + databaseDrivers: ['bigquery'], + databaseConnectionId: 'analytics', + databaseSchemas: [], + skipDatabases: false, + }, + io.io, + { + prompts, + testConnection: vi.fn(async () => 0), + scanConnection, + historicSqlReadinessProbe, + listSchemas: vi.fn(async () => ['analytics']), + listTables: vi.fn(async () => [{ catalog: null, schema: 'analytics', name: 'orders', kind: 'table' as const }]), + }, + ); + + expect(result.status).toBe('ready'); + expect(scanConnection).toHaveBeenCalledTimes(2); + expect(historicSqlReadinessProbe).toHaveBeenCalledTimes(1); + expect(failurePromptOptions[0]).toContainEqual({ + value: 'disable-query-history', + label: 'Disable query history and retry', + }); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.analytics).toMatchObject({ + context: { + queryHistory: { + enabled: false, + }, + }, + }); + }); + it('enables query history on an existing Postgres connection', async () => { await writeFile( join(tempDir, 'ktx.yaml'), @@ -2400,7 +3192,15 @@ describe('setup databases step', () => { { testConnection: vi.fn(async () => 0), scanConnection: vi.fn(async () => 0), - historicSqlProbe: vi.fn(async () => ({ ok: true, lines: [' OK pg_stat_statements ready (PostgreSQL 16.4)'] })), + historicSqlReadinessProbe: vi.fn(async () => { + const runner = fakeHistoricSqlRunner('postgres', 'pg_stat_statements'); + return { + ok: true as const, + dialect: 'postgres' as const, + runner, + result: { pgServerVersion: 'PostgreSQL 16.4', warnings: [], info: [] }, + }; + }), }, ); @@ -2417,17 +3217,104 @@ describe('setup databases step', () => { }, }, }); + expect(config.connections.warehouse.historicSql).toBeUndefined(); + }); + + it('migrates legacy historicSql to context.queryHistory during database setup', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'connections:', + ' warehouse:', + ' driver: postgres', + ' readonly: true', + ' historicSql:', + ' enabled: true', + ' dialect: postgres', + ' windowDays: 45', + ' minExecutions: 9', + ' concurrency: 3', + ' staleArchiveAfterDays: 120', + ' filters:', + ' dropTrivialProbes: true', + ' serviceAccounts:', + ' mode: exclude', + ' patterns:', + " - '^svc_'", + ' orchestrators:', + ' mode: exclude', + ' patterns:', + ' - airflow', + ' dropFailedBelow: 2', + ' redactionPatterns:', + " - '(?i)secret'", + '', + ].join('\n'), + 'utf-8', + ); + + const io = makeIo(); + + await expect( + runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'disabled', + databaseConnectionIds: ['warehouse'], + databaseSchemas: [], + skipDatabases: false, + }, + io.io, + { + testConnection: vi.fn(async () => 0), + scanConnection: vi.fn(async () => 0), + historicSqlReadinessProbe: vi.fn(async () => { + const runner = fakeHistoricSqlRunner('postgres', 'pg_stat_statements'); + return { + ok: true as const, + dialect: 'postgres' as const, + runner, + result: { pgServerVersion: 'PostgreSQL 16.4', warnings: [], info: [] }, + }; + }), + }, + ), + ).resolves.toMatchObject({ status: 'ready' }); + + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse.historicSql).toBeUndefined(); + expect(config.connections.warehouse.context).toMatchObject({ + queryHistory: { + enabled: true, + windowDays: 45, + minExecutions: 9, + concurrency: 3, + staleArchiveAfterDays: 120, + filters: { + dropTrivialProbes: true, + serviceAccounts: { mode: 'exclude', patterns: ['^svc_'] }, + orchestrators: { mode: 'exclude', patterns: ['airflow'] }, + dropFailedBelow: 2, + }, + redactionPatterns: ['(?i)secret'], + }, + }); }); it('prints a non-blocking Postgres query history probe failure after connection test succeeds', async () => { const io = makeIo(); - const historicSqlProbe = vi.fn(async () => ({ - ok: false, - lines: [ - ' FAIL pg_stat_statements extension is not installed in the connection database', - ' Fix: Run (against this database): CREATE EXTENSION pg_stat_statements;', - " Fix: Ensure shared_preload_libraries includes 'pg_stat_statements'.", - ], + const runner = { + ...fakeHistoricSqlRunner('postgres', 'pg_stat_statements'), + fixAdvice: () => ({ + failHeadline: 'pg_stat_statements extension is not installed in the connection database', + remediation: 'Run (against this database): CREATE EXTENSION pg_stat_statements;', + }), + }; + const historicSqlReadinessProbe = vi.fn(async () => ({ + ok: false as const, + dialect: 'postgres' as const, + runner, + error: new Error('missing extension'), })); const result = await runKtxSetupDatabasesStep( @@ -2445,16 +3332,16 @@ describe('setup databases step', () => { { testConnection: vi.fn(async () => 0), scanConnection: vi.fn(async () => 0), - historicSqlProbe, + historicSqlReadinessProbe, }, ); expect(result.status).toBe('ready'); - expect(historicSqlProbe).toHaveBeenCalledWith( + expect(historicSqlReadinessProbe).toHaveBeenCalledWith( expect.objectContaining({ projectDir: tempDir, connectionId: 'warehouse', - dialect: 'postgres', + connection: expect.objectContaining({ driver: 'postgres' }), }), ); expect(io.stdout()).toContain('Query history probe...'); @@ -2465,12 +3352,19 @@ describe('setup databases step', () => { it('prints a non-blocking Snowflake query history probe failure with the grants remediation', async () => { const io = makeIo(); - const historicSqlProbe = vi.fn(async () => ({ - ok: false, - lines: [ - ' FAIL Snowflake role cannot read SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY', - ' Fix: Run (as ACCOUNTADMIN): GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE ;', - ], + const runner = { + ...fakeHistoricSqlRunner('snowflake', 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY'), + fixAdvice: () => ({ + failHeadline: 'Snowflake role cannot read SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY', + remediation: + 'Run (as ACCOUNTADMIN): GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE ;', + }), + }; + const historicSqlReadinessProbe = vi.fn(async () => ({ + ok: false as const, + dialect: 'snowflake' as const, + runner, + error: new Error('role cannot read SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY'), })); const result = await runKtxSetupDatabasesStep( @@ -2487,7 +3381,7 @@ describe('setup databases step', () => { { testConnection: vi.fn(async () => 0), scanConnection: vi.fn(async () => 0), - historicSqlProbe, + historicSqlReadinessProbe, prompts: makePromptAdapter({ textValues: ['env:SNOWFLAKE_ACCOUNT', 'WH', 'ANALYTICS', 'reader', ''], passwordValues: ['env:SNOWFLAKE_PASSWORD'], @@ -2496,11 +3390,11 @@ describe('setup databases step', () => { ); expect(result.status).toBe('ready'); - expect(historicSqlProbe).toHaveBeenCalledWith( + expect(historicSqlReadinessProbe).toHaveBeenCalledWith( expect.objectContaining({ projectDir: tempDir, connectionId: 'warehouse', - dialect: 'snowflake', + connection: expect.objectContaining({ driver: 'snowflake' }), }), ); expect(io.stdout()).toContain('Query history probe...'); @@ -2511,7 +3405,15 @@ describe('setup databases step', () => { it('does not run the query history probe when the regular connection test fails', async () => { const io = makeIo(); - const historicSqlProbe = vi.fn(async () => ({ ok: true, lines: [] })); + const historicSqlReadinessProbe = vi.fn(async () => { + const runner = fakeHistoricSqlRunner('postgres', 'pg_stat_statements'); + return { + ok: true as const, + dialect: 'postgres' as const, + runner, + result: { pgServerVersion: 'PostgreSQL 16.4', warnings: [], info: [] }, + }; + }); const result = await runKtxSetupDatabasesStep( { @@ -2528,12 +3430,12 @@ describe('setup databases step', () => { { testConnection: vi.fn(async () => 1), scanConnection: vi.fn(async () => 0), - historicSqlProbe, + historicSqlReadinessProbe, }, ); expect(result.status).toBe('failed'); - expect(historicSqlProbe).not.toHaveBeenCalled(); + expect(historicSqlReadinessProbe).not.toHaveBeenCalled(); }); it('returns missing input when non-interactive database flags are incomplete', async () => { @@ -2554,6 +3456,25 @@ describe('setup databases step', () => { expect(io.stderr()).toContain('Missing database connection id'); }); + it('returns missing input when a non-interactive new connection is missing required details', async () => { + const io = makeIo(); + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'disabled', + databaseDrivers: ['postgres'], + databaseConnectionId: 'warehouse', + databaseSchemas: [], + skipDatabases: false, + }, + io.io, + ); + + expect(result.status).toBe('missing-input'); + expect(io.stderr()).toContain('Missing connection details'); + }); + it('accepts former ingest subcommand names as non-interactive database connection ids', async () => { const io = makeIo(); diff --git a/packages/cli/src/setup-demo-tour.test.ts b/packages/cli/test/setup-demo-tour.test.ts similarity index 98% rename from packages/cli/src/setup-demo-tour.test.ts rename to packages/cli/test/setup-demo-tour.test.ts index 1d57b010..3916076c 100644 --- a/packages/cli/src/setup-demo-tour.test.ts +++ b/packages/cli/test/setup-demo-tour.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import type { KtxSetupAgentsResult } from './setup-agents.js'; +import type { KtxSetupAgentsResult } from '../src/setup-agents.js'; import { buildDemoReplayTimeline, DEMO_REPLAY_TARGETS, @@ -8,7 +8,7 @@ import { renderDemoCardContent, renderDemoCompletionSummary, runDemoTour, -} from './setup-demo-tour.js'; +} from '../src/setup-demo-tour.js'; /** Strip ANSI escape sequences for plain-text assertions. */ function stripAnsi(text: string): string { diff --git a/packages/cli/src/setup-embeddings.test.ts b/packages/cli/test/setup-embeddings.test.ts similarity index 92% rename from packages/cli/src/setup-embeddings.test.ts rename to packages/cli/test/setup-embeddings.test.ts index bf9e2b2d..9af9f913 100644 --- a/packages/cli/src/setup-embeddings.test.ts +++ b/packages/cli/test/setup-embeddings.test.ts @@ -1,11 +1,12 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject } from './context/project/project.js'; -import { parseKtxProjectConfig } from './context/project/config.js'; -import { readKtxSetupState, writeKtxSetupState } from './context/project/setup-config.js'; +import { initKtxProject } from '../src/context/project/project.js'; +import { parseKtxProjectConfig } from '../src/context/project/config.js'; +import { readKtxSetupState, writeKtxSetupState } from '../src/context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { type KtxSetupEmbeddingsPromptAdapter, runKtxSetupEmbeddingsStep } from './setup-embeddings.js'; +import { ManagedPythonDaemonStartError } from '../src/managed-python-daemon.js'; +import { type KtxSetupEmbeddingsPromptAdapter, runKtxSetupEmbeddingsStep } from '../src/setup-embeddings.js'; const EMBEDDING_OPTION_PROMPT_MESSAGE = [ 'Which embedding option should KTX use?', @@ -366,6 +367,40 @@ describe('setup embeddings step', () => { expect(io.stderr()).not.toContain('daemon traceback line 5'); }); + it('prints the daemon stderr tail when the daemon fails to start', async () => { + const io = makeIo(); + const stderrLog = join(tempDir, '.ktx', 'runtime', 'daemon.stderr.log'); + await mkdir(join(tempDir, '.ktx', 'runtime'), { recursive: true }); + await writeFile( + stderrLog, + Array.from({ length: 45 }, (_value, index) => `daemon startup traceback ${index + 1}`).join('\n'), + ); + + const result = await runKtxSetupEmbeddingsStep( + { + projectDir: tempDir, + inputMode: 'disabled', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + skipEmbeddings: false, + }, + io.io, + { + env: {}, + ensureLocalEmbeddings: vi.fn(async () => { + throw new ManagedPythonDaemonStartError('fetch failed: connect ECONNREFUSED 127.0.0.1:61234', stderrLog); + }), + }, + ); + + expect(result.status).toBe('failed'); + expect(io.stderr()).toContain('Local embedding health check failed: fetch failed: connect ECONNREFUSED'); + expect(io.stderr()).toContain('Recent KTX daemon stderr:'); + expect(io.stderr()).toContain('daemon startup traceback 6'); + expect(io.stderr()).toContain('daemon startup traceback 45'); + expect(io.stderr()).not.toContain('daemon startup traceback 5'); + }); + it('does not print daemon stderr diagnostics when the log is unavailable or empty', async () => { const io = makeIo(); diff --git a/packages/cli/src/setup-interrupt.test.ts b/packages/cli/test/setup-interrupt.test.ts similarity index 99% rename from packages/cli/src/setup-interrupt.test.ts rename to packages/cli/test/setup-interrupt.test.ts index 62917db6..a1ff6b10 100644 --- a/packages/cli/src/setup-interrupt.test.ts +++ b/packages/cli/test/setup-interrupt.test.ts @@ -4,7 +4,7 @@ import { KtxSetupExitError, withSetupInterruptConfirmation, type SetupInterruptTracker, -} from './setup-interrupt.js'; +} from '../src/setup-interrupt.js'; const CANCEL = Symbol('cancel'); diff --git a/packages/cli/src/setup-models.test.ts b/packages/cli/test/setup-models.test.ts similarity index 93% rename from packages/cli/src/setup-models.test.ts rename to packages/cli/test/setup-models.test.ts index 444c3b4d..dedf03bd 100644 --- a/packages/cli/src/setup-models.test.ts +++ b/packages/cli/test/setup-models.test.ts @@ -1,16 +1,16 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject } from './context/project/project.js'; -import { parseKtxProjectConfig } from './context/project/config.js'; -import { readKtxSetupState, writeKtxSetupState } from './context/project/setup-config.js'; +import { initKtxProject } from '../src/context/project/project.js'; +import { parseKtxProjectConfig } from '../src/context/project/config.js'; +import { readKtxSetupState, writeKtxSetupState } from '../src/context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { BUNDLED_ANTHROPIC_MODELS, fetchAnthropicModels, type KtxSetupModelPromptAdapter, runKtxSetupAnthropicModelStep, -} from './setup-models.js'; +} from '../src/setup-models.js'; function makeIo() { let stdout = ''; @@ -66,6 +66,7 @@ function makePromptAdapter(options: { nextProviderChoice === 'anthropic' || nextProviderChoice === 'vertex' || nextProviderChoice === 'claude-code' || + nextProviderChoice === 'codex' || nextProviderChoice === 'back' ) { return selectValues.shift() ?? nextProviderChoice; @@ -183,6 +184,7 @@ describe('setup Anthropic model step', () => { message: expect.stringContaining('Which LLM provider should KTX use?'), options: [ { value: 'claude-code', label: 'Claude subscription (Pro/Max)' }, + { value: 'codex', label: 'Codex subscription' }, { value: 'anthropic', label: 'Anthropic API key' }, { value: 'vertex', label: 'Google Vertex AI for Anthropic Claude' }, { value: 'back', label: 'Back' }, @@ -215,6 +217,85 @@ describe('setup Anthropic model step', () => { expect(authProbe).toHaveBeenCalledWith(expect.objectContaining({ projectDir: tempDir, model: 'sonnet' })); }); + it('configures Codex backend and validates local auth', async () => { + const io = makeIo(); + const codexAuthProbe = vi.fn(async () => ({ ok: true as const })); + + const result = await runKtxSetupAnthropicModelStep( + { + projectDir: tempDir, + inputMode: 'disabled', + llmBackend: 'codex', + llmModel: 'gpt-5.5', + skipLlm: false, + }, + io.io, + { codexAuthProbe }, + ); + + expect(result.status).toBe('ready'); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.llm).toMatchObject({ + provider: { backend: 'codex' }, + models: { default: 'gpt-5.5' }, + }); + expect(codexAuthProbe).toHaveBeenCalledWith(expect.objectContaining({ projectDir: tempDir, model: 'gpt-5.5' })); + // The warning carries the clack gutter so it renders inside the setup frame. + expect(io.stderr()).toContain('│ Codex backend isolation is limited'); + expect(io.stderr()).toContain('may still load user Codex config'); + }); + + it('defaults the Codex model to gpt-5.5 when none is provided non-interactively', async () => { + const io = makeIo(); + const codexAuthProbe = vi.fn(async () => ({ ok: true as const })); + + const result = await runKtxSetupAnthropicModelStep( + { + projectDir: tempDir, + inputMode: 'disabled', + llmBackend: 'codex', + skipLlm: false, + }, + io.io, + { codexAuthProbe }, + ); + + expect(result.status).toBe('ready'); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.llm).toMatchObject({ + provider: { backend: 'codex' }, + models: { default: 'gpt-5.5' }, + }); + expect(codexAuthProbe).toHaveBeenCalledWith(expect.objectContaining({ projectDir: tempDir, model: 'gpt-5.5' })); + }); + + it('offers the curated Codex models during interactive setup', async () => { + const io = makeIo(); + const prompts = makePromptAdapter({ selectValues: ['codex', 'gpt-5.5'] }); + const codexAuthProbe = vi.fn(async () => ({ ok: true as const })); + + const result = await runKtxSetupAnthropicModelStep( + { projectDir: tempDir, inputMode: 'auto', skipLlm: false }, + io.io, + { prompts, codexAuthProbe }, + ); + + expect(result.status).toBe('ready'); + expect(prompts.select).toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining('Which Codex model should KTX use?'), + options: [ + { value: 'gpt-5.5', label: 'GPT-5.5', hint: 'recommended' }, + { value: 'gpt-5.4', label: 'GPT-5.4' }, + { value: 'gpt-5.4-mini', label: 'GPT-5.4 mini' }, + { value: 'manual', label: 'Enter a Codex model ID manually' }, + { value: 'back', label: 'Back' }, + ], + }), + ); + expect(codexAuthProbe).toHaveBeenCalledWith(expect.objectContaining({ model: 'gpt-5.5' })); + }); + it('prompts for the Claude Code model during interactive setup', async () => { const io = makeIo(); const prompts = makePromptAdapter({ selectValues: ['claude-code', 'opus'] }); diff --git a/packages/cli/src/setup-project.test.ts b/packages/cli/test/setup-project.test.ts similarity index 98% rename from packages/cli/src/setup-project.test.ts rename to packages/cli/test/setup-project.test.ts index 89663bbf..c77a2080 100644 --- a/packages/cli/src/setup-project.test.ts +++ b/packages/cli/test/setup-project.test.ts @@ -1,10 +1,10 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { readKtxSetupState } from './context/project/setup-config.js'; +import { readKtxSetupState } from '../src/context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { gray } from './io/symbols.js'; -import { type KtxSetupProjectPromptAdapter, runKtxSetupProjectStep } from './setup-project.js'; +import { gray } from '../src/io/symbols.js'; +import { type KtxSetupProjectPromptAdapter, runKtxSetupProjectStep } from '../src/setup-project.js'; function makeIo(options: { stdoutIsTty?: boolean } = {}) { let stdout = ''; diff --git a/packages/cli/src/setup-prompts.test.ts b/packages/cli/test/setup-prompts.test.ts similarity index 94% rename from packages/cli/src/setup-prompts.test.ts rename to packages/cli/test/setup-prompts.test.ts index 95f4b68b..8e83c558 100644 --- a/packages/cli/src/setup-prompts.test.ts +++ b/packages/cli/test/setup-prompts.test.ts @@ -2,7 +2,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { createKtxSetupPromptAdapter, type KtxSetupPromptOption, -} from './setup-prompts.js'; +} from '../src/setup-prompts.js'; const mocks = vi.hoisted(() => { const cancelSymbol = Symbol('cancel'); @@ -17,7 +17,7 @@ const mocks = vi.hoisted(() => { autocomplete: vi.fn(), autocompleteMultiselect: vi.fn(), note: vi.fn(), - password: vi.fn(), + revealPassword: vi.fn(), select: vi.fn(), text: vi.fn(), withSetupInterruptConfirmation: vi.fn((prompt: () => Promise) => prompt()), @@ -34,12 +34,15 @@ vi.mock('@clack/prompts', () => ({ autocomplete: mocks.autocomplete, autocompleteMultiselect: mocks.autocompleteMultiselect, note: mocks.note, - password: mocks.password, select: mocks.select, text: mocks.text, })); -vi.mock('./setup-interrupt.js', () => ({ +vi.mock('../src/reveal-password-prompt.js', () => ({ + revealPassword: mocks.revealPassword, +})); + +vi.mock('../src/setup-interrupt.js', () => ({ withSetupInterruptConfirmation: mocks.withSetupInterruptConfirmation, })); @@ -54,7 +57,7 @@ describe('setup prompt adapter', () => { mocks.autocomplete.mockReset(); mocks.autocompleteMultiselect.mockReset(); mocks.note.mockReset(); - mocks.password.mockReset(); + mocks.revealPassword.mockReset(); mocks.select.mockReset(); mocks.text.mockReset(); mocks.withSetupInterruptConfirmation.mockClear(); @@ -96,7 +99,7 @@ describe('setup prompt adapter', () => { it('decorates text and password prompts with setup navigation copy', async () => { mocks.text.mockResolvedValueOnce('analytics-ktx'); - mocks.password.mockResolvedValueOnce('secret'); + mocks.revealPassword.mockResolvedValueOnce('secret'); const adapter = createKtxSetupPromptAdapter({ selectCancelValue: 'back' }); await expect(adapter.text({ message: 'Project folder path', placeholder: './analytics-ktx' })).resolves.toBe( @@ -108,7 +111,7 @@ describe('setup prompt adapter', () => { message: 'Project folder path\n│ Press Escape to go back.\n│', placeholder: './analytics-ktx', }); - expect(mocks.password).toHaveBeenCalledWith({ + expect(mocks.revealPassword).toHaveBeenCalledWith({ message: 'Anthropic API key\n│ Press Escape to go back.\n│', }); }); @@ -213,7 +216,7 @@ describe('setup prompt adapter', () => { }); it('keeps setup intro and note plain for non-stream output', async () => { - const { createKtxSetupUiAdapter } = await import('./setup-prompts.js'); + const { createKtxSetupUiAdapter } = await import('../src/setup-prompts.js'); const chunks: string[] = []; const io = { stdout: { @@ -235,7 +238,7 @@ describe('setup prompt adapter', () => { }); it('uses Clack intro and note for writable TTY output', async () => { - const { createKtxSetupUiAdapter } = await import('./setup-prompts.js'); + const { createKtxSetupUiAdapter } = await import('../src/setup-prompts.js'); const output = { columns: 80, isTTY: true, diff --git a/packages/cli/test/setup-ready-menu.test.ts b/packages/cli/test/setup-ready-menu.test.ts new file mode 100644 index 00000000..39c62a32 --- /dev/null +++ b/packages/cli/test/setup-ready-menu.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + classifyKtxSetupCompletion, + runKtxSetupReadyChangeMenu, + runKtxSetupReadyMenu, +} from '../src/setup-ready-menu.js'; +import type { KtxSetupStatus } from '../src/setup.js'; + +const readyStatus: KtxSetupStatus = { + project: { path: '/tmp/revenue', ready: true }, + llm: { backend: 'anthropic', ready: true, model: 'claude-sonnet-4-6' }, + embeddings: { backend: 'openai', ready: true, model: 'text-embedding-3-small', dimensions: 1536 }, + databases: [{ connectionId: 'warehouse', ready: true }], + sources: [], + runtime: { required: false, ready: true, features: [] }, + context: { ready: true, status: 'completed' }, + agents: [{ target: 'codex', scope: 'project', ready: true }], +}; + +describe('classifyKtxSetupCompletion', () => { + it('reports ready only when config, context, and agents are all ready', () => { + expect(classifyKtxSetupCompletion(readyStatus)).toBe('ready'); + }); + + it('reports needs-agents when config and context are ready but no agent is installed', () => { + expect(classifyKtxSetupCompletion({ ...readyStatus, agents: [] })).toBe('needs-agents'); + }); + + it('reports needs-context when config is ready but context is not built', () => { + expect( + classifyKtxSetupCompletion({ ...readyStatus, context: { ready: false, status: 'not_started' } }), + ).toBe('needs-context'); + }); + + it('reports incomplete when a required config section is not ready', () => { + expect(classifyKtxSetupCompletion({ ...readyStatus, embeddings: { ready: false } })).toBe('incomplete'); + expect( + classifyKtxSetupCompletion({ ...readyStatus, runtime: { required: true, ready: false, features: ['core'] } }), + ).toBe('incomplete'); + }); + + it('reports incomplete when no context targets are configured', () => { + expect(classifyKtxSetupCompletion({ ...readyStatus, databases: [], sources: [] })).toBe('incomplete'); + }); +}); + +describe('runKtxSetupReadyMenu', () => { + it('exits when the user is done', async () => { + const prompts = { select: vi.fn(async () => 'done'), cancel: vi.fn() }; + + await expect(runKtxSetupReadyMenu(readyStatus, { prompts })).resolves.toEqual({ action: 'exit' }); + + expect(prompts.select).toHaveBeenCalledTimes(1); + expect(prompts.select).toHaveBeenCalledWith({ + message: 'Anything else?', + options: [ + { value: 'done', label: "Done — I'll start using ktx" }, + { value: 'change', label: 'Change a setting' }, + ], + }); + }); + + it('opens the section menu when the user chooses to change a setting', async () => { + const select = vi.fn().mockResolvedValueOnce('change').mockResolvedValueOnce('models'); + const prompts = { select, cancel: vi.fn() }; + + await expect(runKtxSetupReadyMenu(readyStatus, { prompts })).resolves.toEqual({ action: 'models' }); + + expect(select).toHaveBeenCalledTimes(2); + expect(select).toHaveBeenLastCalledWith({ + message: 'What would you like to change?', + options: [ + { value: 'models', label: 'Models' }, + { value: 'embeddings', label: 'Embeddings' }, + { value: 'databases', label: 'Databases' }, + { value: 'sources', label: 'Context sources' }, + { value: 'context', label: 'Rebuild KTX context' }, + { value: 'agents', label: 'Agent integration' }, + { value: 'exit', label: 'Exit' }, + ], + }); + }); +}); + +describe('runKtxSetupReadyChangeMenu', () => { + it('maps ready-project menu choices to setup sections', async () => { + const prompts = { select: vi.fn(async () => 'agents'), cancel: vi.fn() }; + + await expect(runKtxSetupReadyChangeMenu(readyStatus, { prompts })).resolves.toEqual({ action: 'agents' }); + + expect(prompts.select).toHaveBeenCalledWith({ + message: 'What would you like to change?', + options: [ + { value: 'models', label: 'Models' }, + { value: 'embeddings', label: 'Embeddings' }, + { value: 'databases', label: 'Databases' }, + { value: 'sources', label: 'Context sources' }, + { value: 'context', label: 'Rebuild KTX context' }, + { value: 'agents', label: 'Agent integration' }, + { value: 'exit', label: 'Exit' }, + ], + }); + }); + + it('includes the runtime option only when the runtime is required', async () => { + const prompts = { select: vi.fn(async () => 'runtime'), cancel: vi.fn() }; + + await runKtxSetupReadyChangeMenu( + { ...readyStatus, runtime: { required: true, ready: true, features: ['core'] } }, + { prompts }, + ); + + expect(prompts.select).toHaveBeenCalledWith( + expect.objectContaining({ + options: expect.arrayContaining([{ value: 'runtime', label: 'Runtime' }]), + }), + ); + }); +}); diff --git a/packages/cli/src/setup-runtime.test.ts b/packages/cli/test/setup-runtime.test.ts similarity index 95% rename from packages/cli/src/setup-runtime.test.ts rename to packages/cli/test/setup-runtime.test.ts index 2fb1f1f2..ab5777e4 100644 --- a/packages/cli/src/setup-runtime.test.ts +++ b/packages/cli/test/setup-runtime.test.ts @@ -1,10 +1,10 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; -import { readKtxSetupState } from './context/project/setup-config.js'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from '../src/context/project/config.js'; +import { readKtxSetupState } from '../src/context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { runKtxSetupRuntimeStep } from './setup-runtime.js'; +import { runKtxSetupRuntimeStep } from '../src/setup-runtime.js'; function makeIo() { let stdout = ''; diff --git a/packages/cli/src/setup-secrets.test.ts b/packages/cli/test/setup-secrets.test.ts similarity index 97% rename from packages/cli/src/setup-secrets.test.ts rename to packages/cli/test/setup-secrets.test.ts index 16589db0..67a288f3 100644 --- a/packages/cli/src/setup-secrets.test.ts +++ b/packages/cli/test/setup-secrets.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { envCredentialReference, writeProjectLocalSecretReference } from './setup-secrets.js'; +import { envCredentialReference, writeProjectLocalSecretReference } from '../src/setup-secrets.js'; describe('setup secrets', () => { let tempDir: string; diff --git a/packages/cli/src/setup-sources-notion.test.ts b/packages/cli/test/setup-sources-notion.test.ts similarity index 90% rename from packages/cli/src/setup-sources-notion.test.ts rename to packages/cli/test/setup-sources-notion.test.ts index 1306b07b..ce9210c1 100644 --- a/packages/cli/src/setup-sources-notion.test.ts +++ b/packages/cli/test/setup-sources-notion.test.ts @@ -1,13 +1,13 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject } from './context/project/project.js'; -import { type KtxProjectConnectionConfig, parseKtxProjectConfig, serializeKtxProjectConfig } from './context/project/config.js'; +import { initKtxProject } from '../src/context/project/project.js'; +import { type KtxProjectConnectionConfig, parseKtxProjectConfig, serializeKtxProjectConfig } from '../src/context/project/config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runKtxSetupSourcesStep, type KtxSetupSourcesPromptAdapter, -} from './setup-sources.js'; +} from '../src/setup-sources.js'; const notionMocks = vi.hoisted(() => ({ tokens: [] as string[], @@ -15,8 +15,8 @@ const notionMocks = vi.hoisted(() => ({ retrievePage: vi.fn(async () => ({ id: 'page-1' })), })); -vi.mock('./context/ingest/adapters/notion/notion-client.js', async (importOriginal) => { - const actual = await importOriginal(); +vi.mock('../src/context/ingest/adapters/notion/notion-client.js', async (importOriginal) => { + const actual = await importOriginal(); return { ...actual, NotionClient: vi.fn().mockImplementation(function NotionClient(token: string) { diff --git a/packages/cli/src/setup-sources.test.ts b/packages/cli/test/setup-sources.test.ts similarity index 87% rename from packages/cli/src/setup-sources.test.ts rename to packages/cli/test/setup-sources.test.ts index c0b2c781..ef18a1b6 100644 --- a/packages/cli/src/setup-sources.test.ts +++ b/packages/cli/test/setup-sources.test.ts @@ -1,17 +1,17 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject } from './context/project/project.js'; -import { type KtxProjectConnectionConfig, parseKtxProjectConfig, serializeKtxProjectConfig } from './context/project/config.js'; -import { readKtxSetupState } from './context/project/setup-config.js'; +import { initKtxProject } from '../src/context/project/project.js'; +import { type KtxProjectConnectionConfig, parseKtxProjectConfig, serializeKtxProjectConfig } from '../src/context/project/config.js'; +import { readKtxSetupState } from '../src/context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { KtxCliIo } from './cli-runtime.js'; +import type { KtxCliIo } from '../src/cli-runtime.js'; import { runKtxSetupSourcesStep, type KtxSetupSourcesDeps, type KtxSetupSourcesPromptAdapter, type KtxSetupSourceType, -} from './setup-sources.js'; +} from '../src/setup-sources.js'; function makeIo() { let stdout = ''; @@ -260,7 +260,7 @@ describe('setup sources step', () => { inputMode: 'disabled', source: 'notion', sourceConnectionId: 'notion-main', - sourceApiKeyRef: 'env:NOTION_TOKEN', // pragma: allowlist secret + sourceAuthTokenRef: 'env:NOTION_TOKEN', // pragma: allowlist secret notionCrawlMode: 'selected_roots', notionRootPageIds: ['page-1'], runInitialSourceIngest: false, @@ -281,6 +281,81 @@ describe('setup sources step', () => { expect((await readConfig()).connections['notion-main']?.last_successful_cursor).toBeUndefined(); }); + it('rejects --source-api-key-ref for Notion and points at --source-auth-token-ref', async () => { + await addPrimarySource(); + const io = makeIo(); + + await expect( + runKtxSetupSourcesStep( + { + projectDir, + inputMode: 'disabled', + source: 'notion', + sourceConnectionId: 'notion-main', + sourceApiKeyRef: 'env:NOTION_TOKEN', // pragma: allowlist secret + notionCrawlMode: 'selected_roots', + notionRootPageIds: ['page-1'], + runInitialSourceIngest: false, + skipSources: false, + }, + io.io, + {}, + ), + ).resolves.toEqual({ status: 'failed', projectDir }); + + expect(io.stderr()).toContain('--source-api-key-ref does not apply to --source notion; use --source-auth-token-ref.'); + expect((await readConfig()).connections['notion-main']).toBeUndefined(); + }); + + it('rejects --source-auth-token-ref for Metabase and points at --source-api-key-ref', async () => { + await addPrimarySource(); + const io = makeIo(); + + await expect( + runKtxSetupSourcesStep( + { + projectDir, + inputMode: 'disabled', + source: 'metabase', + sourceConnectionId: 'prod_metabase', + sourceUrl: 'https://metabase.example.com', + sourceAuthTokenRef: 'env:METABASE_API_KEY', // pragma: allowlist secret + sourceWarehouseConnectionId: 'warehouse', + metabaseDatabaseId: 1, + runInitialSourceIngest: false, + skipSources: false, + }, + io.io, + {}, + ), + ).resolves.toEqual({ status: 'failed', projectDir }); + + expect(io.stderr()).toContain('--source-auth-token-ref does not apply to --source metabase; use --source-api-key-ref.'); + }); + + it('rejects --source-client-secret-ref for dbt and points at --source-auth-token-ref', async () => { + await addPrimarySource(); + const io = makeIo(); + + await expect( + runKtxSetupSourcesStep( + { + projectDir, + inputMode: 'disabled', + source: 'dbt', + sourceConnectionId: 'dbt-main', + sourceClientSecretRef: 'env:DBT_SECRET', // pragma: allowlist secret + runInitialSourceIngest: false, + skipSources: false, + }, + io.io, + {}, + ), + ).resolves.toEqual({ status: 'failed', projectDir }); + + expect(io.stderr()).toContain('--source-client-secret-ref does not apply to --source dbt; use --source-auth-token-ref.'); + }); + it('accepts former ingest subcommand names as interactive source connection ids', async () => { await addPrimarySource(); const io = makeIo(); @@ -323,7 +398,7 @@ describe('setup sources step', () => { inputMode: 'disabled', source: 'notion', sourceConnectionId: 'notion-main', - sourceApiKeyRef: 'env:NOTION_TOKEN', // pragma: allowlist secret + sourceAuthTokenRef: 'env:NOTION_TOKEN', // pragma: allowlist secret notionCrawlMode: 'all_accessible', notionRootPageIds: ['page-1'], runInitialSourceIngest: false, @@ -372,8 +447,8 @@ describe('setup sources step', () => { expect(testPrompts.select).toHaveBeenCalledWith({ message: 'Which Notion pages should KTX ingest?', options: [ - { value: 'selected_roots', label: 'Specific pages and their subpages (choose them in a picker)' }, { value: 'all_accessible', label: 'All pages the integration can access' }, + { value: 'selected_roots', label: 'Specific pages and their subpages (choose them in a picker)' }, { value: 'back', label: 'Back' }, ], }); @@ -631,7 +706,18 @@ describe('setup sources step', () => { ); expect(io.stderr()).toContain('1: Metabase database does not match KTX connection database'); expect(io.stderr()).not.toContain('Metabase mapping validation failed'); - expect(testPrompts.log).toHaveBeenCalledWith('Edit the connection or pick a different source to continue.'); + expect(testPrompts.log).toHaveBeenCalledWith('Validating Metabase mapping...'); + expect(testPrompts.select).toHaveBeenCalledWith( + expect.objectContaining({ + message: 'Connection setup failed for metabase-main', + options: expect.arrayContaining([ + { value: 'retry', label: 'Retry connection test' }, + { value: 're-enter', label: 'Re-enter connection details' }, + { value: 'skip', label: 'Skip this connection' }, + { value: 'back', label: 'Back' }, + ]), + }), + ); }); it('does not mark sources complete when validation fails', async () => { @@ -805,8 +891,8 @@ describe('setup sources step', () => { expect(testPrompts.select).toHaveBeenCalledWith({ message: 'This repo requires authentication.', options: [ - { value: 'env', label: 'Use GITHUB_TOKEN from the environment' }, { value: 'paste', label: 'Paste a token and save it as a local secret file' }, + { value: 'env', label: 'Use GITHUB_TOKEN from the environment' }, { value: 'skip', label: 'Skip — try without authentication' }, { value: 'back', label: 'Back' }, ], @@ -886,7 +972,153 @@ describe('setup sources step', () => { expect(result.status).not.toBe('failed'); expect(io.stderr()).toContain('Failed to clone https://github.com/acme/private-repo: Authentication failed'); - expect(testPrompts.log).toHaveBeenCalledWith('Edit the connection or pick a different source to continue.'); + expect(testPrompts.select).toHaveBeenCalledWith( + expect.objectContaining({ + message: 'Connection setup failed for dbt-main', + options: expect.arrayContaining([ + { value: 'retry', label: 'Retry connection test' }, + { value: 're-enter', label: 'Re-enter connection details' }, + { value: 'skip', label: 'Skip this connection' }, + { value: 'back', label: 'Back' }, + ]), + }), + ); + }); + + it('recovers from an existing context-source validation failure by re-entering details', async () => { + await addPrimarySource(); + await addConnection('dbt-main', { + driver: 'dbt', + source_dir: '/repo/bad-dbt', + project_name: 'analytics', + }); + let attempts = 0; + const validateDbt = vi.fn(async () => { + attempts += 1; + return attempts === 1 + ? { ok: false as const, message: 'dbt project not found' } + : { ok: true as const, detail: 'project=analytics' }; + }); + const testPrompts = prompts({ + multiselect: [['dbt']], + select: ['existing:dbt-main', 're-enter', 'path', 'done'], + text: ['/repo/fixed-dbt', ''], + }); + const io = makeIo(); + + const result = await runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + io.io, + { prompts: testPrompts, validateDbt }, + ); + + expect(result.status).toBe('ready'); + expect(validateDbt).toHaveBeenCalledTimes(2); + expect(vi.mocked(testPrompts.select)).toHaveBeenCalledWith( + expect.objectContaining({ + message: 'Connection setup failed for dbt-main', + options: expect.arrayContaining([ + { value: 'retry', label: 'Retry connection test' }, + { value: 're-enter', label: 'Re-enter connection details' }, + { value: 'skip', label: 'Skip this connection' }, + { value: 'back', label: 'Back' }, + ]), + }), + ); + expect((await readConfig()).connections['dbt-main']).toMatchObject({ + driver: 'dbt', + source_dir: '/repo/fixed-dbt', + }); + }); + + it('restores a context-source edit and adapter enablement when recovery goes back', async () => { + await addPrimarySource(); + await addConnection('dbt-main', { + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + const testPrompts = prompts({ + multiselect: [['dbt']], + select: ['edit:dbt-main', 'path', 'back'], + text: ['/repo/bad-dbt', ''], + }); + const io = makeIo(); + + const result = await runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + io.io, + { + prompts: testPrompts, + validateDbt: vi.fn(async () => ({ ok: false as const, message: 'dbt project not found' })), + }, + ); + + expect(result.status).toBe('skipped'); + const config = await readConfig(); + expect(config.connections['dbt-main']).toMatchObject({ + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + expect(config.ingest.adapters).not.toContain('dbt'); + }); + + it('lets Metabase mapping failure retry through source recovery', async () => { + await addPrimarySource(); + let mappingAttempts = 0; + const runMapping = vi.fn(async () => { + mappingAttempts += 1; + return mappingAttempts === 1 ? 1 : 0; + }); + const testPrompts = prompts({ + multiselect: [['metabase']], + select: ['env', 'retry', 'done'], + text: ['metabase-main', 'https://metabase.example.com'], + }); + const io = makeIo(); + + const result = await runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + io.io, + { + prompts: testPrompts, + discoverMetabaseDatabases: vi.fn(async () => [ + { id: 1, name: 'Analytics', engine: 'postgres', host: 'db.example.com', dbName: 'analytics' }, + ]), + runMapping, + }, + ); + + expect(result.status).toBe('ready'); + expect(runMapping).toHaveBeenCalledTimes(2); + }); + + it('keeps noninteractive source setup fail-fast without rolling back attempted config', async () => { + await addPrimarySource(); + const io = makeIo(); + + const result = await runKtxSetupSourcesStep( + { + projectDir, + inputMode: 'disabled', + source: 'lookml', + sourceConnectionId: 'looker-repo', + sourceGitUrl: 'https://github.com/acme/lookml.git', + runInitialSourceIngest: false, + skipSources: false, + }, + io.io, + { + validateLookml: vi.fn(async () => ({ ok: false as const, message: 'No LookML files found' })), + }, + ); + + expect(result.status).toBe('failed'); + expect((await readConfig()).connections['looker-repo']).toMatchObject({ + driver: 'lookml', + repoUrl: 'https://github.com/acme/lookml.git', + }); }); it('adds a dbt source connection and enables its adapter', async () => { @@ -1175,8 +1407,8 @@ describe('setup sources step', () => { message: 'How should KTX find your Notion integration token?', options: [ { value: 'keep', label: 'Keep existing credential' }, - { value: 'env', label: 'Use NOTION_TOKEN from the environment' }, { value: 'paste', label: 'Paste a key and save it as a local secret file' }, + { value: 'env', label: 'Use NOTION_TOKEN from the environment' }, { value: 'back', label: 'Back' }, ], }); @@ -1244,8 +1476,8 @@ describe('setup sources step', () => { message: 'How should KTX find your Metabase API key?', options: [ { value: 'keep', label: 'Keep existing credential' }, - { value: 'env', label: 'Use METABASE_API_KEY from the environment' }, { value: 'paste', label: 'Paste a key and save it as a local secret file' }, + { value: 'env', label: 'Use METABASE_API_KEY from the environment' }, { value: 'back', label: 'Back' }, ], }); @@ -1296,7 +1528,17 @@ describe('setup sources step', () => { source_dir: '/repo/new-dbt', })); expect(io.stderr()).toContain('dbt project not found'); - expect(testPrompts.log).toHaveBeenCalledWith('Edit the connection or pick a different source to continue.'); + expect(testPrompts.select).toHaveBeenCalledWith( + expect.objectContaining({ + message: 'Connection setup failed for dbt-main', + options: expect.arrayContaining([ + { value: 'retry', label: 'Retry connection test' }, + { value: 're-enter', label: 'Re-enter connection details' }, + { value: 'skip', label: 'Skip this connection' }, + { value: 'back', label: 'Back' }, + ]), + }), + ); const config = await readConfig(); expect(config.connections['dbt-main']).toMatchObject({ driver: 'dbt', @@ -1340,8 +1582,8 @@ describe('setup sources step', () => { message: 'This MetricFlow repo requires authentication.', options: [ { value: 'keep', label: 'Keep existing credential' }, - { value: 'env', label: 'Use GITHUB_TOKEN from the environment' }, { value: 'paste', label: 'Paste a token and save it as a local secret file' }, + { value: 'env', label: 'Use GITHUB_TOKEN from the environment' }, { value: 'skip', label: 'Skip — try without authentication' }, { value: 'back', label: 'Back' }, ], @@ -1385,7 +1627,7 @@ describe('setup sources step', () => { expect(testPrompts.select).toHaveBeenCalledWith({ message: '1 context source configured (dbt-main). Add another?', options: [ - { value: 'done', label: 'Done — continue to context build' }, + { value: 'done', label: 'Done adding context sources' }, { value: 'edit', label: 'Edit an existing context source' }, { value: 'add', label: 'Add another context source' }, ], diff --git a/packages/cli/src/setup.test.ts b/packages/cli/test/setup.test.ts similarity index 85% rename from packages/cli/src/setup.test.ts rename to packages/cli/test/setup.test.ts index ff8513b7..9b8bf689 100644 --- a/packages/cli/src/setup.test.ts +++ b/packages/cli/test/setup.test.ts @@ -1,17 +1,17 @@ import { execFile } from 'node:child_process'; -import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; -import { writeKtxSetupState } from './context/project/setup-config.js'; +import { writeKtxSetupState } from '../src/context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { localFakeBundleReport, persistLocalBundleReport } from './ingest.test-utils.js'; -import { contextBuildCommands, writeKtxSetupContextState } from './setup-context.js'; -import { runDemoTour } from './setup-demo-tour.js'; -import { formatKtxSetupCompletionSummary, formatKtxSetupStatus, readKtxSetupStatus, runKtxSetup } from './setup.js'; +import { contextBuildCommands, writeKtxSetupContextState } from '../src/setup-context.js'; +import { runDemoTour } from '../src/setup-demo-tour.js'; +import { formatKtxSetupCompletionSummary, formatKtxSetupStatus, readKtxSetupStatus, runKtxSetup } from '../src/setup.js'; -vi.mock('./setup-demo-tour.js', () => ({ +vi.mock('../src/setup-demo-tour.js', () => ({ runDemoTour: vi.fn(async () => 0), })); @@ -398,6 +398,59 @@ describe('setup status', () => { expect(rendered).toContain('KTX context built: yes'); }); + it('reports context ready after a partial ingest report saved memory', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'setup:', + ' database_connection_ids:', + ' - warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + 'ingest:', + ' embeddings:', + ' backend: none', + ' dimensions: 8', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['project', 'databases'] }); + await persistLocalBundleReport( + tempDir, + localFakeBundleReport('warehouse-job-partial', { + connectionId: 'warehouse', + sourceKey: 'fake', + body: { + failedWorkUnits: ['orders-bad'], + workUnits: [ + { + unitKey: 'orders-ok', + rawFiles: ['orders/orders.json'], + status: 'success', + actions: [{ target: 'wiki', type: 'created', key: 'wiki/orders.md', detail: 'orders' }], + touchedSlSources: [], + }, + { + unitKey: 'orders-bad', + rawFiles: ['orders/bad.json'], + status: 'failed', + reason: 'writer tool failed', + actions: [], + touchedSlSources: [], + }, + ], + }, + }), + ); + + const status = await readKtxSetupStatus(tempDir); + + expect(status.context).toMatchObject({ ready: true, status: 'completed' }); + }); + it('formats plain and JSON setup status payloads', async () => { const status = await readKtxSetupStatus(tempDir); const rendered = formatKtxSetupStatus(status); @@ -602,7 +655,7 @@ describe('setup status', () => { expect(testIo.stderr()).toBe(''); }); - it('removes a newly created missing project directory when a later runtime step fails', async () => { + it('preserves a newly created missing project directory when a later setup step fails', async () => { const projectDir = join(tempDir, 'missing-project'); const testIo = makeIo(); @@ -634,10 +687,12 @@ describe('setup status', () => { ), ).resolves.toBe(1); - await expect(stat(projectDir)).rejects.toThrow(); + await expect(stat(projectDir)).resolves.toBeDefined(); + await expect(stat(join(projectDir, 'ktx.yaml'))).resolves.toBeDefined(); + await expect(stat(join(projectDir, '.ktx'))).resolves.toBeDefined(); }); - it('removes KTX scaffold files from an initially empty project directory when runtime setup fails', async () => { + it('preserves KTX scaffold files in an initially empty project directory when setup fails', async () => { const testIo = makeIo(); await expect( @@ -668,8 +723,59 @@ describe('setup status', () => { ), ).resolves.toBe(1); - await expect(stat(tempDir)).resolves.toBeDefined(); - expect(await readdir(tempDir)).toEqual([]); + await expect(stat(join(tempDir, 'ktx.yaml'))).resolves.toBeDefined(); + await expect(stat(join(tempDir, '.ktx'))).resolves.toBeDefined(); + }); + + it('preserves partial context-build artifacts and resume state when the context step fails', async () => { + const projectDir = join(tempDir, 'partial-context'); + const testIo = makeIo(); + + await expect( + runKtxSetup( + { + command: 'run', + projectDir, + mode: 'auto', + agents: false, + skipAgents: true, + inputMode: 'disabled', + yes: true, + cliVersion: '0.2.0', + skipLlm: true, + skipEmbeddings: true, + databaseSchemas: [], + skipDatabases: true, + skipSources: true, + }, + testIo.io, + { + model: async () => ({ status: 'skipped', projectDir }), + embeddings: async () => ({ status: 'skipped', projectDir }), + databases: async () => ({ status: 'skipped', projectDir }), + sources: async () => ({ status: 'skipped', projectDir }), + runtime: async () => runtimeReady(projectDir), + context: async () => { + await mkdir(join(projectDir, '.ktx', 'setup'), { recursive: true }); + await writeFile( + join(projectDir, '.ktx', 'setup', 'state.json'), + JSON.stringify({ status: 'failed', retryableFailedTargets: [{ source: 'metabase' }] }), + 'utf-8', + ); + await mkdir(join(projectDir, 'wiki'), { recursive: true }); + await writeFile(join(projectDir, 'wiki', 'postgres-warehouse.md'), '# warehouse\n', 'utf-8'); + await mkdir(join(projectDir, 'semantic-layer'), { recursive: true }); + await writeFile(join(projectDir, 'semantic-layer', 'orders.yaml'), 'name: orders\n', 'utf-8'); + return { status: 'failed', projectDir }; + }, + }, + ), + ).resolves.toBe(1); + + await expect(stat(join(projectDir, 'ktx.yaml'))).resolves.toBeDefined(); + await expect(readFile(join(projectDir, '.ktx', 'setup', 'state.json'), 'utf-8')).resolves.toContain('"status":"failed"'); + await expect(readFile(join(projectDir, 'wiki', 'postgres-warehouse.md'), 'utf-8')).resolves.toContain('warehouse'); + await expect(readFile(join(projectDir, 'semantic-layer', 'orders.yaml'), 'utf-8')).resolves.toContain('orders'); }); it('preserves a pre-existing non-empty project directory when runtime setup fails', async () => { @@ -1578,6 +1684,9 @@ describe('setup status', () => { expect.objectContaining({ projectDir: tempDir, inputMode: 'disabled', + yes: true, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', databaseDrivers: ['postgres'], databaseConnectionId: 'warehouse', databaseUrl: 'env:DATABASE_URL', @@ -1641,6 +1750,67 @@ describe('setup status', () => { expect(calls).toEqual(['model', 'embeddings', 'databases', 'sources']); }); + it('passes context-source skip selection from database setup into the sources step', async () => { + const calls: string[] = []; + const io = makeIo(); + await writeFile(join(tempDir, 'ktx.yaml'), ['connections: {}', ''].join('\n'), 'utf-8'); + + await expect( + runKtxSetup( + { + command: 'run', + projectDir: tempDir, + mode: 'auto', + agents: false, + skipAgents: true, + inputMode: 'disabled', + yes: true, + cliVersion: '0.2.0', + skipLlm: true, + skipEmbeddings: true, + skipDatabases: false, + skipSources: false, + databaseSchemas: [], + }, + io.io, + { + model: async () => { + calls.push('model'); + return { status: 'skipped', projectDir: tempDir }; + }, + embeddings: async () => { + calls.push('embeddings'); + return { status: 'skipped', projectDir: tempDir }; + }, + databases: async () => { + calls.push('databases'); + return { + status: 'ready', + projectDir: tempDir, + connectionIds: ['warehouse'], + skipSources: true, + }; + }, + sources: async (args) => { + expect(args.skipSources).toBe(true); + calls.push('sources'); + return { status: 'skipped', projectDir: tempDir }; + }, + runtime: async () => { + calls.push('runtime'); + return runtimeReady(tempDir); + }, + context: async () => { + calls.push('context'); + return { status: 'ready', projectDir: tempDir, runId: 'setup-context-local-test' }; + }, + }, + ), + ).resolves.toBe(0); + + expect(calls).toEqual(['model', 'embeddings', 'databases', 'sources', 'runtime', 'context']); + }); + it.each([ { backend: 'vertex', @@ -2038,8 +2208,11 @@ describe('setup status', () => { join(tempDir, 'ktx.yaml'), [ 'setup:', - ' database_connection_ids: []', - 'connections: {}', + ' database_connection_ids: [warehouse]', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', 'llm:', ' provider:', ' backend: anthropic', @@ -2055,7 +2228,7 @@ describe('setup status', () => { 'utf-8', ); await writeKtxSetupState(tempDir, { - completed_steps: ['project', 'llm', 'embeddings', 'sources', 'runtime', 'context', 'agents'], + completed_steps: ['project', 'llm', 'embeddings', 'databases', 'sources', 'runtime', 'context', 'agents'], }); await writeFile( join(tempDir, '.ktx/agents/install-manifest.json'), @@ -2108,7 +2281,12 @@ describe('setup status', () => { }, io.io, { - readyMenuDeps: { prompts: { select: vi.fn(async () => 'agents'), cancel: vi.fn() } }, + readyMenuDeps: { + prompts: { + select: vi.fn().mockResolvedValueOnce('change').mockResolvedValueOnce('agents'), + cancel: vi.fn(), + }, + }, model: async (args) => { expect(args.skipLlm).toBe(true); return { status: 'skipped', projectDir: tempDir }; @@ -2158,8 +2336,11 @@ describe('setup status', () => { join(tempDir, 'ktx.yaml'), [ 'setup:', - ' database_connection_ids: []', - 'connections: {}', + ' database_connection_ids: [warehouse]', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', 'llm:', ' provider:', ' backend: anthropic', @@ -2175,7 +2356,7 @@ describe('setup status', () => { 'utf-8', ); await writeKtxSetupState(tempDir, { - completed_steps: ['project', 'llm', 'embeddings', 'sources', 'context'], + completed_steps: ['project', 'llm', 'embeddings', 'databases', 'sources', 'context'], }); await writeKtxSetupContextState(tempDir, { runId: 'setup-context-local-ready', @@ -2248,6 +2429,171 @@ describe('setup status', () => { expect(calls).toEqual(['agents']); }); + it('routes a returning user to the context build when config is ready but context is not built', async () => { + const calls: string[] = []; + const io = makeIo(); + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'setup:', + ' database_connection_ids: [warehouse]', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + 'llm:', + ' provider:', + ' backend: anthropic', + ' models:', + ' default: claude-sonnet-4-6', + 'ingest:', + ' embeddings:', + ' backend: openai', + ' model: text-embedding-3-small', + ' dimensions: 1536', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { + completed_steps: ['project', 'llm', 'embeddings', 'databases', 'sources', 'runtime'], + }); + + const readyMenuSelect = vi.fn(); + await expect( + runKtxSetup( + { + command: 'run', + projectDir: tempDir, + mode: 'auto', + agents: false, + inputMode: 'auto', + yes: false, + cliVersion: '0.2.0', + skipLlm: false, + skipEmbeddings: false, + skipDatabases: false, + skipSources: false, + skipAgents: false, + databaseSchemas: [], + }, + io.io, + { + readyMenuDeps: { prompts: { select: readyMenuSelect, cancel: vi.fn() } }, + model: async (args) => { + expect(args.skipLlm).toBe(true); + return { status: 'skipped', projectDir: tempDir }; + }, + embeddings: async (args) => { + expect(args.skipEmbeddings).toBe(true); + return { status: 'skipped', projectDir: tempDir }; + }, + databases: async (args) => { + expect(args.skipDatabases).toBe(true); + return { status: 'skipped', projectDir: tempDir }; + }, + sources: async (args) => { + expect(args.skipSources).toBe(true); + return { status: 'skipped', projectDir: tempDir }; + }, + runtime: async () => { + calls.push('runtime'); + return runtimeReady(tempDir); + }, + context: async (args) => { + calls.push('context'); + expect(args.forcePrompt).toBe(true); + return { status: 'skipped', projectDir: tempDir }; + }, + agents: async () => { + calls.push('agents'); + return { status: 'ready', projectDir: tempDir, installs: [] }; + }, + }, + ), + ).resolves.toBe(0); + + // Config is done, so the change-everything menu is not shown; setup routes straight + // to the build prompt and never re-walks config or installs agents. + expect(readyMenuSelect).not.toHaveBeenCalled(); + expect(calls).toContain('context'); + expect(calls).not.toContain('agents'); + const output = io.stdout(); + expect(output).toContain('Setup is complete. The only step left is to build context'); + expect(output).toContain('ktx ingest'); + }); + + it('reaches the completion screen instead of a bare shell when the context build is skipped', async () => { + const calls: string[] = []; + const io = makeIo(); + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'setup:', + ' database_connection_ids: [warehouse]', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + 'llm:', + ' provider:', + ' backend: anthropic', + ' models:', + ' default: claude-sonnet-4-6', + 'ingest:', + ' embeddings:', + ' backend: openai', + ' model: text-embedding-3-small', + ' dimensions: 1536', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { + completed_steps: ['project', 'llm', 'embeddings', 'databases', 'sources', 'runtime'], + }); + + await expect( + runKtxSetup( + { + command: 'run', + projectDir: tempDir, + mode: 'auto', + agents: false, + inputMode: 'disabled', + yes: true, + cliVersion: '0.2.0', + skipLlm: true, + skipEmbeddings: true, + skipDatabases: true, + skipSources: true, + skipAgents: false, + databaseSchemas: [], + }, + io.io, + { + model: async () => ({ status: 'skipped', projectDir: tempDir }), + embeddings: async () => ({ status: 'skipped', projectDir: tempDir }), + databases: async () => ({ status: 'skipped', projectDir: tempDir }), + sources: async () => ({ status: 'skipped', projectDir: tempDir }), + runtime: async () => runtimeReady(tempDir), + context: async () => ({ status: 'skipped', projectDir: tempDir }), + agents: async () => { + calls.push('agents'); + return { status: 'ready', projectDir: tempDir, installs: [] }; + }, + }, + ), + ).resolves.toBe(0); + + // A skipped build must not install agents nor drop to a bare shell; the end screen + // states readiness and points at `ktx ingest`. + expect(calls).not.toContain('agents'); + const output = io.stdout(); + expect(output).toContain('Setup is complete. The only step left is to build context'); + expect(output).toContain('ktx ingest'); + }); + it('runs only project resolution and agent setup in --agents mode', async () => { const io = makeIo(); const runtime = vi.fn(async () => runtimeReady(tempDir)); diff --git a/packages/cli/src/sl.test.ts b/packages/cli/test/sl.test.ts similarity index 69% rename from packages/cli/src/sl.test.ts rename to packages/cli/test/sl.test.ts index 7fa855d0..489ea950 100644 --- a/packages/cli/src/sl.test.ts +++ b/packages/cli/test/sl.test.ts @@ -1,11 +1,18 @@ -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { stripVTControlCharacters } from 'node:util'; import Database from 'better-sqlite3'; -import { initKtxProject } from './context/project/project.js'; +import { parseKtxProjectConfig, serializeKtxProjectConfig } from '../src/context/project/config.js'; +import { initKtxProject } from '../src/context/project/project.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { runKtxSl } from './sl.js'; +import { runKtxSl } from '../src/sl.js'; + +const reportExceptionMock = vi.hoisted(() => vi.fn(async () => {})); + +vi.mock('../src/telemetry/exception.js', () => ({ + reportException: reportExceptionMock, +})); const ORDERS_YAML = [ 'name: orders', @@ -61,6 +68,7 @@ describe('runKtxSl', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-sl-')); + reportExceptionMock.mockClear(); }); afterEach(async () => { @@ -113,6 +121,273 @@ describe('runKtxSl', () => { }); }); + it('reads a semantic-layer source as raw YAML', async () => { + const projectDir = join(tempDir, 'read-project'); + await seedSlSource({ projectDir }); + + const readIo = makeIo(); + await expect( + runKtxSl( + { + command: 'read', + projectDir, + connectionId: 'warehouse', + sourceName: 'orders', + }, + readIo.io, + ), + ).resolves.toBe(0); + + expect(readIo.stdout()).toBe(ORDERS_YAML); + expect(readIo.stderr()).toBe(''); + }); + + it('reads a unique semantic-layer source without a connection id', async () => { + const projectDir = join(tempDir, 'read-unique-project'); + const project = await initKtxProject({ projectDir }); + await project.fileStore.writeFile( + 'semantic-layer/warehouse/orders.yaml', + ORDERS_YAML, + 'ktx', + 'ktx@example.com', + 'Add semantic-layer source', + ); + await project.fileStore.writeFile( + 'semantic-layer/analytics/tickets.yaml', + [ + 'name: tickets', + 'table: public.tickets', + 'grain:', + ' - ticket_id', + 'columns:', + ' - name: ticket_id', + ' type: string', + '', + ].join('\n'), + 'ktx', + 'ktx@example.com', + 'Add semantic-layer source', + ); + + const readIo = makeIo(); + await expect( + runKtxSl( + { + command: 'read', + projectDir, + sourceName: 'tickets', + }, + readIo.io, + ), + ).resolves.toBe(0); + + expect(readIo.stdout()).toContain('name: tickets'); + expect(readIo.stderr()).toBe(''); + }); + + it('reports ambiguous unscoped reads with sorted connection ids', async () => { + const projectDir = join(tempDir, 'read-ambiguous-project'); + const project = await initKtxProject({ projectDir }); + await project.fileStore.writeFile( + 'semantic-layer/warehouse/orders.yaml', + ORDERS_YAML, + 'ktx', + 'ktx@example.com', + 'Add semantic-layer source', + ); + await project.fileStore.writeFile( + 'semantic-layer/analytics/orders.yaml', + ORDERS_YAML, + 'ktx', + 'ktx@example.com', + 'Add semantic-layer source', + ); + + const readIo = makeIo(); + await expect( + runKtxSl( + { + command: 'read', + projectDir, + sourceName: 'orders', + }, + readIo.io, + ), + ).resolves.toBe(1); + + expect(readIo.stdout()).toBe(''); + expect(readIo.stderr()).toBe( + "Source 'orders' exists in multiple connections: analytics, warehouse. Re-run with --connection-id .\n", + ); + }); + + it('reports a clear error when an unscoped semantic-layer source is missing', async () => { + const projectDir = join(tempDir, 'missing-unscoped-read-project'); + await seedSlSource({ projectDir }); + + const readIo = makeIo(); + await expect( + runKtxSl( + { + command: 'read', + projectDir, + sourceName: 'missing_orders', + }, + readIo.io, + ), + ).resolves.toBe(1); + + expect(readIo.stdout()).toBe(''); + expect(readIo.stderr()).toBe("No semantic-layer source 'missing_orders'\n"); + }); + + it('reports a clear error when a semantic-layer source is missing', async () => { + const projectDir = join(tempDir, 'missing-read-project'); + await seedSlSource({ projectDir }); + + const readIo = makeIo(); + await expect( + runKtxSl( + { + command: 'read', + projectDir, + connectionId: 'warehouse', + sourceName: 'missing_orders', + }, + readIo.io, + ), + ).resolves.toBe(1); + + expect(readIo.stdout()).toBe(''); + expect(readIo.stderr()).toBe("No semantic-layer source 'missing_orders' for connection 'warehouse'\n"); + }); + + it('validates a unique semantic-layer source without a connection id', async () => { + const projectDir = join(tempDir, 'validate-unique-project'); + const project = await initKtxProject({ projectDir }); + await project.fileStore.writeFile( + 'semantic-layer/warehouse/orders.yaml', + ORDERS_YAML, + 'ktx', + 'ktx@example.com', + 'Add semantic-layer source', + ); + await project.fileStore.writeFile( + 'semantic-layer/analytics/tickets.yaml', + [ + 'name: tickets', + 'table: public.tickets', + 'grain:', + ' - ticket_id', + 'columns:', + ' - name: ticket_id', + ' type: string', + '', + ].join('\n'), + 'ktx', + 'ktx@example.com', + 'Add semantic-layer source', + ); + + const validateIo = makeIo(); + await expect( + runKtxSl( + { + command: 'validate', + projectDir, + sourceName: 'tickets', + }, + validateIo.io, + ), + ).resolves.toBe(0); + + expect(validateIo.stdout()).toBe('Valid semantic-layer source: analytics/tickets\n'); + expect(validateIo.stderr()).toBe(''); + }); + + it('reports ambiguous unscoped validation with sorted connection ids', async () => { + const projectDir = join(tempDir, 'validate-ambiguous-project'); + const project = await initKtxProject({ projectDir }); + await project.fileStore.writeFile( + 'semantic-layer/warehouse/orders.yaml', + ORDERS_YAML, + 'ktx', + 'ktx@example.com', + 'Add semantic-layer source', + ); + await project.fileStore.writeFile( + 'semantic-layer/analytics/orders.yaml', + ORDERS_YAML, + 'ktx', + 'ktx@example.com', + 'Add semantic-layer source', + ); + + const validateIo = makeIo(); + await expect( + runKtxSl( + { + command: 'validate', + projectDir, + sourceName: 'orders', + }, + validateIo.io, + ), + ).resolves.toBe(1); + + expect(validateIo.stdout()).toBe(''); + expect(validateIo.stderr()).toBe( + "Source 'orders' exists in multiple connections: analytics, warehouse. Re-run with --connection-id .\n", + ); + }); + + it('reports a clear error when an unscoped semantic-layer source validation target is missing', async () => { + const projectDir = join(tempDir, 'missing-unscoped-validate-project'); + await seedSlSource({ projectDir }); + + const validateIo = makeIo(); + await expect( + runKtxSl( + { + command: 'validate', + projectDir, + sourceName: 'missing_orders', + }, + validateIo.io, + ), + ).resolves.toBe(1); + + expect(validateIo.stdout()).toBe(''); + expect(validateIo.stderr()).toBe('Semantic-layer source "missing_orders" was not found\n'); + expect(reportExceptionMock).toHaveBeenCalledWith( + expect.objectContaining({ + context: expect.objectContaining({ source: 'sl validate', handled: true, fatal: false }), + projectDir, + }), + ); + }); + + it('keeps scoped validation not-found wording', async () => { + const projectDir = join(tempDir, 'missing-scoped-validate-project'); + await seedSlSource({ projectDir }); + + const validateIo = makeIo(); + await expect( + runKtxSl( + { + command: 'validate', + projectDir, + connectionId: 'warehouse', + sourceName: 'missing_orders', + }, + validateIo.io, + ), + ).resolves.toBe(1); + + expect(validateIo.stdout()).toBe(''); + expect(validateIo.stderr()).toBe('Semantic-layer source "warehouse/missing_orders" was not found\n'); + }); + it('prints semantic-layer search rank badges in pretty output', async () => { const projectDir = join(tempDir, 'rank-project'); await seedSlSource({ projectDir }); @@ -291,6 +566,53 @@ joins: [] expect(stderr.write).not.toHaveBeenCalled(); }); + it('reports sl query exceptions at the query catch boundary', async () => { + vi.stubEnv('ANTHROPIC_API_KEY', 'sl-anthropic-secret'); // pragma: allowlist secret + const projectDir = join(tempDir, 'missing-query-input'); + await seedSlSource({ projectDir }); + const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')); + await writeFile( + join(projectDir, 'ktx.yaml'), + serializeKtxProjectConfig({ + ...config, + llm: { + ...config.llm, + provider: { + backend: 'anthropic', + anthropic: { api_key: 'env:ANTHROPIC_API_KEY' }, // pragma: allowlist secret + }, + models: { default: 'claude-sonnet-4-6' }, + }, + }), + 'utf-8', + ); + const io = makeIo(); + + await expect( + runKtxSl( + { + command: 'query', + projectDir, + connectionId: 'warehouse', + format: 'json', + execute: false, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + }, + io.io, + ), + ).resolves.toBe(1); + + expect(io.stderr()).toContain('sl query requires query input'); + expect(reportExceptionMock).toHaveBeenCalledWith( + expect.objectContaining({ + context: expect.objectContaining({ source: 'sl query', handled: true, fatal: false }), + projectDir, + redactionSecrets: expect.arrayContaining(['sl-anthropic-secret']), + }), + ); + }); + it('emits debug telemetry for sl query without project paths', async () => { vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); vi.stubEnv('CI', ''); diff --git a/packages/cli/src/source-mapping.test.ts b/packages/cli/test/source-mapping.test.ts similarity index 94% rename from packages/cli/src/source-mapping.test.ts rename to packages/cli/test/source-mapping.test.ts index 83f9496b..5099d4c0 100644 --- a/packages/cli/src/source-mapping.test.ts +++ b/packages/cli/test/source-mapping.test.ts @@ -2,8 +2,8 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import type { KtxCliIo } from './cli-runtime.js'; -import { runKtxSourceMapping } from './source-mapping.js'; +import type { KtxCliIo } from '../src/cli-runtime.js'; +import { runKtxSourceMapping } from '../src/source-mapping.js'; function makeIo() { let stdout = ''; diff --git a/packages/cli/src/sql.test.ts b/packages/cli/test/sql.test.ts similarity index 88% rename from packages/cli/src/sql.test.ts rename to packages/cli/test/sql.test.ts index 51cfe920..5e297429 100644 --- a/packages/cli/src/sql.test.ts +++ b/packages/cli/test/sql.test.ts @@ -1,12 +1,18 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject } from './context/project/project.js'; -import { parseKtxProjectConfig, serializeKtxProjectConfig } from './context/project/config.js'; -import type { KtxScanConnector } from './context/scan/types.js'; -import type { SqlAnalysisPort } from './context/sql-analysis/ports.js'; +import { initKtxProject } from '../src/context/project/project.js'; +import { parseKtxProjectConfig, serializeKtxProjectConfig } from '../src/context/project/config.js'; +import type { KtxScanConnector } from '../src/context/scan/types.js'; +import type { SqlAnalysisPort } from '../src/context/sql-analysis/ports.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { runKtxSql } from './sql.js'; +import { runKtxSql } from '../src/sql.js'; + +const reportExceptionMock = vi.hoisted(() => vi.fn(async () => {})); + +vi.mock('../src/telemetry/exception.js', () => ({ + reportException: reportExceptionMock, +})); function makeIo(options: { isTTY?: boolean } = {}) { let stdout = ''; @@ -33,7 +39,7 @@ function makeIo(options: { isTTY?: boolean } = {}) { function makeSqlAnalysis(result: Awaited>): SqlAnalysisPort { return { analyzeForFingerprint: vi.fn(), - analyzeBatch: vi.fn(async () => new Map([['cli-sql', { tablesTouched: ['orders'], columnsByClause: {} }]])), + analyzeBatch: vi.fn(async () => new Map([['cli-sql', { tablesTouched: [{ catalog: null, db: null, name: 'orders' }], columnsByClause: {} }]])), validateReadOnly: vi.fn(async () => result), }; } @@ -66,6 +72,8 @@ function makeConnector(overrides: Partial = {}): KtxScanConnec })), cleanup: vi.fn(async () => undefined), ...overrides, + listSchemas: overrides.listSchemas ?? vi.fn(async () => []), + listTables: overrides.listTables ?? vi.fn(async () => []), }; } @@ -74,6 +82,7 @@ describe('runKtxSql', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-sql-')); + reportExceptionMock.mockClear(); }); afterEach(async () => { @@ -234,9 +243,10 @@ describe('runKtxSql', () => { }); it('rejects non-read-only SQL before executing connector SQL', async () => { + vi.stubEnv('SQL_DB_PASSWORD', 'sql-db-password'); // pragma: allowlist secret const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir }); - await writeConnections(projectDir, { warehouse: { driver: 'sqlite', path: 'warehouse.db' } }); + await writeConnections(projectDir, { warehouse: { driver: 'postgres', password: 'env:SQL_DB_PASSWORD' } }); // pragma: allowlist secret const connector = makeConnector(); const io = makeIo(); @@ -263,6 +273,13 @@ describe('runKtxSql', () => { expect(connector.executeReadOnly).not.toHaveBeenCalled(); expect(connector.cleanup).not.toHaveBeenCalled(); expect(io.stderr()).toContain('SQL contains read/write operation: Delete'); + expect(reportExceptionMock).toHaveBeenCalledWith( + expect.objectContaining({ + context: expect.objectContaining({ source: 'sql run', handled: true, fatal: false }), + projectDir, + redactionSecrets: expect.arrayContaining(['sql-db-password']), + }), + ); }); it('rejects missing connections', async () => { diff --git a/packages/cli/src/standalone-smoke.test.ts b/packages/cli/test/standalone-smoke.test.ts similarity index 91% rename from packages/cli/src/standalone-smoke.test.ts rename to packages/cli/test/standalone-smoke.test.ts index 7e6ed56e..7dde8979 100644 --- a/packages/cli/src/standalone-smoke.test.ts +++ b/packages/cli/test/standalone-smoke.test.ts @@ -3,7 +3,7 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { promisify } from 'node:util'; -import { parseKtxProjectConfig } from './context/project/config.js'; +import { parseKtxProjectConfig } from '../src/context/project/config.js'; import Database from 'better-sqlite3'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; @@ -185,7 +185,7 @@ describe('standalone built ktx CLI smoke', () => { expect([0, 1]).toContain(result.code); }); - it('runs fast public database ingest through the built binary with manifest artifacts', async () => { + it('blocks public database ingest through the built binary when enrichment is not configured', async () => { const projectDir = join(tempDir, 'database-ingest-project'); const init = await runSetupNewProject(projectDir); expectSetupStderr(init); @@ -200,19 +200,10 @@ describe('standalone built ktx CLI smoke', () => { expect(connectionTest.stdout).toContain('Driver: sqlite'); expect(connectionTest.stdout).toContain('Status: ok'); - const ingest = await runBuiltCli(['ingest', 'warehouse', '--project-dir', projectDir, '--fast', '--no-input']); - expectProjectStderr(ingest, projectDir); - expect(ingest.stdout).toContain('Ingest finished'); - expect(ingest.stdout).toContain('warehouse'); - expect(ingest.stdout).toContain('Database schema'); - expect(ingest.stdout).toContain('warehouse done'); + const ingest = await runBuiltCli(['ingest', 'warehouse', '--project-dir', projectDir, '--no-input']); + expect(ingest.code).toBe(1); + expect(ingest.stdout).toContain('warehouse cannot be ingested: enrichment is not configured'); expect(ingest.stdout).not.toContain('KTX scan completed'); - - const manifest = await readFile(join(projectDir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8'); - expect(manifest).toContain('customers:'); - expect(manifest).toContain('orders:'); - expect(manifest).toContain('source: formal'); - expect(manifest).not.toContain('ai:'); }, 30_000); it('parses gateway LLM config and OpenAI enrichment embeddings used by standalone scans without network calls', async () => { diff --git a/packages/cli/src/status-project.test.ts b/packages/cli/test/status-project.test.ts similarity index 72% rename from packages/cli/src/status-project.test.ts rename to packages/cli/test/status-project.test.ts index 83862bfb..cd63cf19 100644 --- a/packages/cli/src/status-project.test.ts +++ b/packages/cli/test/status-project.test.ts @@ -3,13 +3,13 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import Database from 'better-sqlite3'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; -import type { KtxLocalProject } from './context/project/project.js'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from '../src/context/project/config.js'; +import type { KtxLocalProject } from '../src/context/project/project.js'; import { buildLocalStatsStatus, buildProjectStatus, renderProjectStatus, -} from './status-project.js'; +} from '../src/status-project.js'; function projectWithConfig(config: KtxProjectConfig): KtxLocalProject { return { @@ -44,6 +44,17 @@ function withClaudeCodeLlm(config: KtxProjectConfig): KtxProjectConfig { }; } +function withCodexLlm(config: KtxProjectConfig): KtxProjectConfig { + return { + ...config, + llm: { + ...config.llm, + provider: { backend: 'codex' }, + models: { ...config.llm.models, default: 'gpt-5.5' }, + }, + }; +} + function baseProjectConfig(): KtxProjectConfig { return withClaudeCodeLlm(buildDefaultKtxProjectConfig()); } @@ -197,26 +208,58 @@ function withMysqlQueryHistory(config: KtxProjectConfig): KtxProjectConfig { }; } +function fakeStatusRunner( + dialect: 'postgres' | 'snowflake' | 'bigquery', + catalogName: string, +) { + return { + dialect, + catalogName, + async run() { + return { warnings: [], info: [] }; + }, + formatSuccessDetail(result: unknown) { + const typed = result as { warnings: string[]; info?: string[]; pgServerVersion?: string }; + const info = typed.info && typed.info.length > 0 ? `; ${typed.info.join('; ')}` : ''; + const base = + dialect === 'postgres' + ? `pg_stat_statements ready (${typed.pgServerVersion ?? 'PostgreSQL 16.4'})` + : `${catalogName} ready`; + return { detail: `${base}${info}`, warnings: typed.warnings }; + }, + fixAdvice(error: unknown) { + return { + failHeadline: error instanceof Error ? error.message : String(error), + remediation: 'Fix query-history grants.', + }; + }, + }; +} + describe('buildProjectStatus query history dispatch', () => { - it('runs the snowflake probe for snowflake connections, not the postgres one', async () => { - let postgresCalls = 0; - let snowflakeCalls = 0; + it('runs the shared probe for snowflake connections', async () => { + let probeCalls = 0; + const runner = fakeStatusRunner( + 'snowflake', + 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY', + ); const project = projectWithConfig(withSnowflakeQueryHistory(baseProjectConfig())); const status = await buildProjectStatus(project, { claudeCodeAuthProbe: stubClaudeCodeAuthProbe, - postgresQueryHistoryProbe: async () => { - postgresCalls += 1; - throw new Error('postgres probe should not run for snowflake'); - }, - snowflakeQueryHistoryProbe: async () => { - snowflakeCalls += 1; - return { warnings: [], info: [] }; + queryHistoryReadinessProbe: async (input) => { + probeCalls += 1; + expect(input.connectionId).toBe('warehouse'); + return { + ok: true, + dialect: 'snowflake', + runner, + result: { warnings: [], info: [] }, + }; }, }); - expect(postgresCalls).toBe(0); - expect(snowflakeCalls).toBe(1); + expect(probeCalls).toBe(1); expect(status.queryHistory).toHaveLength(1); expect(status.queryHistory[0]).toMatchObject({ connection: 'warehouse', @@ -231,19 +274,21 @@ describe('buildProjectStatus query history dispatch', () => { it('reports snowflake probe failures with the reader-provided remediation', async () => { const project = projectWithConfig(withSnowflakeQueryHistory(baseProjectConfig())); - const { HistoricSqlGrantsMissingError } = await import( - './context/ingest/adapters/historic-sql/errors.js' - ); const status = await buildProjectStatus(project, { claudeCodeAuthProbe: stubClaudeCodeAuthProbe, - snowflakeQueryHistoryProbe: async () => { - throw new HistoricSqlGrantsMissingError({ - dialect: 'snowflake', - message: 'role cannot read SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY', - remediation: 'GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE ktx;', - }); - }, + queryHistoryReadinessProbe: async () => ({ + ok: false, + dialect: 'snowflake', + runner: { + ...fakeStatusRunner('snowflake', 'SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY'), + fixAdvice: () => ({ + failHeadline: 'Snowflake role cannot read SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY', + remediation: 'GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE ktx;', + }), + }, + error: new Error('role cannot read SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY'), + }), }); expect(status.queryHistory[0]).toMatchObject({ @@ -257,18 +302,25 @@ describe('buildProjectStatus query history dispatch', () => { }); it('runs the bigquery probe for bigquery connections', async () => { - let bigqueryCalls = 0; + let probeCalls = 0; + const runner = fakeStatusRunner('bigquery', 'INFORMATION_SCHEMA.JOBS_BY_PROJECT'); const project = projectWithConfig(withBigQueryQueryHistory(baseProjectConfig())); const status = await buildProjectStatus(project, { claudeCodeAuthProbe: stubClaudeCodeAuthProbe, - bigqueryQueryHistoryProbe: async () => { - bigqueryCalls += 1; - return { warnings: [], info: [] }; + queryHistoryReadinessProbe: async (input) => { + probeCalls += 1; + expect(input.connectionId).toBe('bq'); + return { + ok: true, + dialect: 'bigquery', + runner, + result: { warnings: [], info: [] }, + }; }, }); - expect(bigqueryCalls).toBe(1); + expect(probeCalls).toBe(1); expect(status.queryHistory[0]).toMatchObject({ connection: 'bq', driver: 'bigquery', @@ -283,7 +335,7 @@ describe('buildProjectStatus query history dispatch', () => { const status = await buildProjectStatus(project, { claudeCodeAuthProbe: stubClaudeCodeAuthProbe, - postgresQueryHistoryProbe: async () => { + queryHistoryReadinessProbe: async () => { throw new Error('postgres probe must not run for mysql'); }, }); @@ -306,7 +358,7 @@ describe('buildProjectStatus query history dispatch', () => { describe('buildProjectStatus --fast', () => { it('skips claude-code probe and Postgres query-history probe', async () => { let claudeProbeCalls = 0; - let pgProbeCalls = 0; + let queryHistoryProbeCalls = 0; const project = projectWithConfig(withPostgresQueryHistory(baseProjectConfig())); const status = await buildProjectStatus(project, { @@ -316,14 +368,14 @@ describe('buildProjectStatus --fast', () => { claudeProbeCalls += 1; return { ok: true }; }, - postgresQueryHistoryProbe: async () => { - pgProbeCalls += 1; + queryHistoryReadinessProbe: async () => { + queryHistoryProbeCalls += 1; throw new Error('should not be called'); }, }); expect(claudeProbeCalls).toBe(0); - expect(pgProbeCalls).toBe(0); + expect(queryHistoryProbeCalls).toBe(0); expect(status.llm.status).toBe('skipped'); expect(status.llm.detail).toMatch(/--fast/); expect(status.queryHistory).toHaveLength(1); @@ -340,7 +392,7 @@ describe('buildProjectStatus --fast', () => { env: { ANALYTICS_DATABASE_URL: 'postgres://example' }, fast: true, claudeCodeAuthProbe: stubClaudeCodeAuthProbe, - postgresQueryHistoryProbe: async () => { + queryHistoryReadinessProbe: async () => { throw new Error('should not be called'); }, }); @@ -350,6 +402,126 @@ describe('buildProjectStatus --fast', () => { }); }); +describe('buildProjectStatus codex', () => { + it('reports authenticated local Codex session', async () => { + const project = projectWithConfig(withCodexLlm(buildDefaultKtxProjectConfig())); + const status = await buildProjectStatus(project, { + codexAuthProbe: async () => ({ ok: true as const }), + }); + + expect(status.llm).toMatchObject({ + backend: 'codex', + model: 'gpt-5.5', + status: 'ok', + detail: 'local Codex session authenticated', + }); + expect(status.warnings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + message: expect.stringContaining('Codex backend isolation is limited'), + fix: expect.stringContaining('claude-code'), + }), + ]), + ); + const rendered = renderProjectStatus(status, { verbose: false, useColor: false }); + expect(rendered).toContain('Codex backend isolation is limited'); + }); + + it('skips Codex auth probe with --fast', async () => { + let probeCalls = 0; + const project = projectWithConfig(withCodexLlm(buildDefaultKtxProjectConfig())); + const status = await buildProjectStatus(project, { + fast: true, + codexAuthProbe: async () => { + probeCalls += 1; + return { ok: true }; + }, + }); + + expect(probeCalls).toBe(0); + expect(status.llm.status).toBe('skipped'); + expect(status.llm.detail).toMatch(/--fast/); + }); + + it('surfaces the probe fix for a model-access failure instead of an auth fix', async () => { + const project = projectWithConfig(withCodexLlm(buildDefaultKtxProjectConfig())); + const status = await buildProjectStatus(project, { + codexAuthProbe: async () => ({ + ok: false, + message: 'Codex is authenticated, but the configured model "gpt-5.5" is not available...', + fix: 'Run `codex` to see the models your account supports, then set llm.models.default in ktx.yaml (or rerun `ktx setup`).', + }), + }); + + expect(status.llm.status).toBe('fail'); + expect(status.llm.fix).toContain('llm.models.default'); + expect(status.llm.fix).not.toContain('Authenticate Codex'); + }); +}); + +describe('buildProjectStatus llm models.default requirement', () => { + function withBackendNoModel( + backend: KtxProjectConfig['llm']['provider']['backend'], + ): KtxProjectConfig { + const config = buildDefaultKtxProjectConfig(); + return { + ...config, + llm: { ...config.llm, provider: { backend }, models: {} }, + }; + } + + it('fails codex without llm.models.default and never probes', async () => { + let probeCalls = 0; + const project = projectWithConfig(withBackendNoModel('codex')); + const status = await buildProjectStatus(project, { + codexAuthProbe: async () => { + probeCalls += 1; + return { ok: true }; + }, + }); + + expect(probeCalls).toBe(0); + expect(status.llm.status).toBe('fail'); + expect(status.llm.detail).toContain('llm.models.default'); + expect(status.verdict).toBe('blocked'); + }); + + it('fails claude-code without llm.models.default and never probes', async () => { + let probeCalls = 0; + const project = projectWithConfig(withBackendNoModel('claude-code')); + const status = await buildProjectStatus(project, { + claudeCodeAuthProbe: async () => { + probeCalls += 1; + return { ok: true }; + }, + }); + + expect(probeCalls).toBe(0); + expect(status.llm.status).toBe('fail'); + expect(status.llm.detail).toContain('llm.models.default'); + expect(status.verdict).toBe('blocked'); + }); + + it('fails anthropic without llm.models.default even when the key is set', async () => { + const config = withBackendNoModel('anthropic'); + const project = projectWithConfig({ + ...config, + llm: { + ...config.llm, + provider: { backend: 'anthropic', anthropic: { api_key: 'env:ANTHROPIC_API_KEY' } }, // pragma: allowlist secret + models: {}, + }, + }); + const status = await buildProjectStatus(project, { + env: { ANTHROPIC_API_KEY: 'sk-test' }, // pragma: allowlist secret + }); + + expect(status.llm.status).toBe('fail'); + expect(status.llm.detail).toContain('llm.models.default'); + expect(status.verdict).toBe('blocked'); + }); +}); + describe('buildLocalStatsStatus', () => { let tempDir: string; diff --git a/packages/cli/src/telemetry/command-hook.test.ts b/packages/cli/test/telemetry/command-hook.test.ts similarity index 58% rename from packages/cli/src/telemetry/command-hook.test.ts rename to packages/cli/test/telemetry/command-hook.test.ts index ffd0485b..63909ac6 100644 --- a/packages/cli/src/telemetry/command-hook.test.ts +++ b/packages/cli/test/telemetry/command-hook.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { beginCommandSpan, completeCommandSpan, resetCommandSpan } from './command-hook.js'; +import { beginCommandSpan, completeCommandSpan, resetCommandSpan } from '../../src/telemetry/command-hook.js'; describe('telemetry command hook', () => { it('builds a completed command event from a span', () => { @@ -34,4 +34,23 @@ describe('telemetry command hook', () => { resetCommandSpan(); expect(completeCommandSpan({ completedAt: 200, outcome: 'ok' })).toBeUndefined(); }); + + it('captures errorClass and raw errorDetail on a failed command', () => { + resetCommandSpan(); + beginCommandSpan({ + commandPath: ['ktx', 'ingest'], + flagsPresent: {}, + hasProject: true, + attachProjectGroup: false, + startedAt: 0, + }); + + class KtxConnectionError extends Error {} + const error = new KtxConnectionError('connect ECONNREFUSED 127.0.0.1:5432'); + + const completed = completeCommandSpan({ completedAt: 10, outcome: 'error', error }); + expect(completed?.outcome).toBe('error'); + expect(completed?.errorClass).toBe('KtxConnectionError'); + expect(completed?.errorDetail).toBe('connect ECONNREFUSED 127.0.0.1:5432'); + }); }); diff --git a/packages/cli/src/telemetry/demo-detect.test.ts b/packages/cli/test/telemetry/demo-detect.test.ts similarity index 91% rename from packages/cli/src/telemetry/demo-detect.test.ts rename to packages/cli/test/telemetry/demo-detect.test.ts index b371694e..4640766f 100644 --- a/packages/cli/src/telemetry/demo-detect.test.ts +++ b/packages/cli/test/telemetry/demo-detect.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { isDemoConnection } from './demo-detect.js'; +import { isDemoConnection } from '../../src/telemetry/demo-detect.js'; describe('isDemoConnection', () => { it('detects only the packaged Orbit SQLite demo recipe', () => { diff --git a/packages/cli/src/telemetry/emitter.test.ts b/packages/cli/test/telemetry/emitter.test.ts similarity index 96% rename from packages/cli/src/telemetry/emitter.test.ts rename to packages/cli/test/telemetry/emitter.test.ts index 9c732997..98400860 100644 --- a/packages/cli/src/telemetry/emitter.test.ts +++ b/packages/cli/test/telemetry/emitter.test.ts @@ -4,8 +4,8 @@ import { __resetTelemetryEmitterForTests, shutdownTelemetryEmitter, trackTelemetryEvent, -} from './emitter.js'; -import type { BuiltTelemetryEvent } from './events.js'; +} from '../../src/telemetry/emitter.js'; +import type { BuiltTelemetryEvent } from '../../src/telemetry/events.js'; const captures: unknown[] = []; const shutdown = vi.fn(async () => {}); diff --git a/packages/cli/src/telemetry/events.snapshot.test.ts b/packages/cli/test/telemetry/events.snapshot.test.ts similarity index 96% rename from packages/cli/src/telemetry/events.snapshot.test.ts rename to packages/cli/test/telemetry/events.snapshot.test.ts index 1df95aa0..1ea67339 100644 --- a/packages/cli/src/telemetry/events.snapshot.test.ts +++ b/packages/cli/test/telemetry/events.snapshot.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { buildTelemetryEvent, type TelemetryCommonEnvelope } from './events.js'; +import { buildTelemetryEvent, type TelemetryCommonEnvelope } from '../../src/telemetry/events.js'; const BLACKLIST = [ '/Users/', @@ -128,7 +128,9 @@ describe('telemetry privacy snapshot', () => { outcome: 'error', errorClass: 'KtxProjectMissingAbortError', durationMs: 12, - sampleRate: 0.1, + sampleRate: 1, + mcpClientName: 'Claude Desktop', + mcpClientVersion: '0.7.1', }), ]; diff --git a/packages/cli/src/telemetry/events.test.ts b/packages/cli/test/telemetry/events.test.ts similarity index 98% rename from packages/cli/src/telemetry/events.test.ts rename to packages/cli/test/telemetry/events.test.ts index 3726ddde..033c2def 100644 --- a/packages/cli/src/telemetry/events.test.ts +++ b/packages/cli/test/telemetry/events.test.ts @@ -5,7 +5,7 @@ import { telemetryEventCatalog, telemetryEventSchemas, type TelemetryCommonEnvelope, -} from './events.js'; +} from '../../src/telemetry/events.js'; const envelope: TelemetryCommonEnvelope = { cliVersion: '0.4.1', @@ -37,6 +37,7 @@ describe('telemetry event schemas', () => { 'daemon_stopped', 'sl_plan_completed', 'sql_gen_completed', + 'query_history_filter_completed', ]); }); diff --git a/packages/cli/test/telemetry/exception-payload.test.ts b/packages/cli/test/telemetry/exception-payload.test.ts new file mode 100644 index 00000000..da81e62e --- /dev/null +++ b/packages/cli/test/telemetry/exception-payload.test.ts @@ -0,0 +1,150 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { createServer, type IncomingMessage } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { gunzipSync } from 'node:zlib'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { KtxCliIo } from '../../src/cli-runtime.js'; +import { __resetTelemetryEmitterForTests } from '../../src/telemetry/emitter.js'; +import { + __resetTelemetryExceptionStateForTests, + reportException, +} from '../../src/telemetry/exception.js'; + +function makeIo(): KtxCliIo { + return { + stdout: { write: () => {} }, + stderr: { write: () => {} }, + }; +} + +async function body(req: IncomingMessage): Promise { + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + const raw = Buffer.concat(chunks); + return req.headers['content-encoding'] === 'gzip' ? gunzipSync(raw).toString('utf-8') : raw.toString('utf-8'); +} + +async function withCaptureServer(run: (url: string, payloads: unknown[]) => Promise): Promise { + const payloads: unknown[] = []; + const server = createServer(async (req, res) => { + if (req.method === 'POST') { + payloads.push(JSON.parse(await body(req))); + } + res.statusCode = 200; + res.setHeader('content-type', 'application/json'); + res.end('{}'); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('test server did not bind to a TCP port'); + } + try { + return await run(`http://127.0.0.1:${address.port}`, payloads); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } +} + +function findExceptionEvent(payloads: unknown[]): Record { + for (const payload of payloads) { + if (typeof payload !== 'object' || payload === null) { + continue; + } + const record = payload as Record; + const batch = Array.isArray(record.batch) ? record.batch : [record]; + for (const item of batch) { + if (typeof item === 'object' && item !== null && (item as Record).event === '$exception') { + return item as Record; + } + } + } + throw new Error(`No $exception payload found: ${JSON.stringify(payloads)}`); +} + +describe('prepared Node exception payload', () => { + let homeDir: string; + + beforeEach(async () => { + homeDir = await mkdtemp(join(tmpdir(), 'ktx-node-exception-payload-')); + await mkdir(join(homeDir, '.ktx'), { recursive: true }); + await writeFile( + join(homeDir, '.ktx', 'telemetry.json'), + `${JSON.stringify({ + installId: '00000000-0000-4000-8000-000000000000', + enabled: true, + createdAt: '2026-06-05T00:00:00.000Z', + })}\n`, + 'utf-8', + ); + vi.stubEnv('HOME', homeDir); + vi.stubEnv('CI', ''); + vi.stubEnv('KTX_TELEMETRY_DISABLED', ''); + vi.stubEnv('DO_NOT_TRACK', ''); + __resetTelemetryEmitterForTests(); + __resetTelemetryExceptionStateForTests(); + }); + + afterEach(async () => { + vi.unstubAllEnvs(); + await rm(homeDir, { recursive: true, force: true }); + }); + + it('sends projectId, omits $groups, and redacts the serialized exception list', async () => { + await withCaptureServer(async (endpoint, payloads) => { + vi.stubEnv('KTX_TELEMETRY_ENDPOINT', endpoint); + const projectDir = join(homeDir, 'project'); + const snapshotSecret = ['plain', 'secret', 'value'].join('-'); + const dbPassword = ['db', 'url', 'secret'].join('-'); + const authToken = ['abc', '123'].join(''); + const error = new Error( + `${snapshotSecret} postgres://svc:${dbPassword}@db.example.test/analytics Authorization: Basic ${authToken}`, + ); + + await reportException({ + error, + context: { source: 'scan run', handled: true, fatal: false }, + io: makeIo(), + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + projectDir, + immediate: true, + redactionSecrets: [snapshotSecret], + }); + + const event = findExceptionEvent(payloads); + const properties = event.properties as Record; + expect(properties.projectId).toMatch(/^[a-f0-9]{64}$/); + expect(properties.$groups).toBeUndefined(); + expect(JSON.stringify(properties.$exception_list)).toContain('[redacted]'); + expect(JSON.stringify(properties.$exception_list)).not.toContain(snapshotSecret); + expect(JSON.stringify(properties.$exception_list)).not.toContain(dbPassword); + expect(JSON.stringify(properties.$exception_list)).not.toContain(authToken); + for (const key of [ + 'argv', + 'args', + 'env', + 'environment', + 'sql', + 'query', + 'prompt', + 'mcpArguments', + 'tableName', + 'schemaName', + 'columnName', + 'databaseUrl', + 'connectionString', + 'url', + 'password', + 'token', + 'apiKey', + 'authorization', + ]) { + expect(properties).not.toHaveProperty(key); + } + }); + }); +}); diff --git a/packages/cli/test/telemetry/exception.test.ts b/packages/cli/test/telemetry/exception.test.ts new file mode 100644 index 00000000..01608935 --- /dev/null +++ b/packages/cli/test/telemetry/exception.test.ts @@ -0,0 +1,456 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { KtxCliIo } from '../../src/cli-runtime.js'; +import { __resetTelemetryEmitterForTests } from '../../src/telemetry/emitter.js'; +import { + __resetTelemetryExceptionStateForTests, + reportException, +} from '../../src/telemetry/exception.js'; + +const captures: unknown[] = []; +const immediateCaptures: unknown[] = []; +const shutdown = vi.fn(async () => {}); + +vi.mock('posthog-node', () => ({ + PostHog: vi.fn(function PostHog() { + return { + captureException: ( + error: unknown, + distinctId?: string, + properties?: Record, + ) => { + captures.push({ error, distinctId, properties }); + }, + captureExceptionImmediate: async ( + error: unknown, + distinctId?: string, + properties?: Record, + ) => { + immediateCaptures.push({ error, distinctId, properties }); + }, + capture: vi.fn(), + shutdown, + }; + }), +})); + +function makeIo(): { io: KtxCliIo; stderr: () => string } { + let stderr = ''; + return { + io: { + stdout: { write: () => {} }, + stderr: { + write: (chunk) => { + stderr += chunk; + }, + }, + }, + stderr: () => stderr, + }; +} + +async function writeIdentity(homeDir: string, enabled = true): Promise { + const path = join(homeDir, '.ktx', 'telemetry.json'); + await mkdir(join(homeDir, '.ktx'), { recursive: true }); + await writeFile( + path, + `${JSON.stringify({ + installId: '00000000-0000-4000-8000-000000000000', + enabled, + createdAt: '2026-06-05T00:00:00.000Z', + })}\n`, + 'utf-8', + ); +} + +describe('reportException', () => { + let homeDir: string; + + beforeEach(async () => { + homeDir = await mkdtemp(join(tmpdir(), 'ktx-exception-')); + await writeIdentity(homeDir); + vi.stubEnv('HOME', homeDir); + vi.stubEnv('CI', ''); + vi.stubEnv('KTX_TELEMETRY_DISABLED', ''); + vi.stubEnv('DO_NOT_TRACK', ''); + captures.length = 0; + immediateCaptures.length = 0; + shutdown.mockClear(); + __resetTelemetryEmitterForTests(); + __resetTelemetryExceptionStateForTests(); + }); + + afterEach(async () => { + vi.unstubAllEnvs(); + await rm(homeDir, { recursive: true, force: true }); + }); + + it('honors telemetry kill switches', async () => { + vi.stubEnv('KTX_TELEMETRY_DISABLED', '1'); + const { io } = makeIo(); + + await reportException({ + error: new Error('boom'), + context: { source: 'scan run', handled: true, fatal: false }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + projectDir: join(homeDir, 'project'), + }); + + expect(captures).toEqual([]); + expect(immediateCaptures).toEqual([]); + }); + + it('prints debug payloads without sending', async () => { + vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); + vi.stubEnv('KTX_TELEMETRY_DISABLED', '1'); + const { io, stderr } = makeIo(); + + await reportException({ + error: new Error('debug boom'), + context: { source: 'scan run', handled: true, fatal: false }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + projectDir: join(homeDir, 'project'), + }); + + expect(stderr()).toContain('[telemetry-exception]'); + expect(stderr()).toContain('"source":"scan run"'); + expect(captures).toEqual([]); + }); + + it('sends projectId as a property and omits $groups for Node exceptions', async () => { + const { io } = makeIo(); + + await reportException({ + error: new Error('project boom'), + context: { source: 'sql run', handled: true, fatal: false }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + projectDir: join(homeDir, 'project'), + }); + + expect(captures).toHaveLength(1); + expect(captures[0]).toMatchObject({ + distinctId: '00000000-0000-4000-8000-000000000000', + properties: { + source: 'sql run', + handled: true, + fatal: false, + cliVersion: '0.0.0-test', + runtime: 'node', + }, + }); + expect( + (captures[0] as { properties: Record }).properties.projectId, + ).toMatch(/^[a-f0-9]{64}$/); + expect((captures[0] as { properties: Record }).properties.$groups).toBeUndefined(); + }); + + it('uses captureExceptionImmediate for fatal reports', async () => { + const { io } = makeIo(); + + await reportException({ + error: new Error('fatal boom'), + context: { source: 'uncaughtException', handled: false, fatal: true }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + immediate: true, + }); + + expect(immediateCaptures).toHaveLength(1); + expect(captures).toEqual([]); + }); + + it('redacts snapshot secrets and static credential patterns from message and cause', async () => { + const { io } = makeIo(); + const cause = new Error('cause has sk-live-fixture-value and Authorization: Bearer token-123'); + const error = new Error('message has sk-live-fixture-value and password=hunter2', { cause }); + + await reportException({ + error, + context: { source: 'connection test', handled: true, fatal: false }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + redactionSecrets: ['sk-live-fixture-value'], + }); + + const sent = captures[0] as { error: Error & { cause?: Error } }; + expect(sent.error.message).toContain('[redacted]'); + expect(sent.error.message).not.toContain('sk-live-fixture-value'); + expect(sent.error.message).not.toContain('hunter2'); + expect(sent.error.cause?.message).not.toContain('token-123'); + }); + + it('redacts URL userinfo credentials and non-bearer authorization values', async () => { + const { io } = makeIo(); + const error = new Error( + 'connect postgres://svc:db-url-secret@db.example.test/analytics Authorization: Basic abc123', // pragma: allowlist secret + ); + + await reportException({ + error, + context: { source: 'connection test', handled: true, fatal: false }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + }); + + const sent = captures[0] as { error: Error }; + expect(sent.error.message).toContain('postgres://svc:[redacted]@db.example.test/analytics'); + expect(sent.error.message).toContain('Authorization: [redacted]'); + expect(sent.error.message).not.toContain('db-url-secret'); + expect(sent.error.message).not.toContain('abc123'); + }); + + it('does not use process-global secret discovery when no snapshot is supplied', async () => { + vi.stubEnv('KTX_FAKE_SECRET', 'plain-secret-without-pattern'); + const { io } = makeIo(); + + await reportException({ + error: new Error('plain-secret-without-pattern'), + context: { source: 'uncaughtException', handled: false, fatal: true }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + }); + + const sent = captures[0] as { error: Error }; + expect(sent.error.message).toContain('plain-secret-without-pattern'); + }); + + it('dedupes the same Error instance between operation and global tiers', async () => { + const { io } = makeIo(); + const error = new Error('same object'); + + await reportException({ + error, + context: { source: 'scan run', handled: true, fatal: false }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + }); + await reportException({ + error, + context: { source: 'uncaughtException', handled: false, fatal: true }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + immediate: true, + }); + + expect(captures).toHaveLength(1); + expect(immediateCaptures).toHaveLength(0); + }); + + it('captures wrapped Error causes as distinct logical occurrences', async () => { + const { io } = makeIo(); + const inner = new Error('inner'); + const wrapper = new Error('outer', { cause: inner }); + + await reportException({ + error: inner, + context: { source: 'sl query', handled: true, fatal: false }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + }); + await reportException({ + error: wrapper, + context: { source: 'uncaughtException', handled: false, fatal: true }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + immediate: true, + }); + + expect(captures).toHaveLength(1); + expect(immediateCaptures).toHaveLength(1); + }); + + it('dedupes primitive and plain-object throwables propagated to the global tier', async () => { + const { io } = makeIo(); + const objectThrowable = { message: 'plain object' }; + + await reportException({ + error: 'primitive boom', + context: { source: 'mcp:sql_execution', handled: true, fatal: false }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + }); + await reportException({ + error: 'primitive boom', + context: { source: 'unhandledRejection', handled: false, fatal: true }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + immediate: true, + }); + await reportException({ + error: objectThrowable, + context: { source: 'mcp:discover_data', handled: true, fatal: false }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + }); + await reportException({ + error: objectThrowable, + context: { source: 'unhandledRejection', handled: false, fatal: true }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + immediate: true, + }); + + expect(captures).toHaveLength(2); + expect(immediateCaptures).toHaveLength(0); + }); + + it('does not collapse independent primitive throw events with the same value', async () => { + const { io } = makeIo(); + + await reportException({ + error: 'oops', + context: { source: 'scan run', handled: true, fatal: false }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + }); + await reportException({ + error: 'oops', + context: { source: 'sql run', handled: true, fatal: false }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + }); + + expect(captures).toHaveLength(2); + }); + + it('drops forbidden caller-supplied extra property keys', async () => { + const { io } = makeIo(); + + await reportException({ + error: new Error('extra property boom'), + context: { + source: 'sql run', + handled: true, + fatal: false, + extra: { + sql: 'select * from private_table', + tableName: 'private_table', + schemaName: 'private_schema', + columnName: 'private_column', + argv: '--password secret', + env: 'KTX_TOKEN=secret', + password: 'secret-password', // pragma: allowlist secret + token: 'secret-token', + prompt: 'user prompt', + safeCount: 3, + }, + }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + }); + + const sent = captures[0] as { properties: Record }; + expect(sent.properties.safeCount).toBe(3); + for (const key of [ + 'sql', + 'tableName', + 'schemaName', + 'columnName', + 'argv', + 'env', + 'password', + 'token', + 'prompt', + ]) { + expect(sent.properties).not.toHaveProperty(key); + } + }); + + it('redacts every required static credential pattern and leaves benign text intact', async () => { + const { io } = makeIo(); + const cases: Array<{ message: string; leaked: string; expected: string }> = [ + { + message: 'dsn password=hunter2', + leaked: 'hunter2', + expected: 'password=[redacted]', + }, + { + message: 'dsn pwd=swordfish', + leaked: 'swordfish', + expected: 'pwd=[redacted]', + }, + { + message: 'Authorization: Basic abc123', + leaked: 'abc123', + expected: 'Authorization: [redacted]', + }, + { + message: 'Authorization: Bearer token-123', + leaked: 'token-123', + expected: 'Authorization: [redacted]', + }, + { + message: 'Bearer standalone-token', + leaked: 'standalone-token', + expected: 'Bearer [redacted]', + }, + { + message: 'api_key=sk-live-secret', + leaked: 'sk-live-secret', + expected: 'api_key=[redacted]', + }, + { + message: 'api-key: sk-dash-secret', + leaked: 'sk-dash-secret', + expected: 'api-key=[redacted]', + }, + { + message: 'KTX_PROVIDER_TOKEN=ktx-secret', + leaked: 'ktx-secret', + expected: 'KTX_PROVIDER_TOKEN=[redacted]', + }, + { + message: 'REFRESH_SECRET: refresh-secret', + leaked: 'refresh-secret', + expected: 'REFRESH_SECRET=[redacted]', + }, + { + message: 'https://s3.example.test/file?X-Amz-Signature=aws-secret&ok=1', + leaked: 'aws-secret', + expected: 'X-Amz-Signature=[redacted]', + }, + { + message: 'https://storage.example.test/file?X-Goog-Signature=goog-secret&ok=1', + leaked: 'goog-secret', + expected: 'X-Goog-Signature=[redacted]', + }, + { + message: 'https://cdn.example.test/file?sig=signed-secret&ok=1', + leaked: 'signed-secret', + expected: 'sig=[redacted]', + }, + { + message: 'postgres://svc:url-password@db.example.test/analytics', // pragma: allowlist secret + leaked: 'url-password', + expected: 'postgres://svc:[redacted]@db.example.test/analytics', + }, + ]; + + for (const item of cases) { + await reportException({ + error: new Error(item.message), + context: { source: 'connection test', handled: true, fatal: false }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + }); + const sent = captures[captures.length - 1] as { error: Error }; + expect(sent.error.message).toContain(item.expected); + expect(sent.error.message).not.toContain(item.leaked); + } + + await reportException({ + error: new Error('token bucket metrics and passwordless auth are benign'), + context: { source: 'connection test', handled: true, fatal: false }, + io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + }); + const benign = captures[captures.length - 1] as { error: Error }; + expect(benign.error.message).toBe('token bucket metrics and passwordless auth are benign'); + }); +}); diff --git a/packages/cli/src/telemetry/identity.test.ts b/packages/cli/test/telemetry/identity.test.ts similarity index 58% rename from packages/cli/src/telemetry/identity.test.ts rename to packages/cli/test/telemetry/identity.test.ts index 06d76043..6c7e3f46 100644 --- a/packages/cli/src/telemetry/identity.test.ts +++ b/packages/cli/test/telemetry/identity.test.ts @@ -9,20 +9,17 @@ import { readExistingTelemetryProjectId, TELEMETRY_NOTICE, type TelemetryIdentityEnv, -} from './identity.js'; +} from '../../src/telemetry/identity.js'; -function makeIo(stdoutIsTTY = true) { +function makeIo() { let stderr = ''; return { - io: { - stdout: { isTTY: stdoutIsTTY, write: () => {} }, - stderr: { - write: (chunk: string) => { - stderr += chunk; - }, + stderr: { + write: (chunk: string) => { + stderr += chunk; }, }, - stderr: () => stderr, + read: () => stderr, }; } @@ -39,14 +36,13 @@ describe('telemetry identity', () => { await rm(homeDir, { recursive: true, force: true }); }); - it('creates the telemetry file and one-line notice on first interactive enabled load', async () => { - const testIo = makeIo(true); + it('creates the telemetry file and one-line notice on first enabled load', async () => { + const testIo = makeIo(); const identity = await loadTelemetryIdentity({ homeDir, env, - stdoutIsTTY: true, - stderr: testIo.io.stderr, + stderr: testIo.stderr, now: () => new Date('2026-05-22T14:33:02.000Z'), }); @@ -54,7 +50,7 @@ describe('telemetry identity', () => { expect(identity.installId).toMatch(/^[0-9a-f-]{36}$/); expect(identity.createdFile).toBe(true); expect(identity.noticeShown).toBe(true); - expect(testIo.stderr()).toBe(`${TELEMETRY_NOTICE}\n`); + expect(testIo.read()).toBe(`\x1b[2m${TELEMETRY_NOTICE}\x1b[22m\n`); const stored = JSON.parse(await readFile(join(homeDir, '.ktx', 'telemetry.json'), 'utf-8')) as { enabled: boolean; @@ -64,26 +60,46 @@ describe('telemetry identity', () => { expect(stored.noticeShownVersion).toBe(1); }); + it('mints an identity on a headless first run (no TTY required)', async () => { + // A fresh install whose first invocation is headless (IDE-launched + // `ktx mcp stdio`, a scripted run) must still be counted. The one-time + // notice goes to stderr, which is safe even under the MCP stdio protocol. + const testIo = makeIo(); + + const identity = await loadTelemetryIdentity({ + homeDir, + env, + stderr: testIo.stderr, + now: () => new Date('2026-05-22T14:33:02.000Z'), + }); + + expect(identity).toMatchObject({ enabled: true, createdFile: true, noticeShown: true }); + expect(identity.installId).toMatch(/^[0-9a-f-]{36}$/); + expect(testIo.read()).toBe(`\x1b[2m${TELEMETRY_NOTICE}\x1b[22m\n`); + const stored = JSON.parse(await readFile(join(homeDir, '.ktx', 'telemetry.json'), 'utf-8')) as { + enabled: boolean; + }; + expect(stored.enabled).toBe(true); + }); + it('emits the notice without ANSI when NO_COLOR is set', async () => { - const testIo = makeIo(true); + const testIo = makeIo(); await loadTelemetryIdentity({ homeDir, env: { NO_COLOR: '1' }, - stdoutIsTTY: true, - stderr: testIo.io.stderr, + stderr: testIo.stderr, now: () => new Date('2026-05-22T14:33:02.000Z'), }); - expect(testIo.stderr()).toBe(`${TELEMETRY_NOTICE}\n`); + expect(testIo.read()).toBe(`${TELEMETRY_NOTICE}\n`); }); it('does not create a file when env disables telemetry', async () => { const identity = await loadTelemetryIdentity({ homeDir, env: { KTX_TELEMETRY_DISABLED: '1' }, - stdoutIsTTY: true, - stderr: makeIo(true).io.stderr, + stderr: makeIo().stderr, now: () => new Date('2026-05-22T14:33:02.000Z'), }); @@ -91,26 +107,16 @@ describe('telemetry identity', () => { await expect(readFile(join(homeDir, '.ktx', 'telemetry.json'), 'utf-8')).rejects.toThrow(); }); - it('does not create a file for CI or non-TTY command invocations', async () => { + it('does not create a file under CI', async () => { await expect( loadTelemetryIdentity({ homeDir, env: { CI: '1' }, - stdoutIsTTY: true, - stderr: makeIo(true).io.stderr, - now: () => new Date('2026-05-22T14:33:02.000Z'), - }), - ).resolves.toMatchObject({ enabled: false, createdFile: false }); - - await expect( - loadTelemetryIdentity({ - homeDir, - env: {}, - stdoutIsTTY: false, - stderr: makeIo(false).io.stderr, + stderr: makeIo().stderr, now: () => new Date('2026-05-22T14:33:02.000Z'), }), ).resolves.toMatchObject({ enabled: false, createdFile: false }); + await expect(readFile(join(homeDir, '.ktx', 'telemetry.json'), 'utf-8')).rejects.toThrow(); }); it('honors persistent enabled false', async () => { @@ -135,8 +141,7 @@ describe('telemetry identity', () => { loadTelemetryIdentity({ homeDir, env, - stdoutIsTTY: true, - stderr: makeIo(true).io.stderr, + stderr: makeIo().stderr, now: () => new Date('2026-05-22T15:00:00.000Z'), }), ).resolves.toMatchObject({ @@ -146,6 +151,72 @@ describe('telemetry identity', () => { }); }); + it('honors a consented identity without re-showing the notice', async () => { + await mkdir(join(homeDir, '.ktx'), { recursive: true }); + await writeFile( + join(homeDir, '.ktx', 'telemetry.json'), + JSON.stringify( + { + installId: '00000000-0000-4000-8000-000000000000', + enabled: true, + noticeShownAt: '2026-05-22T14:33:02.000Z', + noticeShownVersion: 1, + createdAt: '2026-05-22T14:33:02.000Z', + }, + null, + 2, + ) + '\n', + 'utf-8', + ); + const testIo = makeIo(); + + await expect( + loadTelemetryIdentity({ + homeDir, + env, + stderr: testIo.stderr, + now: () => new Date('2026-05-22T15:00:00.000Z'), + }), + ).resolves.toMatchObject({ + installId: '00000000-0000-4000-8000-000000000000', + enabled: true, + createdFile: false, + noticeShown: false, + }); + // An already-consented identity must not re-emit the one-time notice. + expect(testIo.read()).toBe(''); + }); + + it('keeps opt-outs suppressing a consented identity', async () => { + await mkdir(join(homeDir, '.ktx'), { recursive: true }); + await writeFile( + join(homeDir, '.ktx', 'telemetry.json'), + JSON.stringify( + { + installId: '00000000-0000-4000-8000-000000000000', + enabled: true, + noticeShownAt: '2026-05-22T14:33:02.000Z', + noticeShownVersion: 1, + createdAt: '2026-05-22T14:33:02.000Z', + }, + null, + 2, + ) + '\n', + 'utf-8', + ); + + for (const optOut of [{ KTX_TELEMETRY_DISABLED: '1' }, { DO_NOT_TRACK: '1' }, { CI: '1' }]) { + await expect( + loadTelemetryIdentity({ + homeDir, + env: optOut, + stderr: makeIo().stderr, + now: () => new Date('2026-05-22T15:00:00.000Z'), + }), + ).resolves.toMatchObject({ enabled: false }); + } + }); + it('recreates a corrupted file instead of surfacing an error to users', async () => { await mkdir(join(homeDir, '.ktx'), { recursive: true }); await writeFile(join(homeDir, '.ktx', 'telemetry.json'), '{bad json', 'utf-8'); @@ -153,8 +224,7 @@ describe('telemetry identity', () => { const identity = await loadTelemetryIdentity({ homeDir, env, - stdoutIsTTY: true, - stderr: makeIo(true).io.stderr, + stderr: makeIo().stderr, now: () => new Date('2026-05-22T14:33:02.000Z'), }); diff --git a/packages/cli/test/telemetry/index.test.ts b/packages/cli/test/telemetry/index.test.ts new file mode 100644 index 00000000..3531116a --- /dev/null +++ b/packages/cli/test/telemetry/index.test.ts @@ -0,0 +1,155 @@ +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createGlobalExceptionReporter, type KtxCliIo } from '../../src/cli-runtime.js'; +import { beginCommandSpan, emitAbortedCommandAndShutdown, emitTelemetryEvent } from '../../src/telemetry/index.js'; +import { resetCommandSpan } from '../../src/telemetry/command-hook.js'; + +function makeIo(): { io: KtxCliIo; stderr: () => string } { + let stderr = ''; + return { + io: { + stdout: { + isTTY: true, + write: () => {}, + }, + stderr: { + write: (chunk) => { + stderr += chunk; + }, + }, + }, + stderr: () => stderr, + }; +} + +describe('emitTelemetryEvent', () => { + let homeDir: string; + + beforeEach(async () => { + homeDir = await mkdtemp(join(tmpdir(), 'ktx-telemetry-index-')); + vi.stubEnv('HOME', homeDir); + }); + + afterEach(async () => { + vi.unstubAllEnvs(); + await rm(homeDir, { recursive: true, force: true }); + }); + + it('prints debug telemetry when live telemetry is disabled without creating an identity file', async () => { + vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); + vi.stubEnv('KTX_TELEMETRY_DISABLED', '1'); + vi.stubEnv('DO_NOT_TRACK', '1'); + const testIo = makeIo(); + const projectDir = join(homeDir, 'private-project'); + + await emitTelemetryEvent({ + name: 'connection_added', + projectDir, + io: testIo.io, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + fields: { + driver: 'sqlite', + isDemoConnection: false, + }, + }); + + expect(testIo.stderr()).toContain('[telemetry]'); + expect(testIo.stderr()).toContain('"event":"connection_added"'); + expect(testIo.stderr()).not.toContain(projectDir); + await expect(readFile(join(homeDir, '.ktx', 'telemetry.json'), 'utf-8')).rejects.toThrow(); + }); +}); + +describe('emitAbortedCommandAndShutdown', () => { + let homeDir: string; + + beforeEach(async () => { + homeDir = await mkdtemp(join(tmpdir(), 'ktx-telemetry-abort-')); + vi.stubEnv('HOME', homeDir); + vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); + vi.stubEnv('CI', ''); + vi.stubEnv('KTX_TELEMETRY_DISABLED', ''); + vi.stubEnv('DO_NOT_TRACK', ''); + resetCommandSpan(); + }); + + afterEach(async () => { + resetCommandSpan(); + vi.unstubAllEnvs(); + await rm(homeDir, { recursive: true, force: true }); + }); + + it('flushes the active command span as aborted (the signal path)', async () => { + const testIo = makeIo(); + beginCommandSpan({ + commandPath: ['ktx', 'ingest'], + flagsPresent: {}, + hasProject: true, + attachProjectGroup: false, + startedAt: performance.now(), + }); + + await emitAbortedCommandAndShutdown({ + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, + io: testIo.io, + }); + + expect(testIo.stderr()).toContain('"event":"command"'); + expect(testIo.stderr()).toContain('"outcome":"aborted"'); + expect(testIo.stderr()).toContain('"commandPath":["ktx","ingest"]'); + }); + + it('is idempotent: a second call (or no active span) emits nothing', async () => { + const testIo = makeIo(); + beginCommandSpan({ + commandPath: ['ktx', 'ingest'], + flagsPresent: {}, + hasProject: true, + attachProjectGroup: false, + startedAt: performance.now(), + }); + const pkg = { name: '@kaelio/ktx', version: '0.0.0-test' }; + + await emitAbortedCommandAndShutdown({ packageInfo: pkg, io: testIo.io }); + const secondIo = makeIo(); + await emitAbortedCommandAndShutdown({ packageInfo: pkg, io: secondIo.io }); + + expect(secondIo.stderr()).not.toContain('"event":"command"'); + }); +}); + +describe('global exception reporting contract', () => { + let homeDir: string; + + beforeEach(async () => { + homeDir = await mkdtemp(join(tmpdir(), 'ktx-telemetry-global-exception-')); + vi.stubEnv('HOME', homeDir); + vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); + vi.stubEnv('KTX_TELEMETRY_DISABLED', '1'); + vi.stubEnv('DO_NOT_TRACK', ''); + vi.stubEnv('CI', ''); + }); + + afterEach(async () => { + vi.unstubAllEnvs(); + await rm(homeDir, { recursive: true, force: true }); + }); + + it('reports uncaughtException through the fatal debug payload', async () => { + const testIo = makeIo(); + const report = createGlobalExceptionReporter(testIo.io, { + name: '@kaelio/ktx', + version: '0.0.0-test', + }); + + await report('uncaughtException', new Error('global boom')); + + expect(testIo.stderr()).toContain('[telemetry-exception]'); + expect(testIo.stderr()).toContain('"source":"uncaughtException"'); + expect(testIo.stderr()).toContain('"handled":false'); + expect(testIo.stderr()).toContain('"fatal":true'); + }); +}); diff --git a/packages/cli/src/telemetry/project-snapshot.test.ts b/packages/cli/test/telemetry/project-snapshot.test.ts similarity index 86% rename from packages/cli/src/telemetry/project-snapshot.test.ts rename to packages/cli/test/telemetry/project-snapshot.test.ts index a1c06472..ce58f40e 100644 --- a/packages/cli/src/telemetry/project-snapshot.test.ts +++ b/packages/cli/test/telemetry/project-snapshot.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { buildProjectStackSnapshotFields } from './project-snapshot.js'; +import { buildProjectStackSnapshotFields } from '../../src/telemetry/project-snapshot.js'; describe('buildProjectStackSnapshotFields', () => { let projectDir: string; @@ -34,6 +34,18 @@ describe('buildProjectStackSnapshotFields', () => { adapters: [], embeddings: { backend: 'sentence-transformers', dimensions: 384 }, workUnits: { stepBudget: 40, maxConcurrency: 1, failureMode: 'continue' }, + rateLimit: { + enabled: true, + throttleThreshold: 0.8, + minConcurrencyUnderPressure: 1, + retry: { + maxAttempts: 6, + baseDelayMs: 1_000, + maxDelayMs: 60_000, + jitter: true, + }, + }, + profile: false, }, llm: { provider: { backend: 'none' }, models: {}, promptCaching: {} }, scan: { diff --git a/packages/cli/test/telemetry/redaction-secrets.test.ts b/packages/cli/test/telemetry/redaction-secrets.test.ts new file mode 100644 index 00000000..cdc15f22 --- /dev/null +++ b/packages/cli/test/telemetry/redaction-secrets.test.ts @@ -0,0 +1,127 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { parseKtxProjectConfig, serializeKtxProjectConfig } from '../../src/context/project/config.js'; +import { initKtxProject } from '../../src/context/project/project.js'; +import { collectTelemetryRedactionSecrets } from '../../src/telemetry/redaction-secrets.js'; + +describe('collectTelemetryRedactionSecrets', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-redaction-secrets-')); + }); + + afterEach(async () => { + vi.unstubAllEnvs(); + await rm(tempDir, { recursive: true, force: true }); + }); + + async function writeConfig(projectDir: string): Promise { + const configPath = join(projectDir, 'ktx.yaml'); + const config = parseKtxProjectConfig(await readFile(configPath, 'utf-8')); + await writeFile( + configPath, + serializeKtxProjectConfig({ + ...config, + llm: { + ...config.llm, + provider: { + backend: 'anthropic', + anthropic: { api_key: 'env:ANTHROPIC_API_KEY' }, // pragma: allowlist secret + }, + models: { default: 'claude-sonnet-4-6' }, + }, + ingest: { + ...config.ingest, + embeddings: { + backend: 'openai', + model: 'text-embedding-3-small', + dimensions: 1536, + openai: { api_key: 'file:~/.ktx/secrets/openai-key' }, // pragma: allowlist secret + }, + }, + scan: { + ...config.scan, + enrichment: { + ...config.scan.enrichment, + embeddings: { + backend: 'openai', + model: 'text-embedding-3-small', + dimensions: 1536, + openai: { api_key: 'env:SCAN_OPENAI_API_KEY' }, // pragma: allowlist secret + }, + }, + }, + connections: { + warehouse: { + driver: 'postgres', + url: 'env:DATABASE_URL', + password: 'file:~/.ktx/secrets/db-password', // pragma: allowlist secret + }, + docs: { + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', // pragma: allowlist secret + }, + }, + }), + 'utf-8', + ); + } + + it('derives only declared project secrets and parsed URL credentials', async () => { + const homeDir = join(tempDir, 'home'); + const projectDir = join(tempDir, 'project'); + await mkdir(join(homeDir, '.ktx', 'secrets'), { recursive: true }); + await writeFile(join(homeDir, '.ktx', 'secrets', 'openai-key'), 'openai-file-secret\n', 'utf-8'); + await writeFile(join(homeDir, '.ktx', 'secrets', 'db-password'), 'db-file-password\n', 'utf-8'); + vi.stubEnv('HOME', homeDir); + vi.stubEnv('ANTHROPIC_API_KEY', 'anthropic-env-secret'); + vi.stubEnv('SCAN_OPENAI_API_KEY', 'scan-openai-env-secret'); + vi.stubEnv('DATABASE_URL', 'postgres://svc:db-url-password@db.example.test/analytics'); // pragma: allowlist secret + vi.stubEnv('NOTION_TOKEN', 'notion-env-secret'); + vi.stubEnv('UNDECLARED_SECRET', 'must-not-appear'); + await initKtxProject({ projectDir }); + await writeConfig(projectDir); + + const secrets = await collectTelemetryRedactionSecrets({ + projectDir, + connectionId: 'warehouse', + includeLlm: true, + includeEmbeddings: true, + env: process.env, + }); + + expect(secrets).toEqual( + expect.arrayContaining([ + 'anthropic-env-secret', + 'openai-file-secret', + 'scan-openai-env-secret', + 'postgres://svc:db-url-password@db.example.test/analytics', // pragma: allowlist secret + 'db-url-password', + 'db-file-password', + ]), + ); + expect(secrets).not.toContain('notion-env-secret'); + expect(secrets).not.toContain('must-not-appear'); + }); + + it('can derive a named non-database connection secret', async () => { + const projectDir = join(tempDir, 'project'); + vi.stubEnv('NOTION_TOKEN', 'notion-env-secret'); + await initKtxProject({ projectDir }); + await writeConfig(projectDir); + + const secrets = await collectTelemetryRedactionSecrets({ + projectDir, + connectionId: 'docs', + includeLlm: false, + includeEmbeddings: false, + env: process.env, + }); + + expect(secrets).toEqual(['notion-env-secret']); + }); +}); diff --git a/packages/cli/src/telemetry/schema-writer.test.ts b/packages/cli/test/telemetry/schema-writer.test.ts similarity index 90% rename from packages/cli/src/telemetry/schema-writer.test.ts rename to packages/cli/test/telemetry/schema-writer.test.ts index a6539421..498869f9 100644 --- a/packages/cli/src/telemetry/schema-writer.test.ts +++ b/packages/cli/test/telemetry/schema-writer.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { buildTelemetrySchemaArtifact } from './schema-writer.js'; +import { buildTelemetrySchemaArtifact } from '../../src/telemetry/schema-writer.js'; describe('telemetry schema writer', () => { it('exports a schema artifact with the full catalog and strict metadata', () => { diff --git a/packages/cli/test/telemetry/scrubber.test.ts b/packages/cli/test/telemetry/scrubber.test.ts new file mode 100644 index 00000000..a6914665 --- /dev/null +++ b/packages/cli/test/telemetry/scrubber.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from 'vitest'; + +import { formatErrorDetail, scrubErrorClass } from '../../src/telemetry/scrubber.js'; + +class KtxProjectMissingAbortError extends Error {} + +describe('scrubErrorClass', () => { + it('keeps normal JavaScript class names', () => { + expect(scrubErrorClass(new KtxProjectMissingAbortError('missing'))).toBe('KtxProjectMissingAbortError'); + }); + + it('drops path-like, URL-like, email-like, and long values', () => { + expect(scrubErrorClass({ constructor: { name: '/Users/alice/project' } })).toBeUndefined(); + expect(scrubErrorClass({ constructor: { name: 'https://example.test/error' } })).toBeUndefined(); + expect(scrubErrorClass({ constructor: { name: 'alice@example.test' } })).toBeUndefined(); + expect(scrubErrorClass({ constructor: { name: 'A'.repeat(81) } })).toBeUndefined(); + }); + + it('drops lowercase, spaced, and non-error-like values', () => { + expect(scrubErrorClass({ constructor: { name: 'lowercaseError' } })).toBeUndefined(); + expect(scrubErrorClass({ constructor: { name: 'Bad Error' } })).toBeUndefined(); + expect(scrubErrorClass('plain string')).toBeUndefined(); + expect(scrubErrorClass(null)).toBeUndefined(); + }); +}); + +describe('formatErrorDetail', () => { + it('prefixes a string or numeric .code onto the message', () => { + const refused = new Error('connect failed'); + (refused as { code?: unknown }).code = 'ECONNREFUSED'; + expect(formatErrorDetail(refused)).toBe('ECONNREFUSED: connect failed'); + + const forbidden = new Error('forbidden'); + (forbidden as { code?: unknown }).code = 403; + expect(formatErrorDetail(forbidden)).toBe('403: forbidden'); + }); + + it('uses the bare message when there is no .code', () => { + expect(formatErrorDetail(new Error('password authentication failed for user "x"'))).toBe( + 'password authentication failed for user "x"', + ); + }); + + it('accepts non-Error values', () => { + expect(formatErrorDetail('boom')).toBe('boom'); + }); + + it('collapses whitespace to a single line', () => { + expect(formatErrorDetail(new Error('line one\n line two'))).toBe('line one line two'); + }); + + it('caps the length at 1000 characters', () => { + expect(formatErrorDetail(new Error('x'.repeat(2000)))?.length).toBe(1000); + }); + + it('returns undefined for empty, null, or undefined input', () => { + expect(formatErrorDetail(new Error(' '))).toBeUndefined(); + expect(formatErrorDetail(null)).toBeUndefined(); + expect(formatErrorDetail(undefined)).toBeUndefined(); + }); +}); diff --git a/packages/cli/src/text-ingest.test.ts b/packages/cli/test/text-ingest.test.ts similarity index 97% rename from packages/cli/src/text-ingest.test.ts rename to packages/cli/test/text-ingest.test.ts index b7737f36..5122208e 100644 --- a/packages/cli/src/text-ingest.test.ts +++ b/packages/cli/test/text-ingest.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { MemoryIngestStatus } from './context/memory/memory-runs.js'; -import type { KtxLocalProject } from './context/project/project.js'; -import { runKtxTextIngest, type TextMemoryIngestPort } from './text-ingest.js'; +import type { MemoryIngestStatus } from '../src/context/memory/memory-runs.js'; +import type { KtxLocalProject } from '../src/context/project/project.js'; +import { runKtxTextIngest, type TextMemoryIngestPort } from '../src/text-ingest.js'; function makeIo(options: { isTTY?: boolean } = {}) { let stdout = ''; diff --git a/packages/cli/src/tree-picker-state.test.ts b/packages/cli/test/tree-picker-state.test.ts similarity index 88% rename from packages/cli/src/tree-picker-state.test.ts rename to packages/cli/test/tree-picker-state.test.ts index 20c2001e..50d47d89 100644 --- a/packages/cli/src/tree-picker-state.test.ts +++ b/packages/cli/test/tree-picker-state.test.ts @@ -14,7 +14,7 @@ import { toggleChecked, visibleNodeIds, type TreePickerNodeInput, -} from './tree-picker-state.js'; +} from '../src/tree-picker-state.js'; const IDS = { engineering: '11111111-1111-1111-1111-111111111111', @@ -191,7 +191,7 @@ describe('search and cursor movement', () => { }); const searching = { ...state, - search: { editing: false, query: 'architecture' }, + search: { query: 'architecture' }, }; expect(filterTree(searching)).toEqual({ @@ -229,7 +229,7 @@ describe('bulk actions and reducer effects', () => { }); const searching = { ...state, - search: { editing: false, query: 'architecture' }, + search: { query: 'architecture' }, }; const selected = selectAllVisible(searching); @@ -306,12 +306,11 @@ describe('bulk actions and reducer effects', () => { next: { ...state, pendingConfirm: null }, effect: null, }); + expect(reducer(state, { type: 'search-input', value: 'a' }).next.search).toEqual({ query: 'a' }); + expect(reducer({ ...state, isNavigating: true }, { type: 'search-input', value: 'b' }).next.isNavigating).toBe(false); expect( - reducer( - reducer(reducer(state, 'search-start').next, { type: 'search-input', value: 'a' }).next, - 'search-submit', - ).next.search, - ).toEqual({ editing: false, query: 'a' }); + reducer({ ...state, search: { query: 'foo' }, isNavigating: true }, 'search-clear').next, + ).toEqual({ ...state, search: { query: '' }, isNavigating: false }); expect(reducer(state, 'quit')).toEqual({ next: state, effect: 'quit-without-save', @@ -336,6 +335,34 @@ describe('bulk actions and reducer effects', () => { }); }); + it('navigates cursor commands set isNavigating, typed input clears it, and search refocuses cursor', () => { + const state = buildInitialState({ + tree: buildPickerTree(pages()), + existingSelectedIds: [], + }); + + expect(state.isNavigating).toBe(false); + const afterDown = reducer(state, 'cursor-down').next; + expect(afterDown.isNavigating).toBe(true); + + const afterType = reducer(afterDown, { type: 'search-input', value: 'a' }).next; + expect(afterType.isNavigating).toBe(false); + expect(afterType.search.query).toBe('a'); + + const afterBackspace = reducer({ ...afterDown, search: { query: 'foo' } }, 'search-backspace').next; + expect(afterBackspace.search.query).toBe('fo'); + expect(afterBackspace.isNavigating).toBe(false); + + const withCursorOnHidden = { + ...state, + cursorId: IDS.journal, + search: { query: 'arch' }, + }; + const refocused = reducer(withCursorOnHidden, { type: 'search-input', value: 'i' }).next; + expect(refocused.search.query).toBe('archi'); + expect(visibleNodeIds(refocused)).toContain(refocused.cursorId); + }); + it('clears transient hints only when their expiry time has passed', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), diff --git a/packages/cli/src/tree-picker-tui.test.tsx b/packages/cli/test/tree-picker-tui.test.tsx similarity index 76% rename from packages/cli/src/tree-picker-tui.test.tsx rename to packages/cli/test/tree-picker-tui.test.tsx index 3a8dcc7b..f4f642c0 100644 --- a/packages/cli/src/tree-picker-tui.test.tsx +++ b/packages/cli/test/tree-picker-tui.test.tsx @@ -2,7 +2,7 @@ import { render as renderInkTest } from 'ink-testing-library'; import { type ReactNode } from 'react'; import { describe, expect, it, vi } from 'vitest'; -import { buildInitialState, buildPickerTree, type TreePickerNodeInput } from './tree-picker-state.js'; +import { buildInitialState, buildPickerTree, type TreePickerNodeInput } from '../src/tree-picker-state.js'; import { TreePickerApp, renderTreePickerTui, @@ -14,7 +14,7 @@ import { type TreePickerChrome, type TreePickerInkInstance, type TreePickerInkRenderOptions, -} from './tree-picker-tui.js'; +} from '../src/tree-picker-tui.js'; const IDS = { engineering: '11111111-1111-1111-1111-111111111111', @@ -83,38 +83,71 @@ function normalizeFrameWrap(frame: string | undefined): string { } describe('treePickerCommandForInkInput', () => { - it('maps browse, search, and confirm input to reducer commands', () => { - expect(treePickerCommandForInkInput('', { downArrow: true }, state().search, null)).toBe('cursor-down'); - expect(treePickerCommandForInkInput('', { upArrow: true }, state().search, null)).toBe('cursor-up'); - expect(treePickerCommandForInkInput('', { rightArrow: true }, state().search, null)).toBe('cursor-right'); - expect(treePickerCommandForInkInput('', { leftArrow: true }, state().search, null)).toBe('cursor-left'); - expect(treePickerCommandForInkInput(' ', {}, state().search, null)).toBe('toggle-check'); - expect(treePickerCommandForInkInput('/', {}, state().search, null)).toBe('search-start'); - expect(treePickerCommandForInkInput('a', {}, state().search, null)).toBe('toggle-select-all-visible'); - expect(treePickerCommandForInkInput('n', {}, state().search, null)).toBe('select-none'); - expect(treePickerCommandForInkInput('', { return: true }, state().search, null)).toBe('save-request'); - expect(treePickerCommandForInkInput('', { escape: true }, state().search, null)).toBe('quit'); - expect(treePickerCommandForInkInput('c', { ctrl: true }, state().search, null)).toBe('quit'); - expect(treePickerCommandForInkInput('s', {}, state().search, null)).toBeNull(); - expect(treePickerCommandForInkInput('q', {}, state().search, null)).toBeNull(); + const browse = (overrides: Partial<{ search: { query: string }; isNavigating: boolean; pendingConfirm: null }> = {}) => ({ + search: { query: '' }, + isNavigating: false, + pendingConfirm: null, + ...overrides, + }); + const confirming = { ...browse(), pendingConfirm: 'save-confirm' as const }; - expect(treePickerCommandForInkInput('x', {}, { editing: true, query: '' }, null)).toEqual({ + it('routes cursor and confirm keys when no query is typed', () => { + expect(treePickerCommandForInkInput('', { downArrow: true }, browse())).toBe('cursor-down'); + expect(treePickerCommandForInkInput('', { upArrow: true }, browse())).toBe('cursor-up'); + expect(treePickerCommandForInkInput('', { rightArrow: true }, browse())).toBe('cursor-right'); + expect(treePickerCommandForInkInput('', { leftArrow: true }, browse())).toBe('cursor-left'); + expect(treePickerCommandForInkInput('', { return: true }, browse())).toBe('save-request'); + expect(treePickerCommandForInkInput('', { escape: true }, browse())).toBe('quit'); + expect(treePickerCommandForInkInput('c', { ctrl: true }, browse())).toBe('quit'); + }); + + it('Tab toggles selection regardless of search/navigation state', () => { + expect(treePickerCommandForInkInput('', { tab: true }, browse())).toBe('toggle-check'); + expect(treePickerCommandForInkInput('', { tab: true }, browse({ search: { query: 'foo' }, isNavigating: false }))).toBe( + 'toggle-check', + ); + expect(treePickerCommandForInkInput('', { tab: true }, browse({ isNavigating: true }))).toBe('toggle-check'); + }); + + it('Space toggles only when navigating; otherwise typed into the search query', () => { + expect(treePickerCommandForInkInput(' ', {}, browse({ isNavigating: true }))).toBe('toggle-check'); + expect(treePickerCommandForInkInput(' ', {}, browse({ isNavigating: false }))).toEqual({ + type: 'search-input', + value: ' ', + }); + }); + + it('typed printable chars feed the search query — including a, n, and slash', () => { + expect(treePickerCommandForInkInput('a', {}, browse())).toEqual({ type: 'search-input', value: 'a' }); + expect(treePickerCommandForInkInput('n', {}, browse())).toEqual({ type: 'search-input', value: 'n' }); + expect(treePickerCommandForInkInput('/', {}, browse())).toEqual({ type: 'search-input', value: '/' }); + expect(treePickerCommandForInkInput('x', {}, browse({ search: { query: 'foo' } }))).toEqual({ type: 'search-input', value: 'x', }); - expect(treePickerCommandForInkInput('', { backspace: true }, { editing: true, query: 'x' }, null)).toBe( + }); + + it('Ctrl+A and Ctrl+N drive the bulk toggle helpers', () => { + expect(treePickerCommandForInkInput('a', { ctrl: true }, browse())).toBe('toggle-select-all-visible'); + expect(treePickerCommandForInkInput('n', { ctrl: true }, browse())).toBe('select-none'); + }); + + it('Backspace deletes from the query at any time; Esc clears query first then quits', () => { + expect(treePickerCommandForInkInput('', { backspace: true }, browse({ search: { query: 'x' } }))).toBe( 'search-backspace', ); - expect(treePickerCommandForInkInput('', { return: true }, { editing: true, query: 'x' }, null)).toBe( - 'search-submit', - ); - expect(treePickerCommandForInkInput('', { escape: true }, { editing: true, query: 'x' }, null)).toBe( - 'search-cancel', + expect(treePickerCommandForInkInput('', { delete: true }, browse({ search: { query: 'x' } }))).toBe( + 'search-backspace', ); + expect(treePickerCommandForInkInput('', { escape: true }, browse({ search: { query: 'x' } }))).toBe('search-clear'); + expect(treePickerCommandForInkInput('', { escape: true }, browse())).toBe('quit'); + }); - expect(treePickerCommandForInkInput('y', {}, state().search, 'save-confirm')).toBe('save-confirm'); - expect(treePickerCommandForInkInput('', { return: true }, state().search, 'save-confirm')).toBe('save-confirm'); - expect(treePickerCommandForInkInput('n', {}, state().search, 'save-confirm')).toBe('save-cancel'); + it('confirm prompts intercept y/n/Enter/Esc before search routing', () => { + expect(treePickerCommandForInkInput('y', {}, confirming)).toBe('save-confirm'); + expect(treePickerCommandForInkInput('', { return: true }, confirming)).toBe('save-confirm'); + expect(treePickerCommandForInkInput('n', {}, confirming)).toBe('save-cancel'); + expect(treePickerCommandForInkInput('', { escape: true }, confirming)).toBe('save-cancel'); }); }); @@ -160,8 +193,9 @@ describe('TreePickerApp', () => { expect(frame).toContain('◻ Engineering Docs ▸ (1)'); expect(frame).toContain('◻ Marketing'); expect(normalizeFrameWrap(frame)).toContain( - 'Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape to go back, or Ctrl+C to exit.', + 'Up/Down to move, Right/Left to expand or collapse, Tab to select, Type to search, Enter to confirm, Escape to clear search or go back, Ctrl+C to exit.', ); + expect(frame).toContain('Search:'); }); it('renders custom help text when supplied', () => { @@ -238,7 +272,7 @@ describe('TreePickerApp', () => { />, ); - stdin.write(' '); + stdin.write('\t'); await waitForInkInput(); expect(lastFrame()).toContain('◼ Engineering Docs'); diff --git a/packages/cli/test/update-check/cache.test.ts b/packages/cli/test/update-check/cache.test.ts new file mode 100644 index 00000000..446a62be --- /dev/null +++ b/packages/cli/test/update-check/cache.test.ts @@ -0,0 +1,95 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { + readUpdateCheckCache, + updateCheckCachePath, + writeUpdateCheckCache, +} from '../../src/update-check/cache.js'; + +describe('update-check cache', () => { + let homeDir: string; + + beforeEach(async () => { + homeDir = await mkdtemp(join(tmpdir(), 'ktx-update-check-cache-')); + }); + + afterEach(async () => { + await rm(homeDir, { recursive: true, force: true }); + }); + + it('uses ~/.ktx/update-check.json', () => { + expect(updateCheckCachePath(homeDir)).toBe(join(homeDir, '.ktx', 'update-check.json')); + }); + + it('round-trips strict cache data', async () => { + await writeUpdateCheckCache( + { + checkedAt: '2026-06-06T10:00:00.000Z', + channel: 'latest', + installedVersion: '0.9.0', + latestForChannel: '0.10.0', + lastNoticeAt: '2026-06-06T11:00:00.000Z', + }, + { homeDir }, + ); + + await expect(readUpdateCheckCache({ homeDir })).resolves.toEqual({ + checkedAt: '2026-06-06T10:00:00.000Z', + channel: 'latest', + installedVersion: '0.9.0', + latestForChannel: '0.10.0', + lastNoticeAt: '2026-06-06T11:00:00.000Z', + }); + }); + + it('returns null when the cache file is missing', async () => { + await expect(readUpdateCheckCache({ homeDir })).resolves.toBeNull(); + }); + + it('returns null when the cache file is corrupt JSON', async () => { + await mkdir(join(homeDir, '.ktx'), { recursive: true }); + await writeFile(updateCheckCachePath(homeDir), '{bad json', 'utf-8'); + + await expect(readUpdateCheckCache({ homeDir })).resolves.toBeNull(); + }); + + it('returns null when the cache has unknown fields', async () => { + await mkdir(join(homeDir, '.ktx'), { recursive: true }); + await writeFile( + updateCheckCachePath(homeDir), + JSON.stringify( + { + checkedAt: '2026-06-06T10:00:00.000Z', + channel: 'latest', + installedVersion: '0.9.0', + latestForChannel: '0.10.0', + unexpected: true, + }, + null, + 2, + ), + 'utf-8', + ); + + await expect(readUpdateCheckCache({ homeDir })).resolves.toBeNull(); + }); + + it('writes formatted JSON with a trailing newline', async () => { + await writeUpdateCheckCache( + { + checkedAt: '2026-06-06T10:00:00.000Z', + channel: 'next', + installedVersion: '0.10.0-rc.1', + latestForChannel: '0.10.0-rc.2', + }, + { homeDir }, + ); + + const raw = await readFile(updateCheckCachePath(homeDir), 'utf-8'); + expect(raw).toContain('"channel": "next"'); + expect(raw.endsWith('\n')).toBe(true); + }); +}); diff --git a/packages/cli/test/update-check/channel.test.ts b/packages/cli/test/update-check/channel.test.ts new file mode 100644 index 00000000..f7b4a1e6 --- /dev/null +++ b/packages/cli/test/update-check/channel.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from 'vitest'; + +import { decideUpdate, inferUpdateChannel } from '../../src/update-check/channel.js'; + +describe('inferUpdateChannel', () => { + it.each([ + ['0.9.0', 'latest'], + ['0.10.0-rc.3', 'next'], + ['0.10.0-myfeat.2', null], + ['0.0.0', null], + ['not-a-version', null], + ])('maps %s to %s', (installed, expected) => { + expect(inferUpdateChannel(installed)).toBe(expected); + }); +}); + +describe('decideUpdate', () => { + it.each([ + [ + 'stable behind', + '0.9.0', + { latest: '0.10.0', next: '0.11.0-rc.1' }, + { status: 'available', channel: 'latest', target: '0.10.0' }, + ], + [ + 'stable equal', + '0.10.0', + { latest: '0.10.0', next: '0.11.0-rc.1' }, + { status: 'upToDate', channel: 'latest', target: '0.10.0' }, + ], + [ + 'stable ahead', + '0.11.0', + { latest: '0.10.0', next: '0.11.0-rc.1' }, + { status: 'upToDate', channel: 'latest', target: '0.10.0' }, + ], + [ + 'rc behind', + '0.11.0-rc.1', + { latest: '0.10.0', next: '0.11.0-rc.2' }, + { status: 'available', channel: 'next', target: '0.11.0-rc.2' }, + ], + [ + 'rc equal', + '0.11.0-rc.2', + { latest: '0.10.0', next: '0.11.0-rc.2' }, + { status: 'upToDate', channel: 'next', target: '0.11.0-rc.2' }, + ], + ['branch prerelease', '0.11.0-myfeat.1', { latest: '0.10.0', next: '0.11.0-rc.2' }, { status: 'skip' }], + ['missing channel tag', '0.9.0', { next: '0.11.0-rc.2' }, { status: 'skip' }], + ['invalid installed version', 'bad', { latest: '0.10.0' }, { status: 'skip' }], + ['invalid target version', '0.9.0', { latest: 'bad' }, { status: 'skip' }], + ['local development version', '0.0.0', { latest: '0.10.0' }, { status: 'skip' }], + ])('%s', (_name, installed, distTags, expected) => { + expect(decideUpdate(installed, distTags)).toEqual(expected); + }); +}); diff --git a/packages/cli/test/update-check/cli-program.test.ts b/packages/cli/test/update-check/cli-program.test.ts new file mode 100644 index 00000000..78116f97 --- /dev/null +++ b/packages/cli/test/update-check/cli-program.test.ts @@ -0,0 +1,152 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { buildKtxProgram } from '../../src/cli-program.js'; +import type { KtxCliDeps, KtxCliIo, KtxCliPackageInfo } from '../../src/cli-runtime.js'; +import { updateCheckCachePath } from '../../src/update-check/cache.js'; + +function makeIo(stdoutIsTTY = true): { io: KtxCliIo; stdout: () => string; stderr: () => string } { + let stdout = ''; + let stderr = ''; + return { + io: { + stdout: { + isTTY: stdoutIsTTY, + write: (chunk) => { + stdout += chunk; + }, + }, + stderr: { + write: (chunk) => { + stderr += chunk; + }, + }, + }, + stdout: () => stdout, + stderr: () => stderr, + }; +} + +describe('cli-program update check hooks', () => { + let projectDir: string; + let homeDir: string; + const info: KtxCliPackageInfo = { name: '@kaelio/ktx', version: '0.9.0' }; + + beforeEach(async () => { + projectDir = await mkdtemp(join(tmpdir(), 'ktx-update-project-')); + homeDir = await mkdtemp(join(tmpdir(), 'ktx-update-home-')); + await writeFile(join(projectDir, 'ktx.yaml'), '{}\n', 'utf-8'); + await mkdir(join(homeDir, '.ktx'), { recursive: true }); + vi.stubEnv('KTX_TELEMETRY_DISABLED', '1'); + vi.stubEnv('CI', ''); + vi.stubEnv('DO_NOT_TRACK', ''); + }); + + afterEach(async () => { + vi.unstubAllEnvs(); + await rm(projectDir, { recursive: true, force: true }); + await rm(homeDir, { recursive: true, force: true }); + }); + + it('prints a stale-cache notice without awaiting the background refresh', async () => { + await writeFile( + updateCheckCachePath(homeDir), + JSON.stringify( + { + checkedAt: '2026-06-05T10:00:00.000Z', + channel: 'latest', + installedVersion: '0.9.0', + latestForChannel: '0.10.0', + }, + null, + 2, + ), + 'utf-8', + ); + const io = makeIo(true); + const deps: KtxCliDeps = { doctor: async () => 0 }; + const fetchDistTags = vi.fn( + () => + new Promise>(() => { + return; + }), + ); + const program = buildKtxProgram({ + io: io.io, + deps, + packageInfo: info, + runInit: async () => 0, + updateCheck: { + env: { NO_COLOR: '1' }, + fetchDistTags, + homeDir, + now: () => new Date('2026-06-06T12:00:00.000Z'), + }, + }); + + await program.parseAsync(['--project-dir', projectDir, 'status'], { from: 'user' }); + + expect(fetchDistTags).toHaveBeenCalledTimes(1); + expect(io.stderr()).toContain('↑ Update available: ktx 0.9.0 → 0.10.0\n npm i -g @kaelio/ktx\n'); + }); + + it('prints a queued fresh-cache notice after the action', async () => { + await writeFile( + updateCheckCachePath(homeDir), + JSON.stringify( + { + checkedAt: '2026-06-06T11:00:00.000Z', + channel: 'latest', + installedVersion: '0.9.0', + latestForChannel: '0.10.0', + }, + null, + 2, + ), + 'utf-8', + ); + const io = makeIo(true); + const fetchDistTags = vi.fn(async () => ({ latest: '0.10.0' })); + const program = buildKtxProgram({ + io: io.io, + deps: { doctor: async () => 0 }, + packageInfo: info, + runInit: async () => 0, + updateCheck: { + env: { NO_COLOR: '1' }, + fetchDistTags, + homeDir, + now: () => new Date('2026-06-06T12:00:00.000Z'), + }, + }); + + await program.parseAsync(['--project-dir', projectDir, 'status'], { from: 'user' }); + + expect(fetchDistTags).not.toHaveBeenCalled(); + expect(io.stderr()).toContain('↑ Update available: ktx 0.9.0 → 0.10.0\n npm i -g @kaelio/ktx\n'); + }); + + it('does not run update checks for the hidden completion command', async () => { + const io = makeIo(true); + const fetchDistTags = vi.fn(async () => ({ latest: '0.10.0' })); + const program = buildKtxProgram({ + io: io.io, + deps: {}, + packageInfo: info, + runInit: async () => 0, + updateCheck: { + env: { NO_COLOR: '1' }, + fetchDistTags, + homeDir, + now: () => new Date('2026-06-06T12:00:00.000Z'), + }, + }); + + await program.parseAsync(['__complete', '--', 'ktx', 'co'], { from: 'user' }); + + expect(fetchDistTags).not.toHaveBeenCalled(); + expect(io.stderr()).not.toContain('Update available'); + }); +}); diff --git a/packages/cli/test/update-check/registry.test.ts b/packages/cli/test/update-check/registry.test.ts new file mode 100644 index 00000000..a83d360d --- /dev/null +++ b/packages/cli/test/update-check/registry.test.ts @@ -0,0 +1,80 @@ +import { EventEmitter } from 'node:events'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const requestMock = vi.hoisted(() => vi.fn()); + +vi.mock('node:https', () => ({ + request: requestMock, +})); + +type MockResponse = EventEmitter & { statusCode?: number }; +type MockRequest = EventEmitter & { + destroy: ReturnType; + end: () => void; + setTimeout: ReturnType; +}; + +function mockHttpsResponse(statusCode: number, body: string): { socket: { unref: ReturnType } } { + const socket = { unref: vi.fn() }; + requestMock.mockImplementation((_url: unknown, _options: unknown, callback: (response: MockResponse) => void) => { + const request = new EventEmitter() as MockRequest; + request.destroy = vi.fn(); + request.setTimeout = vi.fn(); + request.end = () => { + request.emit('socket', socket); + const response = new EventEmitter() as MockResponse; + response.statusCode = statusCode; + callback(response); + response.emit('data', Buffer.from(body)); + response.emit('end'); + }; + return request; + }); + return { socket }; +} + +describe('fetchDistTags', () => { + beforeEach(() => { + requestMock.mockReset(); + }); + + it('fetches @kaelio/ktx npm dist-tags and unrefs the socket', async () => { + const { socket } = mockHttpsResponse(200, JSON.stringify({ latest: '0.10.0', next: '0.11.0-rc.1' })); + const { fetchDistTags } = await import('../../src/update-check/registry.js'); + + await expect(fetchDistTags()).resolves.toEqual({ latest: '0.10.0', next: '0.11.0-rc.1' }); + + expect(requestMock).toHaveBeenCalledWith( + expect.any(URL), + expect.objectContaining({ + method: 'GET', + headers: expect.objectContaining({ accept: 'application/json' }), + }), + expect.any(Function), + ); + const [url] = requestMock.mock.calls[0] as [URL]; + expect(url.toString()).toBe('https://registry.npmjs.org/-/package/@kaelio/ktx/dist-tags'); + expect(socket.unref).toHaveBeenCalledTimes(1); + }); + + it('rejects non-2xx responses', async () => { + mockHttpsResponse(503, 'registry unavailable'); + const { fetchDistTags } = await import('../../src/update-check/registry.js'); + + await expect(fetchDistTags()).rejects.toThrow('npm dist-tags request failed with 503'); + }); + + it('rejects invalid JSON payloads', async () => { + mockHttpsResponse(200, '{bad json'); + const { fetchDistTags } = await import('../../src/update-check/registry.js'); + + await expect(fetchDistTags()).rejects.toThrow(); + }); + + it('rejects payloads that are not string dist-tag maps', async () => { + mockHttpsResponse(200, JSON.stringify({ latest: 123 })); + const { fetchDistTags } = await import('../../src/update-check/registry.js'); + + await expect(fetchDistTags()).rejects.toThrow(); + }); +}); diff --git a/packages/cli/test/update-check/update-check.test.ts b/packages/cli/test/update-check/update-check.test.ts new file mode 100644 index 00000000..a19b35bf --- /dev/null +++ b/packages/cli/test/update-check/update-check.test.ts @@ -0,0 +1,332 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { updateCheckCachePath } from '../../src/update-check/cache.js'; +import { + prepareUpdateCheckNotice, + renderUpdateNotice, + shouldSuppressUpdateCheck, +} from '../../src/update-check/update-check.js'; + +function makeIo(stdoutIsTTY = true) { + let stderr = ''; + return { + io: { + stdout: { + isTTY: stdoutIsTTY, + write: () => {}, + }, + stderr: { + write: (chunk: string) => { + stderr += chunk; + }, + }, + }, + stderr: () => stderr, + }; +} + +async function flushAsyncWork(): Promise { + await new Promise((resolve) => { + setImmediate(resolve); + }); + await new Promise((resolve) => { + setImmediate(resolve); + }); +} + +describe('update-check orchestration', () => { + let homeDir: string; + + beforeEach(async () => { + homeDir = await mkdtemp(join(tmpdir(), 'ktx-update-check-')); + }); + + afterEach(async () => { + await rm(homeDir, { recursive: true, force: true }); + }); + + it.each([ + ['json option', true, {}, { json: true }], + ['json output option', true, {}, { output: 'json' }], + ['json format option', true, {}, { format: 'json' }], + ['CI', true, { CI: '1' }, {}], + ['non-TTY stdout', false, {}, {}], + ['KTX_NO_UPDATE_CHECK', true, { KTX_NO_UPDATE_CHECK: '1' }, {}], + ['NO_UPDATE_NOTIFIER', true, { NO_UPDATE_NOTIFIER: '1' }, {}], + ['DO_NOT_TRACK', true, { DO_NOT_TRACK: '1' }, {}], + ])('suppresses cache and network work for %s', async (_name, stdoutIsTTY, env, commandOptions) => { + const fetchDistTags = vi.fn(async () => ({ latest: '0.10.0' })); + + const result = await prepareUpdateCheckNotice({ + io: makeIo(stdoutIsTTY).io, + env, + homeDir, + installedVersion: '0.9.0', + commandOptions, + now: () => new Date('2026-06-06T12:00:00.000Z'), + fetchDistTags, + }); + + expect(result.notice).toBeNull(); + expect(fetchDistTags).not.toHaveBeenCalled(); + await expect(readFile(updateCheckCachePath(homeDir), 'utf-8')).rejects.toThrow(); + }); + + it.each([ + ['CI', true, { CI: '1', KTX_OUTPUT: 'pretty' }], + ['non-TTY stdout', false, { KTX_OUTPUT: 'pretty' }], + ])('suppresses cache and network work for %s even when pretty output is forced', async (_name, stdoutIsTTY, env) => { + const fetchDistTags = vi.fn(async () => ({ latest: '0.10.0' })); + + const result = await prepareUpdateCheckNotice({ + io: makeIo(stdoutIsTTY).io, + env, + homeDir, + installedVersion: '0.9.0', + commandOptions: {}, + now: () => new Date('2026-06-06T12:00:00.000Z'), + fetchDistTags, + }); + + expect(result.notice).toBeNull(); + expect(fetchDistTags).not.toHaveBeenCalled(); + await expect(readFile(updateCheckCachePath(homeDir), 'utf-8')).rejects.toThrow(); + }); + + it('does not suppress when only KTX_TELEMETRY_DISABLED is set', () => { + expect( + shouldSuppressUpdateCheck({ + io: makeIo(true).io, + env: { KTX_TELEMETRY_DISABLED: '1' } as NodeJS.ProcessEnv, + commandOptions: {}, + }), + ).toBe(false); + }); + + it('renders a compact no-color stable notice', () => { + expect( + renderUpdateNotice({ + installedVersion: '0.9.0', + targetVersion: '0.10.0', + channel: 'latest', + env: { NO_COLOR: '1' }, + }), + ).toBe('↑ Update available: ktx 0.9.0 → 0.10.0\n npm i -g @kaelio/ktx\n'); + }); + + it('renders the next-channel install command', () => { + expect( + renderUpdateNotice({ + installedVersion: '0.10.0-rc.1', + targetVersion: '0.10.0-rc.2', + channel: 'next', + env: { NO_COLOR: '1' }, + }), + ).toBe('↑ Update available: ktx 0.10.0-rc.1 → 0.10.0-rc.2\n npm i -g @kaelio/ktx@next\n'); + }); + + it('queues a cached notice and stamps lastNoticeAt', async () => { + await mkdir(join(homeDir, '.ktx'), { recursive: true }); + await writeFile( + updateCheckCachePath(homeDir), + JSON.stringify( + { + checkedAt: '2026-06-06T11:00:00.000Z', + channel: 'latest', + installedVersion: '0.9.0', + latestForChannel: '0.10.0', + }, + null, + 2, + ), + 'utf-8', + ); + const fetchDistTags = vi.fn(async () => ({ latest: '0.10.0' })); + + const result = await prepareUpdateCheckNotice({ + io: makeIo(true).io, + env: { NO_COLOR: '1' }, + homeDir, + installedVersion: '0.9.0', + now: () => new Date('2026-06-06T12:00:00.000Z'), + fetchDistTags, + }); + + expect(result.notice).toBe('↑ Update available: ktx 0.9.0 → 0.10.0\n npm i -g @kaelio/ktx\n'); + expect(fetchDistTags).not.toHaveBeenCalled(); + const stored = JSON.parse(await readFile(updateCheckCachePath(homeDir), 'utf-8')) as { lastNoticeAt?: string }; + expect(stored.lastNoticeAt).toBe('2026-06-06T12:00:00.000Z'); + }); + + it('queues a stale cached notice and still refreshes in the background', async () => { + await mkdir(join(homeDir, '.ktx'), { recursive: true }); + await writeFile( + updateCheckCachePath(homeDir), + JSON.stringify( + { + checkedAt: '2026-06-05T10:00:00.000Z', + channel: 'latest', + installedVersion: '0.9.0', + latestForChannel: '0.10.0', + lastNoticeAt: '2026-06-05T11:00:00.000Z', + }, + null, + 2, + ), + 'utf-8', + ); + const fetchDistTags = vi.fn(async () => ({ latest: '0.11.0' })); + + const result = await prepareUpdateCheckNotice({ + io: makeIo(true).io, + env: { NO_COLOR: '1' }, + homeDir, + installedVersion: '0.9.0', + now: () => new Date('2026-06-06T12:00:00.000Z'), + fetchDistTags, + }); + + expect(result.notice).toBe('↑ Update available: ktx 0.9.0 → 0.10.0\n npm i -g @kaelio/ktx\n'); + expect(fetchDistTags).toHaveBeenCalledTimes(1); + + await flushAsyncWork(); + await vi.waitFor(async () => { + const stored = JSON.parse(await readFile(updateCheckCachePath(homeDir), 'utf-8')) as { + latestForChannel: string; + lastNoticeAt?: string; + }; + expect(stored.latestForChannel).toBe('0.11.0'); + expect(stored.lastNoticeAt).toBe('2026-06-06T12:00:00.000Z'); + }); + }); + + it('throttles a cached notice for 24 hours', async () => { + await mkdir(join(homeDir, '.ktx'), { recursive: true }); + await writeFile( + updateCheckCachePath(homeDir), + JSON.stringify( + { + checkedAt: '2026-06-06T11:00:00.000Z', + channel: 'latest', + installedVersion: '0.9.0', + latestForChannel: '0.10.0', + lastNoticeAt: '2026-06-06T11:30:00.000Z', + }, + null, + 2, + ), + 'utf-8', + ); + + await expect( + prepareUpdateCheckNotice({ + io: makeIo(true).io, + env: { NO_COLOR: '1' }, + homeDir, + installedVersion: '0.9.0', + now: () => new Date('2026-06-06T12:00:00.000Z'), + fetchDistTags: vi.fn(async () => ({ latest: '0.10.0' })), + }), + ).resolves.toEqual({ notice: null }); + }); + + it('does not show stale cache after the installed version changes and schedules a refresh', async () => { + await mkdir(join(homeDir, '.ktx'), { recursive: true }); + await writeFile( + updateCheckCachePath(homeDir), + JSON.stringify( + { + checkedAt: '2026-06-06T11:00:00.000Z', + channel: 'latest', + installedVersion: '0.9.0', + latestForChannel: '0.10.0', + }, + null, + 2, + ), + 'utf-8', + ); + const fetchDistTags = vi.fn(async () => ({ latest: '0.10.0' })); + + const result = await prepareUpdateCheckNotice({ + io: makeIo(true).io, + env: { NO_COLOR: '1' }, + homeDir, + installedVersion: '0.10.0', + now: () => new Date('2026-06-06T12:00:00.000Z'), + fetchDistTags, + }); + + expect(result.notice).toBeNull(); + expect(fetchDistTags).toHaveBeenCalledTimes(1); + }); + + it('refreshes stale cache in the background and preserves lastNoticeAt for the same install', async () => { + await mkdir(join(homeDir, '.ktx'), { recursive: true }); + await writeFile( + updateCheckCachePath(homeDir), + JSON.stringify( + { + checkedAt: '2026-06-05T10:00:00.000Z', + channel: 'latest', + installedVersion: '0.9.0', + latestForChannel: '0.10.0', + lastNoticeAt: '2026-06-06T09:00:00.000Z', + }, + null, + 2, + ), + 'utf-8', + ); + + await prepareUpdateCheckNotice({ + io: makeIo(true).io, + env: { NO_COLOR: '1' }, + homeDir, + installedVersion: '0.9.0', + now: () => new Date('2026-06-06T12:00:00.000Z'), + fetchDistTags: vi.fn(async () => ({ latest: '0.11.0' })), + }); + await flushAsyncWork(); + + await vi.waitFor(async () => { + const stored = JSON.parse(await readFile(updateCheckCachePath(homeDir), 'utf-8')) as { + checkedAt: string; + latestForChannel: string; + lastNoticeAt?: string; + }; + expect(stored.checkedAt).toBe('2026-06-06T12:00:00.000Z'); + expect(stored.latestForChannel).toBe('0.11.0'); + expect(stored.lastNoticeAt).toBe('2026-06-06T09:00:00.000Z'); + }); + }); + + it('swallows refresh failures and leaves existing cache untouched', async () => { + await mkdir(join(homeDir, '.ktx'), { recursive: true }); + const originalCache = { + checkedAt: '2026-06-05T10:00:00.000Z', + channel: 'latest', + installedVersion: '0.9.0', + latestForChannel: '0.10.0', + lastNoticeAt: '2026-06-06T09:00:00.000Z', + }; + await writeFile(updateCheckCachePath(homeDir), JSON.stringify(originalCache, null, 2), 'utf-8'); + + await prepareUpdateCheckNotice({ + io: makeIo(true).io, + env: { NO_COLOR: '1' }, + homeDir, + installedVersion: '0.9.0', + now: () => new Date('2026-06-06T12:00:00.000Z'), + fetchDistTags: vi.fn(async () => { + throw new Error('offline'); + }), + }); + await flushAsyncWork(); + + await expect(readFile(updateCheckCachePath(homeDir), 'utf-8')).resolves.toBe(JSON.stringify(originalCache, null, 2)); + }); +}); diff --git a/packages/cli/src/viz-fallback.test.ts b/packages/cli/test/viz-fallback.test.ts similarity index 99% rename from packages/cli/src/viz-fallback.test.ts rename to packages/cli/test/viz-fallback.test.ts index f42eb440..b7b38158 100644 --- a/packages/cli/src/viz-fallback.test.ts +++ b/packages/cli/test/viz-fallback.test.ts @@ -4,7 +4,7 @@ import { resetVizFallbackWarningsForTest, resolveVizFallback, warnVizFallbackOnce, -} from './viz-fallback.js'; +} from '../src/viz-fallback.js'; function io(options: { stdoutTty?: boolean; stdinTty?: boolean; rawMode?: boolean }) { return { diff --git a/packages/cli/tsconfig.test.json b/packages/cli/tsconfig.test.json new file mode 100644 index 00000000..e4b4d755 --- /dev/null +++ b/packages/cli/tsconfig.test.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": true, + "rootDir": ".", + "tsBuildInfoFile": "./dist/.tsbuildinfo.test" + }, + "include": ["src/**/*.ts", "src/**/*.tsx", "test/**/*.ts", "test/**/*.tsx"], + "exclude": ["dist", "node_modules"] +} diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts index ca1a6b26..9260a927 100644 --- a/packages/cli/vitest.config.ts +++ b/packages/cli/vitest.config.ts @@ -9,7 +9,7 @@ export default defineConfig({ }, test: { root: '.', - include: ['src/**/*.test.ts', 'src/**/*.test.tsx'], + include: ['test/**/*.test.ts', 'test/**/*.test.tsx'], testTimeout: 30_000, }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index de0d2c24..cc2fb3d2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -48,7 +48,7 @@ importers: version: 14.1.1(semantic-release@25.0.3(typescript@6.0.3)) '@types/node': specifier: ^24.3.0 - version: 24.12.2 + version: 24.12.4 better-sqlite3: specifier: ^12.10.0 version: 12.10.0 @@ -56,11 +56,11 @@ importers: specifier: ^9.3.1 version: 9.3.1 knip: - specifier: ^6.12.2 - version: 6.12.2(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) + specifier: ^6.14.1 + version: 6.14.1(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) pg: - specifier: ^8.20.0 - version: 8.20.0 + specifier: ^8.21.0 + version: 8.21.0 semantic-release: specifier: ^25.0.3 version: 25.0.3(typescript@6.0.3) @@ -75,19 +75,22 @@ importers: dependencies: '@xyflow/react': specifier: ^12.10.2 - version: 12.10.2(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + version: 12.10.2(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) fumadocs-core: specifier: 16.8.10 - version: 16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.14)(lucide-react@1.14.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3) + version: 16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.15)(lucide-react@1.16.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3) fumadocs-mdx: - specifier: 15.0.4 - version: 15.0.4(@types/mdast@4.0.4)(@types/mdx@2.0.13)(@types/react@19.2.14)(fumadocs-core@16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.14)(lucide-react@1.14.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3))(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react@19.2.6)(vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) + specifier: 15.0.7 + version: 15.0.7(@types/mdast@4.0.4)(@types/mdx@2.0.13)(@types/react@19.2.15)(fumadocs-core@16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.15)(lucide-react@1.16.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) fumadocs-ui: specifier: 16.8.10 - version: 16.8.10(@tailwindcss/oxide@4.3.0)(@types/mdx@2.0.13)(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(fumadocs-core@16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.14)(lucide-react@1.14.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3))(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(tailwindcss@4.3.0) + version: 16.8.10(@tailwindcss/oxide@4.3.0)(@types/mdx@2.0.13)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(fumadocs-core@16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.15)(lucide-react@1.16.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(tailwindcss@4.3.0) + html-to-image: + specifier: 1.11.11 + version: 1.11.11 next: specifier: ^16 - version: 16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + version: 16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) react: specifier: 19.2.6 version: 19.2.6 @@ -100,13 +103,13 @@ importers: version: 4.3.0 '@types/node': specifier: ^24.3.0 - version: 24.12.2 + version: 24.12.4 '@types/react': specifier: ^19 - version: 19.2.14 + version: 19.2.15 '@types/react-dom': specifier: ^19 - version: 19.2.3(@types/react@19.2.14) + version: 19.2.3(@types/react@19.2.15) tailwindcss: specifier: ^4 version: 4.3.0 @@ -117,23 +120,26 @@ importers: packages/cli: dependencies: '@ai-sdk/anthropic': - specifier: 3.0.77 - version: 3.0.77(zod@4.4.3) + specifier: 3.0.78 + version: 3.0.78(zod@4.4.3) '@ai-sdk/devtools': - specifier: 0.0.17 - version: 0.0.17 + specifier: 0.0.18 + version: 0.0.18 '@ai-sdk/google-vertex': - specifier: ^4.0.128 - version: 4.0.128(zod@4.4.3) + specifier: ^4.0.134 + version: 4.0.134(zod@4.4.3) '@anthropic-ai/claude-agent-sdk': - specifier: 0.3.142 - version: 0.3.142(zod@4.4.3) + specifier: 0.3.146 + version: 0.3.146(@anthropic-ai/sdk@0.97.1(zod@4.4.3))(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(zod@4.4.3) + '@clack/core': + specifier: 1.3.1 + version: 1.3.1 '@clack/prompts': specifier: 1.4.0 version: 1.4.0 '@clickhouse/client': - specifier: ^1.18.4 - version: 1.18.4 + specifier: ^1.18.5 + version: 1.18.5 '@commander-js/extra-typings': specifier: 14.0.0 version: 14.0.0(commander@14.0.3) @@ -153,11 +159,14 @@ importers: specifier: ^1.29.0 version: 1.29.0(zod@4.4.3) '@notionhq/client': - specifier: ^5.21.0 - version: 5.21.0 + specifier: ^5.22.0 + version: 5.22.0 + '@openai/codex-sdk': + specifier: ^0.133.0 + version: 0.133.0 ai: - specifier: ^6.0.180 - version: 6.0.180(zod@4.4.3) + specifier: ^6.0.188 + version: 6.0.188(zod@4.4.3) better-sqlite3: specifier: ^12.10.0 version: 12.10.0 @@ -165,14 +174,14 @@ importers: specifier: 14.0.3 version: 14.0.3 fflate: - specifier: ^0.8.2 - version: 0.8.2 + specifier: ^0.8.3 + version: 0.8.3 handlebars: specifier: ^4.7.9 version: 4.7.9 ink: - specifier: ^7.0.2 - version: 7.0.2(@types/react@19.2.14)(react@19.2.6) + specifier: ^7.0.3 + version: 7.0.3(@types/react@19.2.15)(react@19.2.6) lookml-parser: specifier: 7.1.0 version: 7.1.0(js-yaml@4.1.1) @@ -180,32 +189,35 @@ importers: specifier: ^10.2.5 version: 10.2.5 mssql: - specifier: ^12.5.2 - version: 12.5.2(@azure/core-client@1.10.1) + specifier: ^12.5.4 + version: 12.5.4(@azure/core-client@1.10.1) mysql2: specifier: ^3.22.3 - version: 3.22.3(@types/node@24.12.2) + version: 3.22.3(@types/node@24.12.4) openai: - specifier: ^6.37.0 - version: 6.37.0(ws@8.20.1)(zod@4.4.3) + specifier: ^6.38.0 + version: 6.38.0(ws@8.20.1)(zod@4.4.3) p-limit: specifier: ^7.3.0 version: 7.3.0 pg: - specifier: ^8.20.0 - version: 8.20.0 + specifier: ^8.21.0 + version: 8.21.0 posthog-node: - specifier: ^5.0.0 - version: 5.0.0 + specifier: ^5.34.9 + version: 5.34.9 react: specifier: ^19.2.6 version: 19.2.6 + semver: + specifier: ^7.8.1 + version: 7.8.1 simple-git: specifier: 3.36.0 version: 3.36.0 snowflake-sdk: - specifier: ^2.4.1 - version: 2.4.1(asn1.js@5.4.1) + specifier: ^2.4.2 + version: 2.4.2(asn1.js@5.4.1) yaml: specifier: ^2.9.0 version: 2.9.0 @@ -227,28 +239,31 @@ importers: version: 12.3.0(@azure/core-client@1.10.1) '@types/node': specifier: ^24.3.0 - version: 24.12.2 + version: 24.12.4 '@types/pg': specifier: ^8.20.0 version: 8.20.0 '@types/react': - specifier: ^19.2.14 - version: 19.2.14 + specifier: ^19.2.15 + version: 19.2.15 + '@types/semver': + specifier: ^7.7.1 + version: 7.7.1 '@vitest/coverage-v8': - specifier: ^4.1.6 - version: 4.1.6(vitest@4.1.6) + specifier: ^4.1.7 + version: 4.1.7(vitest@4.1.7) ajv: specifier: 8.20.0 version: 8.20.0 ink-testing-library: specifier: ^4.0.0 - version: 4.0.0(@types/react@19.2.14) + version: 4.0.0(@types/react@19.2.15) typescript: specifier: ^6.0.3 version: 6.0.3 vitest: - specifier: ^4.1.6 - version: 4.1.6(@opentelemetry/api@1.9.0)(@types/node@24.12.2)(@vitest/coverage-v8@4.1.6)(vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) + specifier: ^4.1.7 + version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.7)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) packages: @@ -264,31 +279,31 @@ packages: '@actions/io@3.0.2': resolution: {integrity: sha512-nRBchcMM+QK1pdjO7/idu86rbJI5YHUKCvKs0KxnSYbVe3F51UfGxuZX4Qy/fWlp6l7gWFwIkrOzN+oUK03kfw==} - '@ai-sdk/anthropic@3.0.77': - resolution: {integrity: sha512-ML8C2M1YvPA1ulEx4TiyF0k1xvC2ikEiPBIC1PPQ0a5xELUGrO2lAaEzsTEoJ+eCeDd8PSBuFJjs+r+9yIwQXA==} + '@ai-sdk/anthropic@3.0.78': + resolution: {integrity: sha512-0OY12G20cUt6iU6htpEA1491Oz++NVxZxlmWGX4B7rSbeZ5pnDmOu6YtW9BKzdZlNx5Gn23i6WMxyZFoMKNcgA==} engines: {node: '>=18'} peerDependencies: zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/devtools@0.0.17': - resolution: {integrity: sha512-CJgo+3DMHOJbxxq1qTgnW4vpFXgBW1pHePMimBW4Go5FPU7iLqppoGX/UC798IXqlD3hncQRPfyBLZjbsJC91w==} + '@ai-sdk/devtools@0.0.18': + resolution: {integrity: sha512-0J25Q7occrkMJM1MrP0KeR8XNdGGKNgzxhOLfxBe/qZBQP6yXgV4H5Gf2DnDC3UgXDBJBskH9nh23doCo2Pebw==} engines: {node: '>=18'} hasBin: true - '@ai-sdk/gateway@3.0.114': - resolution: {integrity: sha512-MqkZ5sd+qiq6RgIxELkoFQXg2/JwK+WCMaot7U+rtrZpWJl3fSyYvc28SC03b256o4F7OXjQtdjTqs81B2w+dA==} + '@ai-sdk/gateway@3.0.118': + resolution: {integrity: sha512-XYPbVoDo1TDMVLe5Eg42gIjdOyxaizh9H0kiSSnTXr+AdrqZvutk/ypLOiqBXPV3D1K3+BSm/sbFeomZJlM64A==} engines: {node: '>=18'} peerDependencies: zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/google-vertex@4.0.128': - resolution: {integrity: sha512-jK8fixb4km2yfgvb9DUFQRpV/jiDB0v9gyxHoHfPydaQvz+CpAz8DTt1quyaM+Wg9G2R8Zo68CYmHbIkUqW2AA==} + '@ai-sdk/google-vertex@4.0.134': + resolution: {integrity: sha512-EaVwzHk7P/Pj1JQtOfN3uLj+zKY6MQOn2hcEEACpbbjdVgBkpWxDjMRIpyYbVlXCLMUeWSYL0qUM54lmis/1BQ==} engines: {node: '>=18'} peerDependencies: zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/google@3.0.73': - resolution: {integrity: sha512-o2MuIeyvZrFIeIbnbA8Thrr63irdyUBh0uWBZ2lY6yFeXuE/tcwyXF74bDKS4KvTu84uFpQfpbS/LXHGKKXz+g==} + '@ai-sdk/google@3.0.78': + resolution: {integrity: sha512-iPkZHiaaBNreaVX2fLpc+SAa7OJPV6f7pZRK98lWTI4vf0D546+9eEQ6T2FagJAHO0K0gEyzx5zogCoHbJnhQg==} engines: {node: '>=18'} peerDependencies: zod: ^3.25.76 || ^4.1.8 @@ -317,58 +332,60 @@ packages: resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} engines: {node: '>=10'} - '@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.142': - resolution: {integrity: sha512-yBHOiRqJ8JcD9OAMGJALbypaD3u3K8hyUmcnZ+91AHJtymzWxuMkVi4IY1qp8L5jzkKeTnvYfCspzkbiHLuYWg==} + '@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.146': + resolution: {integrity: sha512-0IIvlEaenq2CRSVx5Bo5BaCtHQXS87GancM35WKEYveGVLn6DI+5G7ikYuTE4AKRPkMnogFtY4BJt6LulWGj+A==} cpu: [arm64] os: [darwin] - '@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.142': - resolution: {integrity: sha512-/a/bVMjvAl3gNzWiPIgynYktTYckTcp4YAacV/2F4Jd8XeCV0+DMQW7OFeR+3fnPcBg/8kcOAVYfLZXDExqO1w==} + '@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.146': + resolution: {integrity: sha512-Dk5xJ03Ff1JXbMRP1t2wc/TyfY6xF/2Ysp31wMhFPjoNiKSPHMWaIg242+T3CHdxLWmJ8plWHL1HL5cyZ/LCkw==} cpu: [x64] os: [darwin] - '@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.142': - resolution: {integrity: sha512-KZuwSupNJovnMJ7MZxjp1Qq0yu7rAmbzO4Zlmr3jtKDU95t2kgs3c6j4evzQDCgTQMlwH8QTSV4mItDGxlYEbg==} + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.146': + resolution: {integrity: sha512-QlCid0ucdrmhUAOewfQjaofN2wlokWcfFTxSFePTSj1umk35JO7TDFP700F7jU49r1fPWIdvJpPwWGyB0DeFPA==} cpu: [arm64] os: [linux] libc: [musl] - '@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.142': - resolution: {integrity: sha512-QKG553PSbIcQ5KLvnl2ekfy5lTyU3dW/X5fDQlRLv4YHNHnqf2o7scJ6eUdfaVTQdIZ+Pa7SNN3bsvVs4bNjQw==} + '@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.146': + resolution: {integrity: sha512-mzBXDDWWBAC/vDtAYpO1G/dq5QvJtYSPXsqcb+sNdcDhiuf4IYnYp7ytRncYlsUNDkLmX6Gk2jkWAHUUA2Lozg==} cpu: [arm64] os: [linux] libc: [glibc] - '@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.142': - resolution: {integrity: sha512-QkDwLMsdYO7n/i1zPCt5YZIet5u+Eo07UpF9UX5yD7bnwRZKDe22L6LVVwiLLjeTO0fTz+uNY7w9/XOYQMlxUQ==} + '@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.146': + resolution: {integrity: sha512-E3coK1ThQT08KIX80RLcsq7DWXFllCKOzoOe32it/bdtY56TBgPY9xemwXhIJ+cVBHTI9/MpBSIlKBcFCt+yQA==} cpu: [x64] os: [linux] libc: [musl] - '@anthropic-ai/claude-agent-sdk-linux-x64@0.3.142': - resolution: {integrity: sha512-o1QZmCNRL5BFTc14KEvT23Fxm1jNv0aa0e9T0OZUjua0oW8DRpri3HKvDEM36qEGWUOANBG7h6Ca/KNqxaTnYg==} + '@anthropic-ai/claude-agent-sdk-linux-x64@0.3.146': + resolution: {integrity: sha512-B2baXU1tCBT5CVlD7jJMKjpC4xdO45NUIWpqImmwuOfKvlM/PITjyTXyTY662mGZf1dBmdqBBsqirwFH/jhi8Q==} cpu: [x64] os: [linux] libc: [glibc] - '@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.142': - resolution: {integrity: sha512-x8lbY1m7E/BiFF0Gu/Mx9lkD/zW3vBr3viw0GYNuqY9GYHfLOX9+l9H8C+INeGzB4+ibG8+xD2pnRhWdQxuvUg==} + '@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.146': + resolution: {integrity: sha512-CIwQxGX2r/yWpjCJ6ahB3smKXhghWgGTxL98+LGW52TUwqTiBnlNrH9DPqqgv1/+Hyquw6xfLrKU+StyfMgiLw==} cpu: [arm64] os: [win32] - '@anthropic-ai/claude-agent-sdk-win32-x64@0.3.142': - resolution: {integrity: sha512-NpNxdiCEUNjjwvBltpDnkgdjVQ+nRsALpfM1Pe4GhnYiOkTk/TvjMZUuA2qGh0F8KyF0FbqzUsi0uXIgojJT5w==} + '@anthropic-ai/claude-agent-sdk-win32-x64@0.3.146': + resolution: {integrity: sha512-qmxrsyaqA8s4HShqJls7ZCRjdoqN66Jo/hbjQNB3uHepD8tEO1iD19aPV4+osdLT7feMkhDBfLT07Q30R2NB5w==} cpu: [x64] os: [win32] - '@anthropic-ai/claude-agent-sdk@0.3.142': - resolution: {integrity: sha512-k1xBon6ov0PT/vZNf+Z+SuAqmylGJU/+a+h/k04MW5cBbzOIwiVcGFRTGJ/qbY5pcboJbLtts/yBwSu9AvSipg==} + '@anthropic-ai/claude-agent-sdk@0.3.146': + resolution: {integrity: sha512-hK9/Ng+hOyexUemTxdIUsSWJ9o2LFi2YNWzHwz8/YMCohUYOnFMZkBiENvUAb0WIc5hieOyBZrOIlg5OewuJMg==} engines: {node: '>=18.0.0'} peerDependencies: + '@anthropic-ai/sdk': '>=0.93.0' + '@modelcontextprotocol/sdk': ^1.29.0 zod: ^4.0.0 - '@anthropic-ai/sdk@0.93.0': - resolution: {integrity: sha512-q9vaSZQVFx6B/gPxetGYfLXSJD5v0sOmh0OpZDq7yCrTSA+Rscvrtyol7JJTW40wEpQB4U1B4JXzxQitbQ3CAA==} + '@anthropic-ai/sdk@0.97.1': + resolution: {integrity: sha512-wOf7AUeJPitcVpvKO4UMu63mWH5SaVipkGd7OOQJt/G6VYGlV8D2Gp9dLxOrttDJh/9gqPqdaBwDGcBevumeAg==} hasBin: true peerDependencies: zod: ^3.25.0 || ^4.0.0 @@ -407,136 +424,127 @@ packages: resolution: {integrity: sha512-oDJJ7rM1osvfBdfZuhQ5DM6lHD9iuypL9m2LsEiA/lB8xuE5uPYsftNDcS0J9VRXFSvYTqC14K7Y5vMMKMg0vw==} engines: {node: '>=20.0.0'} - '@aws-sdk/core@3.974.8': - resolution: {integrity: sha512-njR2qoG6ZuB0kvAS2FyICsFZJ6gmCcf2X/7JcD14sUvGDm26wiZ5BrA6LOiUxKFEF+IVe7kdroxyE00YlkiYsw==} + '@aws-sdk/core@3.974.12': + resolution: {integrity: sha512-qrqgioqYFjwR6LatVNS1L2Vk++EwRIxqSQXPKNv5Ofux2D8UNgqMQ1znnMyEImXquVPTtbf71fc128pvmU6y9A==} engines: {node: '>=20.0.0'} - '@aws-sdk/crc64-nvme@3.972.7': - resolution: {integrity: sha512-QUagVVBbC8gODCF6e1aV0mE2TXWB9Opz4k8EJFdNrujUVQm5R4AjJa1mpOqzwOuROBzqJU9zawzig7M96L8Ejg==} + '@aws-sdk/crc64-nvme@3.972.8': + resolution: {integrity: sha512-fVfUCL/Xh2zINYMPZvj+iBn6XWouQf0DAnjaWCI9MkmqXzL2Iy5FoQB8O7syFe6gN6AH1ecDDU58T51Ou0kFkA==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-env@3.972.34': - resolution: {integrity: sha512-XT0jtf8Fw9JE6ppsQeoNnZRiG+jqRixMT1v1ZR17G60UvVdsQmTG8nbEyHuEPfMxDXEhfdARaM/XiEhca4lGHQ==} + '@aws-sdk/credential-provider-env@3.972.38': + resolution: {integrity: sha512-m3WjZEgPtioMhPmwqUt+DhlTJ2i9ufR6DhfkyXojb9puEvfR+ur2U5shavu5/Cc9WHHsDCvALi6UFHgcqjhQ5w==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-http@3.972.36': - resolution: {integrity: sha512-DPoGWfy7J7RKxvbf5kOKIGQkD2ek3dbKgzKIGrnLuvZBz5myU+Im/H6pmc14QcnFbqHMqxvtWSgRDSJW3qXLQg==} + '@aws-sdk/credential-provider-http@3.972.40': + resolution: {integrity: sha512-D78L/m2Dr6cJnnSvWoAudPhQmCwmJ7j6APXsPYmFpPaKfQTfCSu0rdm8j14Np+VmXF9z8Aj8HE3xFpsrwtfgeg==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-ini@3.972.38': - resolution: {integrity: sha512-oDzUBu2MGJFgoar05sPMCwSrhw44ASyccrHzj66vO69OZqi7I6hZZxXfuPLC8OCzW7C+sU+bI73XHij41yekgQ==} + '@aws-sdk/credential-provider-ini@3.972.42': + resolution: {integrity: sha512-Mu5ESvFXeinafVM8jTIvRqcvK2Ehj4kz3auT39yUcHwu1Vfxo6xRlmUafdKLW4tusjAJukQwK09sCSMgOm7OKg==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-login@3.972.38': - resolution: {integrity: sha512-g1NosS8qe4OF++G2UFCM5ovSkgipC7YYor5KCWatG0UoMSO5YFj9C8muePlyVmOBV/WTI16Jo3/s1NUo/o1Bww==} + '@aws-sdk/credential-provider-login@3.972.42': + resolution: {integrity: sha512-O6WkZga3kf0yqyJYd1dbeJqVhEgJx/x1UaLgtbR+XuL/YP+K5y6QTxQKL7ka9z3jnQASESKGAPnRyt4D5hQrxA==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-node@3.972.39': - resolution: {integrity: sha512-HEswDQyxUtadoZ/bJsPPENHg7R0Lzym5LuMksJeHvqhCOpP+rtkDLKI4/ZChH4w3cf5kG8n6bZuI8PzajoiqMg==} + '@aws-sdk/credential-provider-node@3.972.43': + resolution: {integrity: sha512-D/DJmbrWRP5BXEO3FH+ar4el+2n6OlGofiud7dQun2jES+AQEJjczenp1jBb4MBN7CpGpS8nsWGQLtuzc9tQbA==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-process@3.972.34': - resolution: {integrity: sha512-T3IFs4EVmVi1dVN5RciFnklCANSzvrQd/VuHY9ThHSQmYkTogjcGkoJEr+oNUPQZnso52183088NqysMPji1/Q==} + '@aws-sdk/credential-provider-process@3.972.38': + resolution: {integrity: sha512-EnbYVajGgbkb24s0K1eo4VNAPV5mHIET7LSvirTaFCwkfrfaOJxtSE+wY/tJdKDS21cEYkZs2ruCaAm+W4iblg==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-sso@3.972.38': - resolution: {integrity: sha512-5ZxG+t0+3Q3QPh8KEjX6syskhgNf7I0MN7oGioTf6Lm1NTjfP7sIcYGNsthXC2qR8vcD3edNZwCr2ovfSSWuRA==} + '@aws-sdk/credential-provider-sso@3.972.42': + resolution: {integrity: sha512-RVV/9NbFwI8ZHEH5dn39lGyFmSbSVj1+orZdr6QsOe1mW9DCglmlen0cFaNZmCcqkqc7erNRHNBduxbeZuHAnw==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-web-identity@3.972.38': - resolution: {integrity: sha512-lYHFF30DGI20jZcYX8cm6Ns0V7f1dDN6g/MBDLTyD/5iw+bXs3yBr2iAiHDkx4RFU5JgsnZvCHYKiRVPRdmOgw==} + '@aws-sdk/credential-provider-web-identity@3.972.42': + resolution: {integrity: sha512-/67fXX0ddllD4u2Nujc5PvT4byHgpMUfz6+RxIKi/0nFIckeorm7JvXgzBuDyVKw0s58EbofmETDWUf9vTEuHQ==} engines: {node: '>=20.0.0'} '@aws-sdk/ec2-metadata-service@3.1045.0': resolution: {integrity: sha512-cYjEbjbGScw9l8TmI9AFYde1hIu5c9Wt0Qp7/cbWBHBiOzMfLwmjGhd5+4AUm1RsnmC5HZ/WOA9iGJHfHL4cuA==} engines: {node: '>=20.0.0'} - '@aws-sdk/middleware-bucket-endpoint@3.972.10': - resolution: {integrity: sha512-Vbc2frZH7wXlMNd+ZZSXUEs/l1Sv8Jj4zUnIfwrYF5lwaLdXHZ9xx4U3rjUcaye3HRhFVc+E5DbBxpRAbB16BA==} + '@aws-sdk/middleware-bucket-endpoint@3.972.14': + resolution: {integrity: sha512-Aaj0d+xbo1jJquBWJP0/9V/XZRYukO3LWIRp3dOLHmoFrYKb4YZ0aLefgVHfGcNOVBS2ZTq7L/n5JcrE7DaC+Q==} engines: {node: '>=20.0.0'} - '@aws-sdk/middleware-expect-continue@3.972.10': - resolution: {integrity: sha512-2Yn0f1Qiq/DjxYR3wfI3LokXnjOhFM7Ssn4LTdFDIxRMCE6I32MAsVnhPX1cUZsuVA9tiZtwwhlSLAtFGxAZlQ==} + '@aws-sdk/middleware-expect-continue@3.972.12': + resolution: {integrity: sha512-dA5pKTom/Ls9mgeyeaRBNQrRIVOLVjv4AmKOB0/e4yaiXEUy0gSz2d3liP8JHtYoCAEWySU1jWnyzwLOREN+4g==} engines: {node: '>=20.0.0'} - '@aws-sdk/middleware-flexible-checksums@3.974.16': - resolution: {integrity: sha512-6ru8doI0/XzszqLIPXf0E/V7HhAw1Pu94010XCKYtBUfD0LxF0BuOzrUf8OQGR6j2o6wgKTHUniOmndQycHwCA==} + '@aws-sdk/middleware-flexible-checksums@3.974.20': + resolution: {integrity: sha512-NdnMVQCR1YjIcqFAiNLdBiOwr2DyQDB2IiXQrBhzolKOv32ae4d4Ll7IzLMi04eMHiq/o/Y/GjFuVjF9HuG0QA==} engines: {node: '>=20.0.0'} - '@aws-sdk/middleware-host-header@3.972.10': - resolution: {integrity: sha512-IJSsIMeVQ8MMCPbuh1AbltkFhLBLXn7aejzfX5YKT/VLDHn++Dcz8886tXckE+wQssyPUhaXrJhdakO2VilRhg==} + '@aws-sdk/middleware-host-header@3.972.13': + resolution: {integrity: sha512-EA3+u2LD3kGcfRNmCSjyJuzX4XvG4zYv57i4ZksH+1IEciuSyHQGvzivEz7vZ+jbRPdAAe7WWKy/4M8InCKDcw==} engines: {node: '>=20.0.0'} '@aws-sdk/middleware-location-constraint@3.972.10': resolution: {integrity: sha512-rI3NZvJcEvjoD0+0PI0iUAwlPw2IlSlhyvgBK/3WkKJQE/YiKFedd9dMN2lVacdNxPNhxL/jzQaKQdrGtQagjQ==} engines: {node: '>=20.0.0'} - '@aws-sdk/middleware-logger@3.972.10': - resolution: {integrity: sha512-OOuGvvz1Dm20SjZo5oEBePFqxt5nf8AwkNDSyUHvD9/bfNASmstcYxFAHUowy4n6Io7mWUZ04JURZwSBvyQanQ==} + '@aws-sdk/middleware-logger@3.972.12': + resolution: {integrity: sha512-NxB2dS4/mV3380hNkC72TkhMaLLjWGGBeTAEucqlJptVVovTbNmQWZLwaMC74ICo9NZHmFiBVVTHzDfAh/3y6Q==} engines: {node: '>=20.0.0'} - '@aws-sdk/middleware-recursion-detection@3.972.11': - resolution: {integrity: sha512-+zz6f79Kj9V5qFK2P+D8Ehjnw4AhphAlCAsPjUqEcInA9umtSSKMrHbSagEeOIsDNuvVrH98bjRHcyQukTrhaQ==} + '@aws-sdk/middleware-recursion-detection@3.972.14': + resolution: {integrity: sha512-bqL+upATpOJ/7px4IVfMVxcM6Lyt9uRizmEx3mNg4N6+IQlnOaYXXOJ4TNX6P0mKPPW0lwn9ZW8QEhXwQuCH9A==} engines: {node: '>=20.0.0'} - '@aws-sdk/middleware-sdk-s3@3.972.37': - resolution: {integrity: sha512-Km7M+i8DrLArVzrid1gfxeGhYHBd3uxvE77g0s5a52zPSVosxzQBnJ0gwWb6NIp/DOk8gsBMhi7V+cpJG0ndTA==} + '@aws-sdk/middleware-sdk-s3@3.972.41': + resolution: {integrity: sha512-M4T2I2WPuH5WQpU8Tsp+u2bcO29zGRkU14ATzuqb9I4xh8tzsLqtp4hzaJM5aO2dhMZnHDzyQwSFVgc3XbnoGg==} engines: {node: '>=20.0.0'} '@aws-sdk/middleware-ssec@3.972.10': resolution: {integrity: sha512-Gli9A0u8EVVb+5bFDGS/QbSVg28w/wpEidg1ggVcSj65BDTdGR6punsOcVjqdiu1i42WHWo51MCvARPIIz9juw==} engines: {node: '>=20.0.0'} - '@aws-sdk/middleware-user-agent@3.972.38': - resolution: {integrity: sha512-iz+B29TXcAZsJpwB+AwG/TTGA5l/VnmMZ2UxtiySOZjI6gCdmviXPwdgzcmuazMy16rXoPY4mYCGe7zdNKfx5A==} + '@aws-sdk/middleware-user-agent@3.972.42': + resolution: {integrity: sha512-U7jjlJKQnuUlI2swC2umFLFzLAxMLudSRFv+Bqk2F8ORmr5bG25qsFxGm4GEFwoZeGaFFnAFmTY0xReVRfyl2A==} engines: {node: '>=20.0.0'} - '@aws-sdk/nested-clients@3.997.6': - resolution: {integrity: sha512-WBDnqatJl+kGObpfmfSxqnXeYTu3Me8wx8WCtvoxX3pfWrrTv8I4WTMSSs7PZqcRcVh8WeUKMgGFjMG+52SR1w==} + '@aws-sdk/nested-clients@3.997.10': + resolution: {integrity: sha512-FtQ/Bt327peZJuyo4WZSOLVUTw9ujRxntepiC7L65FxA2P82Xlq0g14T22BuqBUeMjDoxa9nvwiMHjLIfP3eUg==} engines: {node: '>=20.0.0'} - '@aws-sdk/region-config-resolver@3.972.13': - resolution: {integrity: sha512-CvJ2ZIjK/jVD/lbOpowBVElJyC1YxLTIJ13yM0AEo0t2v7swOzGjSA6lJGH+DwZXQhcjUjoYwc8bVYCX5MDr1A==} + '@aws-sdk/region-config-resolver@3.972.16': + resolution: {integrity: sha512-/YaivCvKUkEeMN9VTKBSvBn5w/4osAM1YboM58DKaLF/vqFGf/FdJCLmppqiPPJWZaXcASqByVjc3evE7KHKdA==} engines: {node: '>=20.0.0'} - '@aws-sdk/signature-v4-multi-region@3.996.25': - resolution: {integrity: sha512-+CMIt3e1VzlklAECmG+DtP1sV8iKq25FuA0OKpnJ4KA0kxUtd7CgClY7/RU6VzJBQwbN4EJ9Ue6plvqx1qGadw==} + '@aws-sdk/signature-v4-multi-region@3.996.27': + resolution: {integrity: sha512-0Phbz4t6HI3D3skxvG2uI+VWU034/nSIw1T8d+FPzzQG9EQTrw94o9mOKO2Gv3n3Oc8P7JD7RAUxkoneLWv5Eg==} engines: {node: '>=20.0.0'} - '@aws-sdk/token-providers@3.1041.0': - resolution: {integrity: sha512-Th7kPI6YPtvJUcdznooXJMy+9rQWjmEF81LxaJssngBzuysK4a/x+l8kjm1zb7nYsUPbndnBdUnwng/3PLvtGw==} + '@aws-sdk/token-providers@3.1049.0': + resolution: {integrity: sha512-r7+d0lQMTHKypkmaF5jRTBYLYHCUHzt3gaVoN9SidLhQeWhCmHk3AKrboDTpPF5b7Pt7vKu3+oeMjznM2Eu1ow==} engines: {node: '>=20.0.0'} '@aws-sdk/types@3.973.8': resolution: {integrity: sha512-gjlAdtHMbtR9X5iIhVUvbVcy55KnznpC6bkDUWW9z915bi0ckdUr5cjf16Kp6xq0bP5HBD2xzgbL9F9Quv5vUw==} engines: {node: '>=20.0.0'} - '@aws-sdk/util-arn-parser@3.972.3': - resolution: {integrity: sha512-HzSD8PMFrvgi2Kserxuff5VitNq2sgf3w9qxmskKDiDTThWfVteJxuCS9JXiPIPtmCrp+7N9asfIaVhBFORllA==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/util-endpoints@3.996.8': - resolution: {integrity: sha512-oOZHcRDihk5iEe5V25NVWg45b3qEA8OpHWVdU/XQh8Zj4heVPAJqWvMphQnU7LkufmUo10EpvFPZuQMiFLJK3g==} + '@aws-sdk/util-endpoints@3.996.11': + resolution: {integrity: sha512-BUMJ6VoL54r6Udj/wKy8uKRIndL04rGbaS/wTIV0dM1ewxSrR8yARBHdvZKQsK55ZSW2JrmAPk3KP15kBDxJMw==} engines: {node: '>=20.0.0'} '@aws-sdk/util-locate-window@3.965.5': resolution: {integrity: sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ==} engines: {node: '>=20.0.0'} - '@aws-sdk/util-user-agent-browser@3.972.10': - resolution: {integrity: sha512-FAzqXvfEssGdSIz8ejatan0bOdx1qefBWKF/gWmVBXIP1HkS7v/wjjaqrAGGKvyihrXTXW00/2/1nTJtxpXz7g==} + '@aws-sdk/util-user-agent-browser@3.972.13': + resolution: {integrity: sha512-wfk9ZdVwh187gdGXB1EyAoprwjSMt/bSfVtva+OaZx+LyNdKD7smlZf611yMd42UpfQ9vaS8NOftjSajgpdd+w==} - '@aws-sdk/util-user-agent-node@3.973.24': - resolution: {integrity: sha512-ZWwlkjcIp7cEL8ZfTpTAPNkwx25p7xol0xlKoWVVf22+nsjwmLcHYtTPjIV1cSpmB/b6DaK4cb1fSkvCXHgRdw==} + '@aws-sdk/util-user-agent-node@3.973.28': + resolution: {integrity: sha512-A2l/PTRzsOS9L8dmZbXtDyJQgeeX+qjqLJ+fr0UU5Dz0AUQMuxgZCPSLKZgUDlHAmLFuk34owdMEvJxmDTBgRg==} engines: {node: '>=20.0.0'} - peerDependencies: - aws-crt: '>=1.0.0' - peerDependenciesMeta: - aws-crt: - optional: true - '@aws-sdk/xml-builder@3.972.22': - resolution: {integrity: sha512-PMYKKtJd70IsSG0yHrdAbxBr+ZWBKLvzFZfD3/urxgf6hXVMzuU5M+3MJ5G67RpOmLBu1fAUN65SbWuKUCOlAA==} + '@aws-sdk/xml-builder@3.972.24': + resolution: {integrity: sha512-V8z5YcDPfsvzrBlj0xR1vhRtocblhYbqdreCJB/voGd4Sr5zjNAeWxexbnqVtskTJe0vFb5KMqbSL++ePl+zRw==} engines: {node: '>=20.0.0'} '@aws/lambda-invoke-store@0.2.4': @@ -606,16 +614,16 @@ packages: resolution: {integrity: sha512-fCqPIfOcLE+CGqGPd66c8bZpwAji98tZ4JI9i/mlTNTlsIWslCfpg48s/ypyLxZTump5sypjrKn2/kY7q8oAbA==} engines: {node: '>=20.0.0'} - '@azure/msal-browser@5.9.0': - resolution: {integrity: sha512-CzE+4PefDSJWj26zU7G1bKchlGRRHMBFreG4tAlGuzyI8hAPiYGobaJvZBgZBf6L63iphX7VH+ityL8VgEQz9Q==} + '@azure/msal-browser@5.11.0': + resolution: {integrity: sha512-zkGNYS3TwY8lUpPIafAmsFCYZbgFixY9y/LZB9GUg0IILoHTqpN26j5OrkL1AQThh/YdZsawe4iWXfp85lFVxg==} engines: {node: '>=0.8.0'} - '@azure/msal-common@16.5.2': - resolution: {integrity: sha512-GkDEL6TYo3HgT3UuqakdgE9PZfc1hMki6+Hwgy1uddb/EauvAKfu85vVhuofRSo22D1xTnWt8Ucwfg4vSCVwvA==} + '@azure/msal-common@16.6.2': + resolution: {integrity: sha512-hQjjsekAjB00cM1EmatWJlzhEoK2Qhz7Rj5gvM6tYf8iL7RM3tkxlpU9fG0+ofkulzg9AEEA6dIEnSmDr5ZqUA==} engines: {node: '>=0.8.0'} - '@azure/msal-node@5.1.5': - resolution: {integrity: sha512-ObTeMoNPmq19X3z40et9Xvs4ZoWVeJg43PZMRLG5iwVL+2nCtAerG3YTDItqPp1CfXNwmCXBbg8jn1DOx65c3g==} + '@azure/msal-node@5.2.2': + resolution: {integrity: sha512-toS+2AePxqyzb0YOKttDOOiSl3jrkK9aiqIvpurpis0O34QcIS5gToqrgT39p04Dpxw3YoUU0lxJKTpSFFfA6Q==} engines: {node: '>=20'} '@azure/storage-blob@12.26.0': @@ -716,11 +724,11 @@ packages: resolution: {integrity: sha512-S0My7XPGIgpRWMDG8uRqalbgT+a6FmCUdOW+HaIOVVpUPHOb7RrpvjTjiODadKp06fsrVDJZlIzc6yCTp4AnxA==} engines: {node: '>= 20.12.0'} - '@clickhouse/client-common@1.18.4': - resolution: {integrity: sha512-kPPtv8yQmplNAxfrAJvwBJq5dd+IWRewEbXSpUvtyEJXlrB8lt/ZH63jUS81Nmd+lK5MRvpOFXPoN3iogkvg+A==} + '@clickhouse/client-common@1.18.5': + resolution: {integrity: sha512-g9LwcS1dvkatKDsIjT1PwUHldsiYzwdKAB0nXfd9APLd+t4PrNJa+my+dXcqJdmcWyhWjKLP/2/ztBwgxp+sbQ==} - '@clickhouse/client@1.18.4': - resolution: {integrity: sha512-jjCrddI+e2OVXGh/MQY92K9r8Z/iwqaZtUXNI/MfZ/y9VGYwfbQsXRzp4Jv6w4Hgxvr4sLcz9YwIvkCBQ6X/mw==} + '@clickhouse/client@1.18.5': + resolution: {integrity: sha512-4FfoyMkFWhsdNMuXsoEL6l3c12svA63BBJBtDo9SrxRZ14RdmN6jLr/rF3f84BK8cFoxETZCSeKlsbk6NNYebw==} engines: {node: '>=16'} '@colors/colors@1.5.0': @@ -1240,8 +1248,8 @@ packages: '@nodable/entities@2.1.0': resolution: {integrity: sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==} - '@notionhq/client@5.21.0': - resolution: {integrity: sha512-X+T+hzaQFleOUGm4xUOUm51pOpdZ1+6T4BsRjGlcdEOTJLNkUEv8nZATq9O3ZY4NQEgICc0qwQ0I25OdYprX0w==} + '@notionhq/client@5.22.0': + resolution: {integrity: sha512-lZ3JGBCd6O6MNHWn/58QcUqX1FgmlcODcx/EaUEEpuxLXF5tSi+v29Vzoz8mZ6JgDWDn5pMzzjB69QevYjQQZA==} engines: {node: '>=18'} '@octokit/auth-token@6.0.0': @@ -1292,146 +1300,191 @@ packages: '@octokit/types@16.0.0': resolution: {integrity: sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==} - '@opentelemetry/api@1.9.0': - resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} + '@openai/codex-sdk@0.133.0': + resolution: {integrity: sha512-PB82D/1Q0C7nzaV5O+1O4y5LcVwiUvxyHvCUTfz8Cwztv6bOWQ40gFHE5ZFX1EFPJx1cMV0GPVODWuXIKAuayQ==} + engines: {node: '>=18'} + + '@openai/codex@0.133.0': + resolution: {integrity: sha512-Gh42kLLBo/6gpnHmDzUWDVvyS57ekCB1+1Dz0RG2oIl3Lhk1uwrjSj/PwaJWWh4Rw/rUp1RqkwrMugFfFEOlqQ==} + engines: {node: '>=16'} + hasBin: true + + '@openai/codex@0.133.0-darwin-arm64': + resolution: {integrity: sha512-W7f8+DckLujnqGlptKCzgJU+ooeHKMuk6KYgMFP6A9asn7YUsGUgJqjiBaX8oNcXO6w/pTbKGRARx1kCNS8lIg==} + engines: {node: '>=16'} + cpu: [arm64] + os: [darwin] + + '@openai/codex@0.133.0-darwin-x64': + resolution: {integrity: sha512-Ek8ikvLOiXZ8emcIJVBXxK6fm8ratBy0kaEt3JNisTNszxGshUHf/R4xxDxIyKNcUkYYXjW7A/rMwW3iu3OFlg==} + engines: {node: '>=16'} + cpu: [x64] + os: [darwin] + + '@openai/codex@0.133.0-linux-arm64': + resolution: {integrity: sha512-uKXYYSJ3mY16sp4hcG/4BMNRjva/ZS4oARiI1+7k8+NiuoAhdCGWNe5u4KJ3sMuL3tp/IXcmc6B56EFX1+WDBQ==} + engines: {node: '>=16'} + cpu: [arm64] + os: [linux] + + '@openai/codex@0.133.0-linux-x64': + resolution: {integrity: sha512-9YfyqrfUj/UZ2+aXE4zBz47t6RXbVni95ZorGsNh857vxYK/asVpUtR2cymo9lB3JaI4mQaKFfV/t7IRItqkuA==} + engines: {node: '>=16'} + cpu: [x64] + os: [linux] + + '@openai/codex@0.133.0-win32-arm64': + resolution: {integrity: sha512-mRzND0PSGHRoLk0X41GTSoc3tFjZSF4HgDlfjU5fiQcWVi0/kLb7Ku6/tPFT/X2hOLa3YdJkbIcHC0Hc9ni80g==} + engines: {node: '>=16'} + cpu: [arm64] + os: [win32] + + '@openai/codex@0.133.0-win32-x64': + resolution: {integrity: sha512-u3ji78DIPZCGJeELuovsAnaZH+vK9gsA4F6M1y+Uy2s80Sz7/i1S0KL81qGReYji3urSjgBpkQuNP47GXOqxrQ==} + engines: {node: '>=16'} + cpu: [x64] + os: [win32] + + '@opentelemetry/api@1.9.1': + resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==} engines: {node: '>=8.0.0'} '@orama/orama@3.1.18': resolution: {integrity: sha512-a61ljmRVVyG5MC/698C8/FfFDw5a8LOIvyOLW5fztgUXqUpc1jOfQzOitSCbge657OgXXThmY3Tk8fpiDb4UcA==} engines: {node: '>= 20.0.0'} - '@oxc-parser/binding-android-arm-eabi@0.128.0': - resolution: {integrity: sha512-aca6ZvzmCBUGOANQRiRQRZuRKYI3ENhcit6GisnknOOmcezfQc7xJ4dxlPU7MV7mOvrC7RNR1u3LAD7xyaiCxA==} + '@oxc-parser/binding-android-arm-eabi@0.130.0': + resolution: {integrity: sha512-h/xYU8/7ADWzVSf5I+YalLpj33LOy9CI/zgbJNIZ5eunRBG+Czqa3lZsvuPHHf3rOt6z1c5+UzoxjbAzAvhwVw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm] os: [android] - '@oxc-parser/binding-android-arm64@0.128.0': - resolution: {integrity: sha512-BbeDmuohoJ7Rz/it5wnkj69i/OsCPS3Z51nLEzwO/Y6YshtC4JU+15oNwhY8v4LRKRYclRc7ggOikwrsJ/eOEQ==} + '@oxc-parser/binding-android-arm64@0.130.0': + resolution: {integrity: sha512-oFWFJrsGv9siFM4HjMqKNB7IuIZD/SMmZdCXl8xyx7lDplGvPKyewpOo272rSWgMXe2Wx7bWI0Yj+gkHv4qbeg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [android] - '@oxc-parser/binding-darwin-arm64@0.128.0': - resolution: {integrity: sha512-tRUHPt80417QmvNpoSslJT1VY8NUbWdrWR+L14Zn+RbOTcaqB8E6PYE/ZGN8jjWBzqporiA/H4MfO50ew/NCNA==} + '@oxc-parser/binding-darwin-arm64@0.130.0': + resolution: {integrity: sha512-sGUzupdTplK9jQg7eJZ878HfEgQjJNBc6dAYVWJ9W5aU+J8rLfRJhTVsKThiu1pNwm6Y1qKCcbC6WhNWSXR3Ig==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [darwin] - '@oxc-parser/binding-darwin-x64@0.128.0': - resolution: {integrity: sha512-rWI2Hb1Nt3U/vKsjyNvZzDC8i/l144U20DKjhzaTmwIhIiSRGeroPWWiImwypmKLqrw8GuIixbWJkpGWLbkzrQ==} + '@oxc-parser/binding-darwin-x64@0.130.0': + resolution: {integrity: sha512-PsB4cdCISbC00Uy8eiD8bc2AkGWjZqrSrJnkBFuG2ptrrf6mZ2F5gLFSjOAVMMgZPg8B1D7OydJwLWSfyI2Plg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [darwin] - '@oxc-parser/binding-freebsd-x64@0.128.0': - resolution: {integrity: sha512-hhpdVMaNCLgQxjgNPeeFzSeJMmZPc5lKfv0NGSI3egZq9EdnEGqeC8JsYsQjK7PoQgbvZ17xlj0SO5ziH5Obkg==} + '@oxc-parser/binding-freebsd-x64@0.130.0': + resolution: {integrity: sha512-DgABp3l38hS77JbXCV4qk1+n6DPym5u8zzwuweokezm2tX194nDSJDENbDRECxVsiNbprKATLbk+Z5wlHT0OHw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [freebsd] - '@oxc-parser/binding-linux-arm-gnueabihf@0.128.0': - resolution: {integrity: sha512-093zNw0zZ/e/obML+rhlSdmnzR0mVZluPcAkxunEc5E3F0yBVsFn24Y1ILfsEte11Ud041qn/gp2OJ1jxNqUng==} + '@oxc-parser/binding-linux-arm-gnueabihf@0.130.0': + resolution: {integrity: sha512-4Kn3CTEmwFrzhTSC/JuUW16qovmaMdX7jeSKbL8w0pLtLww7To1a2XJi9Z5uD8QWUkfUHhqfV+VD6dVzBnWzoA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm] os: [linux] - '@oxc-parser/binding-linux-arm-musleabihf@0.128.0': - resolution: {integrity: sha512-fq7DmKmfC+dvD97IXrgbph6Jzwe0EDu+PYMofmzZ6fv5X1k9vtaqLpDGMuICO9MmUnyKAQmVl+wIv2RNy4Dz8g==} + '@oxc-parser/binding-linux-arm-musleabihf@0.130.0': + resolution: {integrity: sha512-D35KZM3F4rRu1uAFKyBlg3Gaf/ybCjyaPR1hfgvk5ex8NtcTmRgc0JgSighEyNg96TPrFhemFba68SZuxaha8w==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm] os: [linux] - '@oxc-parser/binding-linux-arm64-gnu@0.128.0': - resolution: {integrity: sha512-Xvm48jJah8TlIrURIjNOP/gNiGe6aKvCB+r06VliflFo8Kq7VOLE8PxtgShJzZIqubrgdMdYfvuPPozn7F6MbQ==} + '@oxc-parser/binding-linux-arm64-gnu@0.130.0': + resolution: {integrity: sha512-Q9o7oVlo955KHwS8l1u0bCzIx+JsZUA3XToLXC+MsMhye/9LeBQbt84nh120cl2XLy+TEzvugYDiHShg5yaX6Q==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] libc: [glibc] - '@oxc-parser/binding-linux-arm64-musl@0.128.0': - resolution: {integrity: sha512-M7iwBGmYJTx+pKOYFjI0buop4gJvlmcVzFGaXPt21DKpQkbQZG1f63Yg7LloIYT/t9yLxCw0Lhfx/RFlAlMSjA==} + '@oxc-parser/binding-linux-arm64-musl@0.130.0': + resolution: {integrity: sha512-EiJ/gC0ljbcwVpycC8YWw6ggMbtsPX8XMOt0mPx0aqWeMsNR+L9m05Flbvd5T+GlivG+GkSWQL7tM9SRFpM/dw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] libc: [musl] - '@oxc-parser/binding-linux-ppc64-gnu@0.128.0': - resolution: {integrity: sha512-21LGNIZb1Pcfk5/EGsqabrxv4yqQOWis1407JJrClS7XpFCrbvr74YAB1V+m54cYbwvO6UWwQqS4WecxiyfCRg==} + '@oxc-parser/binding-linux-ppc64-gnu@0.130.0': + resolution: {integrity: sha512-b+h/lsLLurp756dMGizNs5uPaJfyEdWrTcV5t8M609jWm1DEHB1StpRXCkyvwtkJx3m+qL5BNQ0dEKan/4yGFA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ppc64] os: [linux] libc: [glibc] - '@oxc-parser/binding-linux-riscv64-gnu@0.128.0': - resolution: {integrity: sha512-gyHjOTFpg9bTTYjxPmQirvufb89+VdZwVfcMtAUyPr6F5H8ZswvCQshK4qOW+Q+2Xyb33hduRgY/eFHJQjU/vQ==} + '@oxc-parser/binding-linux-riscv64-gnu@0.130.0': + resolution: {integrity: sha512-O19Cil83XAyjEFfo8WhkMwY58ALqZ7ckjGL+25mjMIuF84urWBeANH0FC8B8BsSSygWU3/1aY3ADdDbp+wlBnw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [riscv64] os: [linux] libc: [glibc] - '@oxc-parser/binding-linux-riscv64-musl@0.128.0': - resolution: {integrity: sha512-X6Q2oKUrP5GyDd2xniuEBLk6aFQCZ97W2+aVXGgJXdjx5t4/oFuA9ri0wLOUrBIX+qdSuK581snMBio4z910eA==} + '@oxc-parser/binding-linux-riscv64-musl@0.130.0': + resolution: {integrity: sha512-BgXRVC0+83n3YzCscLQjj6nbyeBIVeZYPTI4fFMAE4WNm2+4RXhWp03IVizL7esIz36kgmT48aebk1iM+cs8sw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [riscv64] os: [linux] libc: [musl] - '@oxc-parser/binding-linux-s390x-gnu@0.128.0': - resolution: {integrity: sha512-BdzTmqxfxoYkpgokoLaSnOX6T+R3/goL42klre2tnG+kHbG2TXS0VN+P5BPofH1axdKOHy5ei4ENZrjmCOt2lA==} + '@oxc-parser/binding-linux-s390x-gnu@0.130.0': + resolution: {integrity: sha512-6tJz0xvnGhsokE7N1WlUSBXibpYmT9xSJFS1Ce41Km/+8gQvdlW8MLhRv8PD0L7ix8vRG0FDDepp3jdOFzdVdw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [s390x] os: [linux] libc: [glibc] - '@oxc-parser/binding-linux-x64-gnu@0.128.0': - resolution: {integrity: sha512-OO1nW2Q7sSYYvJZpDHdvyFSdRaVcQqRijZSSmWVMqFxPYy8cEF45zJ9fcdIYuzIT3jYq6YRhEFm/VMWNWhE22Q==} + '@oxc-parser/binding-linux-x64-gnu@0.130.0': + resolution: {integrity: sha512-9aCWj83dp3heTQGmGnZGdIWgxjZrr/7VQ0TGFHH5PKByxJKF2Hcr4qvaSUHhhGEa3MSsDjTL1YDP8RAgdL5/Cg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] libc: [glibc] - '@oxc-parser/binding-linux-x64-musl@0.128.0': - resolution: {integrity: sha512-4NehAe404MRdoZVS9DW8C5XbJwbXIc/KfVlYdpi5vE4081zc9Y0YzKVqyOYj/Puye7/Do+ohaONBFWlEHYl9hw==} + '@oxc-parser/binding-linux-x64-musl@0.130.0': + resolution: {integrity: sha512-afXt87aZBqrUVli8TB/I8H1G50RDWcwirjWtXGXYqJ2ZqWEiErH7V72j3LUSDZaivmtu2OLX0KQ/mbhP81mr7A==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] libc: [musl] - '@oxc-parser/binding-openharmony-arm64@0.128.0': - resolution: {integrity: sha512-kVbqgW9xLL8bh8oc7aYOJilRKXE5G33+tE0jan+duo/9OriaFRpijcCwT2waWs2oqYROYq0GlE7/p3ywoshVeg==} + '@oxc-parser/binding-openharmony-arm64@0.130.0': + resolution: {integrity: sha512-I0NCrZV/YZuCGWgqwNN/GO/iXlLF2z+Wgc7u+Aa9N4P51oYeIa0XT+zVBUne4csO9GqxskXgI4g8JzzWGRpfOw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [openharmony] - '@oxc-parser/binding-wasm32-wasi@0.128.0': - resolution: {integrity: sha512-L38ojghJYHmgiz6fJd7jwLB/ESDBpB02NdFxh+smqVM6P2anCEvHn0jhaSrt5eVNR1Ak8+moOeftUlofeyvniA==} + '@oxc-parser/binding-wasm32-wasi@0.130.0': + resolution: {integrity: sha512-sJgQkGaBX0WJvPUDfwciex6IcTk5O5NLQ1bhEb6f3nBruh1GshKMRSMt2bxZlYrgBzjyBbJzsnO+InPG0bg+fA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [wasm32] - '@oxc-parser/binding-win32-arm64-msvc@0.128.0': - resolution: {integrity: sha512-xgvO35GyHBtjlQ5AEpaYr7Rll1rvY7zqIhT6ty8E3ezBW2J1SFLjIDEvI/tcgDg6oaseDAqVcM+jU1HuCekgZw==} + '@oxc-parser/binding-win32-arm64-msvc@0.130.0': + resolution: {integrity: sha512-bjcma99sQrNh6RY4mPO9yTkfxql6TDFoN3HWdK31RCKXwNhcDgJXW/l8PUtzKNiQ+9vpKJfJtQq+LklBuxSOBA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [win32] - '@oxc-parser/binding-win32-ia32-msvc@0.128.0': - resolution: {integrity: sha512-OY+3eM2SN72prHKRB22mPz8o5A/7dJ+f5DFLBVvggyZhEaNDAH9IB+ElMjmOkOIwf5MDCUAowCK7pAncNxzpBA==} + '@oxc-parser/binding-win32-ia32-msvc@0.130.0': + resolution: {integrity: sha512-hRYbv6HhpSTzT4xTiIkadLI7upLQxuOdLPR/9nL1fTjwhgutBTPXrwaAPb/jTFVx6/8C7Jb5HcUKhmNwloTbFA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ia32] os: [win32] - '@oxc-parser/binding-win32-x64-msvc@0.128.0': - resolution: {integrity: sha512-NE9ny+cPUCCObXa0IKLfj0tCdPd7pe/dz9ZpkxpUOymB3miNeMPybdlYYTBSGJUalMWeBM85/4JcCErCNTqOXw==} + '@oxc-parser/binding-win32-x64-msvc@0.130.0': + resolution: {integrity: sha512-RBpA9TsRucJq6HNVNCFF1iKg+QeTkLdZf7hi4xaOGCPvMZWvDHjQgSOEZMUpuW4JNciHbxNhLEYmz5CVygjVGQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [win32] - '@oxc-project/types@0.127.0': - resolution: {integrity: sha512-aIYXQBo4lCbO4z0R3FHeucQHpF46l2LbMdxRvqvuRuW2OxdnSkcng5B8+K12spgLDj93rtN3+J2Vac/TIO+ciQ==} + '@oxc-project/types@0.130.0': + resolution: {integrity: sha512-ibD2usx9JRu7f5pu2tMKMI4cpA4NgXJQoYRP4pQ7Pxmn1l6k/53qWtQWZayhYy3X4QZkt90Ot+mJEaeXouio6Q==} - '@oxc-project/types@0.128.0': - resolution: {integrity: sha512-huv1Y/LzBJkBVHt3OlC7u0zHBW9qXf1FdD7sGmc1rXc2P1mTwHssYv7jyGx5KAACSCH+9B3Bhn6Z9luHRvf7pQ==} + '@oxc-project/types@0.132.0': + resolution: {integrity: sha512-FESMOxil5Se014ui/Eq8fT5uHJo6nIRwH0PfJrZJXs6Gek3ZVFOrpUv3YIZT20m+extU98Hg1Ym72U58rlsxUQ==} '@oxc-resolver/binding-android-arm-eabi@11.19.1': resolution: {integrity: sha512-aUs47y+xyXHUKlbhqHUjBABjvycq6YSD7bpxSW7vplUmdzAlJ93yXY6ZR0c1o1x5A/QKbENCvs3+NlY8IpIVzg==} @@ -1553,6 +1606,12 @@ packages: resolution: {integrity: sha512-h104Kh26rR8tm+a3Qkc5S4VLYint3FE48as7+/5oCEcKR2idC/pF1G6AhIXKI+eHPJa/3J9i5z0Al47IeGHPkA==} engines: {node: '>=12'} + '@posthog/core@1.29.7': + resolution: {integrity: sha512-WcBD9/YQVGI9r/5+/IGeaPgsmTIg0YfyzaTei5TNlhmAeFOccnhs269rhtQJcAXngZFpvWSj+RTxX2ONdgxBDQ==} + + '@posthog/types@1.374.4': + resolution: {integrity: sha512-OHBo+gReFwPJtt/yLY6xxa1EYMp7Ti07O1C1KE9ZXXyyuLNqekRaHZxJ/SKUfEvt1LhFV/9sioz8O0xfsSffsQ==} + '@radix-ui/number@1.1.1': resolution: {integrity: sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g==} @@ -1918,103 +1977,103 @@ packages: '@radix-ui/rect@1.1.1': resolution: {integrity: sha512-HPwpGIzkl28mWyZqG52jiqDJ12waP11Pa1lGoiyUkIEuMLBP0oeK/C89esbXrxsky5we7dfd8U58nm0SgAWpVw==} - '@rolldown/binding-android-arm64@1.0.0-rc.17': - resolution: {integrity: sha512-s70pVGhw4zqGeFnXWvAzJDlvxhlRollagdCCKRgOsgUOH3N1l0LIxf83AtGzmb5SiVM4Hjl5HyarMRfdfj3DaQ==} + '@rolldown/binding-android-arm64@1.0.2': + resolution: {integrity: sha512-ZS4D1JPGn/MYQN/SYDWftIE/nVsM8j/AFOYEzAoOE2O3NktQOZru+/vYXGbR/qtdLdIfGCP0lcoJiYVzsEz+iQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [android] - '@rolldown/binding-darwin-arm64@1.0.0-rc.17': - resolution: {integrity: sha512-4ksWc9n0mhlZpZ9PMZgTGjeOPRu8MB1Z3Tz0Mo02eWfWCHMW1zN82Qz/pL/rC+yQa+8ZnutMF0JjJe7PjwasYw==} + '@rolldown/binding-darwin-arm64@1.0.2': + resolution: {integrity: sha512-vdFA9+C/rekyGce7WqHs/xoT0ioZEWaOFyZLIV1mEeNFaFDUQrPIo8Vs2GvJ6eetb3rzDUtUBgzto3ExpXJB3w==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [darwin] - '@rolldown/binding-darwin-x64@1.0.0-rc.17': - resolution: {integrity: sha512-SUSDOI6WwUVNcWxd02QEBjLdY1VPHvlEkw6T/8nYG322iYWCTxRb1vzk4E+mWWYehTp7ERibq54LSJGjmouOsw==} + '@rolldown/binding-darwin-x64@1.0.2': + resolution: {integrity: sha512-BewSOwTHazv77DTYiAZXSqqKZ4KP/KonFisDMVU7PImxoWfB2aepnPhd2E4SWz3zDzYgDNbs6jBmTdgNnF02GA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [darwin] - '@rolldown/binding-freebsd-x64@1.0.0-rc.17': - resolution: {integrity: sha512-hwnz3nw9dbJ05EDO/PvcjaaewqqDy7Y1rn1UO81l8iIK1GjenME75dl16ajbvSSMfv66WXSRCYKIqfgq2KCfxw==} + '@rolldown/binding-freebsd-x64@1.0.2': + resolution: {integrity: sha512-m41o7M0YWtUdqk61Tb+jnKb2rN++iRdIASlExkUoKfIAH30DOHCB8fVLzSUpbWHHU8esmEioY62PxzexE8MBuA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [freebsd] - '@rolldown/binding-linux-arm-gnueabihf@1.0.0-rc.17': - resolution: {integrity: sha512-IS+W7epTcwANmFSQFrS1SivEXHtl1JtuQA9wlxrZTcNi6mx+FDOYrakGevvvTwgj2JvWiK8B29/qD9BELZPyXQ==} + '@rolldown/binding-linux-arm-gnueabihf@1.0.2': + resolution: {integrity: sha512-jcojB9H7W/jS29pMKWAK1N+fU99vXodHDTatS3b3y/XSOCiHo0kkA74pL3jJmkoQtYpOCxDvaKs1fo2Ij/1X5w==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm] os: [linux] - '@rolldown/binding-linux-arm64-gnu@1.0.0-rc.17': - resolution: {integrity: sha512-e6usGaHKW5BMNZOymS1UcEYGowQMWcgZ71Z17Sl/h2+ZziNJ1a9n3Zvcz6LdRyIW5572wBCTH/Z+bKuZouGk9Q==} + '@rolldown/binding-linux-arm64-gnu@1.0.2': + resolution: {integrity: sha512-1jn6qDU5iiOgFgygDzKUuKP0maTi0/f1+sBLgvij/76C77Nm3ts6ufz9Bjg5q5dduxiUIxtq86JIoBvo1xQ4Ig==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] libc: [glibc] - '@rolldown/binding-linux-arm64-musl@1.0.0-rc.17': - resolution: {integrity: sha512-b/CgbwAJpmrRLp02RPfhbudf5tZnN9nsPWK82znefso832etkem8H7FSZwxrOI9djcdTP7U6YfNhbRnh7djErg==} + '@rolldown/binding-linux-arm64-musl@1.0.2': + resolution: {integrity: sha512-QVLO/czFMdoMFSqlX3bcswcJNm/23r+qoa/jgtmFc/qEp6/jXmIkDjF/XIo8dPfGaiwy1xfQn8o77L79GeXFgw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] libc: [musl] - '@rolldown/binding-linux-ppc64-gnu@1.0.0-rc.17': - resolution: {integrity: sha512-4EII1iNGRUN5WwGbF/kOh/EIkoDN9HsupgLQoXfY+D1oyJm7/F4t5PYU5n8SWZgG0FEwakyM8pGgwcBYruGTlA==} + '@rolldown/binding-linux-ppc64-gnu@1.0.2': + resolution: {integrity: sha512-hgO5Abm0w5UL6FEa2iFnZqo2KlK7TQ5QhV5x09hujBf7t5KzHQ1VmfPuTpqRy/rNlSxua3eWH374xxiVrP+lcA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ppc64] os: [linux] libc: [glibc] - '@rolldown/binding-linux-s390x-gnu@1.0.0-rc.17': - resolution: {integrity: sha512-AH8oq3XqQo4IibpVXvPeLDI5pzkpYn0WiZAfT05kFzoJ6tQNzwRdDYQ45M8I/gslbodRZwW8uxLhbSBbkv96rA==} + '@rolldown/binding-linux-s390x-gnu@1.0.2': + resolution: {integrity: sha512-fy8rXxuYEu602abC8MUNaPjYLIFzReOaEIEMKMUa0rFEUxNpVXhs15KSSQ4qlqSaM7B6rcj9rDZgADh/IGDzLQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [s390x] os: [linux] libc: [glibc] - '@rolldown/binding-linux-x64-gnu@1.0.0-rc.17': - resolution: {integrity: sha512-cLnjV3xfo7KslbU41Z7z8BH/E1y5mzUYzAqih1d1MDaIGZRCMqTijqLv76/P7fyHuvUcfGsIpqCdddbxLLK9rA==} + '@rolldown/binding-linux-x64-gnu@1.0.2': + resolution: {integrity: sha512-0+bOkiQ779+r1WpoHOWHqncvyySci0vKph+myNDYb+im6meJAzHQXay6oEgnkHuUGouM1LKTZwqKpBow6Kj7CQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] libc: [glibc] - '@rolldown/binding-linux-x64-musl@1.0.0-rc.17': - resolution: {integrity: sha512-0phclDw1spsL7dUB37sIARuis2tAgomCJXAHZlpt8PXZ4Ba0dRP1e+66lsRqrfhISeN9bEGNjQs+T/Fbd7oYGw==} + '@rolldown/binding-linux-x64-musl@1.0.2': + resolution: {integrity: sha512-mjSkrzZK5Qsl0a9d1JgILOiuZOSDTVdKENcSXBoqbzSrspLR/4/IRVDo5wd2GgZjNss/viBFJdeq+j7qH2nypw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] libc: [musl] - '@rolldown/binding-openharmony-arm64@1.0.0-rc.17': - resolution: {integrity: sha512-0ag/hEgXOwgw4t8QyQvUCxvEg+V0KBcA6YuOx9g0r02MprutRF5dyljgm3EmR02O292UX7UeS6HzWHAl6KgyhA==} + '@rolldown/binding-openharmony-arm64@1.0.2': + resolution: {integrity: sha512-1v5vHasdfQAZoEHakBV72LIFAC9JjnymsiKxp+GEr/ma3+NJCPSaYK+qavInOovJkgwFrs7GccX2d6IgDA3Z5w==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [openharmony] - '@rolldown/binding-wasm32-wasi@1.0.0-rc.17': - resolution: {integrity: sha512-LEXei6vo0E5wTGwpkJ4KoT3OZJRnglwldt5ziLzOlc6qqb55z4tWNq2A+PFqCJuvWWdP53CVhG1Z9NtToDPJrA==} + '@rolldown/binding-wasm32-wasi@1.0.2': + resolution: {integrity: sha512-mb1VobWn6NheziTk5/WEaR6AKVbrwT5sOi6C7zk3gy/pD1qtJfU1j4PgTo2NJnOtbL9Dl3Aeei8w9jJ7qC2jZQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [wasm32] - '@rolldown/binding-win32-arm64-msvc@1.0.0-rc.17': - resolution: {integrity: sha512-gUmyzBl3SPMa6hrqFUth9sVfcLBlYsbMzBx5PlexMroZStgzGqlZ26pYG89rBb45Mnia+oil6YAIFeEWGWhoZA==} + '@rolldown/binding-win32-arm64-msvc@1.0.2': + resolution: {integrity: sha512-SqKonF56vA/L2yHwHYcEp2P34URpOZ7d1fS635cTkpDnUtEGdUbhI6NzsPdqeSWvAAeGDrxjWjNmibDIdFf9/A==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [win32] - '@rolldown/binding-win32-x64-msvc@1.0.0-rc.17': - resolution: {integrity: sha512-3hkiolcUAvPB9FLb3UZdfjVVNWherN1f/skkGWJP/fgSQhYUZpSIRr0/I8ZK9TkF3F7kxvJAk0+IcKvPHk9qQg==} + '@rolldown/binding-win32-x64-msvc@1.0.2': + resolution: {integrity: sha512-v7qRI7gXLRINcOGXt+7YmAZ6iFuyZVMIoXAxhd8oP+DR9dLfL9GfNIx7PLMxmhZdvq8waUJBQiWN9EKNy+TRBQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [win32] - '@rolldown/pluginutils@1.0.0-rc.17': - resolution: {integrity: sha512-n8iosDOt6Ig1UhJ2AYqoIhHWh/isz0xpicHTzpKBeotdVsTEcxsSA/i3EVM7gQAj0rU27OLAxCjzlj15IWY7bg==} + '@rolldown/pluginutils@1.0.1': + resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} '@sec-ant/readable-stream@0.4.1': resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==} @@ -2063,32 +2122,32 @@ packages: peerDependencies: semantic-release: '>=20.1.0' - '@shikijs/core@4.0.2': - resolution: {integrity: sha512-hxT0YF4ExEqB8G/qFdtJvpmHXBYJ2lWW7qTHDarVkIudPFE6iCIrqdgWxGn5s+ppkGXI0aEGlibI0PAyzP3zlw==} + '@shikijs/core@4.1.0': + resolution: {integrity: sha512-jLJtSJeuFffqX6/inRE1zqU5aFv2hrszvYgq3OjbAgFRZiWv7abKMDdQzYxuSDfmUPQozZvI/kuy6VMTvnvqTQ==} engines: {node: '>=20'} - '@shikijs/engine-javascript@4.0.2': - resolution: {integrity: sha512-7PW0Nm49DcoUIQEXlJhNNBHyoGMjalRETTCcjMqEaMoJRLljy1Bi/EGV3/qLBgLKQejdspiiYuHGQW6dX94Nag==} + '@shikijs/engine-javascript@4.1.0': + resolution: {integrity: sha512-YquhawCUgaBfhsS72e2Y/dI59gCBNPHu3fEO/tvLaXrTssxZrY5ddjtNLTwndrMgPo8b3IscE+xoICDzpTmlFQ==} engines: {node: '>=20'} - '@shikijs/engine-oniguruma@4.0.2': - resolution: {integrity: sha512-UpCB9Y2sUKlS9z8juFSKz7ZtysmeXCgnRF0dlhXBkmQnek7lAToPte8DkxmEYGNTMii72zU/lyXiCB6StuZeJg==} + '@shikijs/engine-oniguruma@4.1.0': + resolution: {integrity: sha512-axLpjVs45YBvvINa+dJF+NPW+KtFkNXsFr4SDw2BMj9GdeMnGxVB9PQb2xXlJYovslt/nz6giedAyOANkfc7hg==} engines: {node: '>=20'} - '@shikijs/langs@4.0.2': - resolution: {integrity: sha512-KaXby5dvoeuZzN0rYQiPMjFoUrz4hgwIE+D6Du9owcHcl6/g16/yT5BQxSW5cGt2MZBz6Hl0YuRqf12omRfUUg==} + '@shikijs/langs@4.1.0': + resolution: {integrity: sha512-nwOMruEkbgdZfQ/b8CgpNBVOpvG1k0N5tbmgiFeqsan401+x3ILqlzZJowSla4Agmq4hG2Uf2wh5jLTEhR8VSg==} engines: {node: '>=20'} - '@shikijs/primitive@4.0.2': - resolution: {integrity: sha512-M6UMPrSa3fN5ayeJwFVl9qWofl273wtK1VG8ySDZ1mQBfhCpdd8nEx7nPZ/tk7k+TYcpqBZzj/AnwxT9lO+HJw==} + '@shikijs/primitive@4.1.0': + resolution: {integrity: sha512-zx2/2Uwj2q9X3KSyYREEhXO23xBw5WUhP4orK2lE4r+t9JGITmEe0JH+wPmJhqHpOT2bRRs6lAL945+LDvOAGw==} engines: {node: '>=20'} - '@shikijs/themes@4.0.2': - resolution: {integrity: sha512-mjCafwt8lJJaVSsQvNVrJumbnnj1RI8jbUKrPKgE6E3OvQKxnuRoBaYC51H4IGHePsGN/QtALglWBU7DoKDFnA==} + '@shikijs/themes@4.1.0': + resolution: {integrity: sha512-emCcTnUM7yO2wltYbaxm+yLvcCI4+h8XBKc4KmJ7EZUXoSGjcCHifkI//R4OFit9ewpg7H2/9tjOuXrT2v/Knw==} engines: {node: '>=20'} - '@shikijs/types@4.0.2': - resolution: {integrity: sha512-qzbeRooUTPnLE+sHD/Z8DStmaDgnbbc/pMrU203950aRqjX/6AFHeDYT+j00y2lPdz0ywJKx7o/7qnqTivtlXg==} + '@shikijs/types@4.1.0': + resolution: {integrity: sha512-3EQWX54fMpniOrDblzAhiwiJwpiTMW6+B9DWyUd9ska483tbayFYuw47UxwuPknI31bKnySfVQ/QW+jFL4rFdA==} engines: {node: '>=20'} '@shikijs/vscode-textmate@10.0.2': @@ -2112,222 +2171,164 @@ packages: resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} engines: {node: '>=18'} - '@smithy/chunked-blob-reader-native@4.2.3': - resolution: {integrity: sha512-jA5k5Udn7Y5717L86h4EIv06wIr3xn8GM1qHRi/Nf31annXcXHJjBKvgztnbn2TxH3xWrPBfgwHsOwZf0UmQWw==} + '@smithy/config-resolver@4.5.3': + resolution: {integrity: sha512-TpS6Am5zSEtx3ow7VynThEL7UwRM06zZZcmFaP6Ij9hqKPfsFhTYCLcgU7gjFjw9QAI2kzwXrfS7InH8BivJTA==} engines: {node: '>=18.0.0'} - '@smithy/chunked-blob-reader@5.2.2': - resolution: {integrity: sha512-St+kVicSyayWQca+I1rGitaOEH6uKgE8IUWoYnnEX26SWdWQcL6LvMSD19Lg+vYHKdT9B2Zuu7rd3i6Wnyb/iw==} + '@smithy/core@3.24.3': + resolution: {integrity: sha512-Ep/7tPamGY8mgESE3LyLKtxJyy6U52WWAqr/3wial47Sj4u3PiIF73AOGI27UyLy9duTkhZbgzodOfLV4TduZg==} engines: {node: '>=18.0.0'} - '@smithy/config-resolver@4.4.17': - resolution: {integrity: sha512-TzDZcAnhTyAHbXVxWZo7/tEcrIeFq20IBk8So3OLOetWpR8EwY/yEqBMBFaJMeyEiREDq4NfEl+qO3OAUD+vbQ==} + '@smithy/credential-provider-imds@4.3.3': + resolution: {integrity: sha512-I2Bti0DKFo2IJyN28ijCsx51BAumEYR4/1yZ1FXyBygy9MqbnMqCev4JPth/MbpRfBSRAX35hITSnAdJRo1u5w==} engines: {node: '>=18.0.0'} - '@smithy/core@3.23.17': - resolution: {integrity: sha512-x7BlLbUFL8NWCGjMF9C+1N5cVCxcPa7g6Tv9B4A2luWx3be3oU8hQ96wIwxe/s7OhIzvoJH73HAUSg5JXVlEtQ==} + '@smithy/eventstream-serde-browser@4.3.3': + resolution: {integrity: sha512-LXg5yYJPYnVSrpa6LOZ+/wqpI2OlIccy7j5F16EFNYDbXWmnhry/PFRRPyM30H+hJeqfVgckFuvNGnAGCt56cA==} engines: {node: '>=18.0.0'} - '@smithy/credential-provider-imds@4.2.14': - resolution: {integrity: sha512-Au28zBN48ZAoXdooGUHemuVBrkE+Ie6RPmGNIAJsFqj33Vhb6xAgRifUydZ2aY+M+KaMAETAlKk5NC5h1G7wpg==} + '@smithy/eventstream-serde-config-resolver@4.4.3': + resolution: {integrity: sha512-MdQxEX5SFNc3QmpiLXtcZXsWk4imCfGVN7Ikz9I/XvavypvHT4mqxwo5JHdr/LBKCfAv89+8193ZWlUwDp8YXQ==} engines: {node: '>=18.0.0'} - '@smithy/eventstream-codec@4.2.14': - resolution: {integrity: sha512-erZq0nOIpzfeZdCyzZjdJb4nVSKLUmSkaQUVkRGQTXs30gyUGeKnrYEg+Xe1W5gE3aReS7IgsvANwVPxSzY6Pw==} + '@smithy/eventstream-serde-node@4.3.3': + resolution: {integrity: sha512-54RbRsw9eVaVnqYUXi3F6nMAPgUyKsBvAKBY2lf+81mIgM7N+yS9V5LYk7yUGbrM789b2e1qBuyDSjX1/Axxcw==} engines: {node: '>=18.0.0'} - '@smithy/eventstream-serde-browser@4.2.14': - resolution: {integrity: sha512-8IelTCtTctWRbb+0Dcy+C0aICh1qa0qWXqgjcXDmMuCvPJRnv26hiDZoAau2ILOniki65mCPKqOQs/BaWvO4CQ==} + '@smithy/fetch-http-handler@5.4.3': + resolution: {integrity: sha512-F+DRf8IJazRJgYog2A/yJK7eYVc0rqTlRzO+5ZxjJd4WkZoKz0IJRncf7G6t1pdVT3kryJcwuTFhN1c5m6N47A==} engines: {node: '>=18.0.0'} - '@smithy/eventstream-serde-config-resolver@4.3.14': - resolution: {integrity: sha512-sqHiHpYRYo3FJlaIxD1J8PhbcmJAm7IuM16mVnwSkCToD7g00IBZzKuiLNMGmftULmEUX6/UAz8/NN5uMP8bVA==} + '@smithy/hash-blob-browser@4.3.3': + resolution: {integrity: sha512-TkGfDlYeWOGwYvAunHHHmKgvFtD7DFAl6gWxATI4pv4B6w0Wnx6RK5zCMoXTTqMVd+zPcWm7w8RPTgHytoCDJA==} engines: {node: '>=18.0.0'} - '@smithy/eventstream-serde-node@4.2.14': - resolution: {integrity: sha512-Ht/8BuGlKfFTy0H3+8eEu0vdpwGztCnaLLXtpXNdQqiR7Hj4vFScU3T436vRAjATglOIPjJXronY+1WxxNLSiw==} + '@smithy/hash-node@4.3.3': + resolution: {integrity: sha512-tSUA38sM7kzMoLhqQ2aCGTwLXovjurz3jjG+a0sxqD4qT/4FhQr/wxMdhCumT70giM+axC1pPjimAHLlEQCfzw==} engines: {node: '>=18.0.0'} - '@smithy/eventstream-serde-universal@4.2.14': - resolution: {integrity: sha512-lWyt4T2XQZUZgK3tQ3Wn0w3XBvZsK/vjTuJl6bXbnGZBHH0ZUSONTYiK9TgjTTzU54xQr3DRFwpjmhp0oLm3gg==} + '@smithy/hash-stream-node@4.3.3': + resolution: {integrity: sha512-ZyDAlpKKc7BKHUp+kDBiTwNhiHrOf3syQdvQadvnwWs0QJhYMHMg6QSarlhpzN6qr+KBFM/oF/xP/bvzR6KI9w==} engines: {node: '>=18.0.0'} - '@smithy/fetch-http-handler@5.3.17': - resolution: {integrity: sha512-bXOvQzaSm6MnmLaWA1elgfQcAtN4UP3vXqV97bHuoOrHQOJiLT3ds6o9eo5bqd0TJfRFpzdGnDQdW3FACiAVdw==} - engines: {node: '>=18.0.0'} - - '@smithy/hash-blob-browser@4.2.15': - resolution: {integrity: sha512-0PJ4Al3fg2nM4qKrAIxyNcApgqHAXcBkN8FeizOz69z0rb26uZ6lMESYtxegaTlXB5Hj84JfwMPavMrwDMjucA==} - engines: {node: '>=18.0.0'} - - '@smithy/hash-node@4.2.14': - resolution: {integrity: sha512-8ZBDY2DD4wr+GGjTpPtiglEsqr0lUP+KHqgZcWczFf6qeZ/YRjMIOoQWVQlmwu7EtxKTd8YXD8lblmYcpBIA1g==} - engines: {node: '>=18.0.0'} - - '@smithy/hash-stream-node@4.2.14': - resolution: {integrity: sha512-tw4GANWkZPb6+BdD4Fgucqzey2+r73Z/GRo9zklsCdwrnxxumUV83ZIaBDdudV4Ylazw3EPTiJZhpX42105ruQ==} - engines: {node: '>=18.0.0'} - - '@smithy/invalid-dependency@4.2.14': - resolution: {integrity: sha512-c21qJiTSb25xvvOp+H2TNZzPCngrvl5vIPqPB8zQ/DmJF4QWXO19x1dWfMJZ6wZuuWUPPm0gV8C0cU3+ifcWuw==} + '@smithy/invalid-dependency@4.3.3': + resolution: {integrity: sha512-wUWowbCm7DGczl6bfLI6wGGtoxwN5Pon8DhF0Q8AA4NvgLwYfLo3h2DWI7sHr33lLcEsyTLQKeUeTHydqSfQ5Q==} engines: {node: '>=18.0.0'} '@smithy/is-array-buffer@2.2.0': resolution: {integrity: sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==} engines: {node: '>=14.0.0'} - '@smithy/is-array-buffer@4.2.2': - resolution: {integrity: sha512-n6rQ4N8Jj4YTQO3YFrlgZuwKodf4zUFs7EJIWH86pSCWBaAtAGBFfCM7Wx6D2bBJ2xqFNxGBSrUWswT3M0VJow==} + '@smithy/md5-js@4.3.3': + resolution: {integrity: sha512-pFw8gEMrHw9BbRwNm//UU4WgnVO7+dhfFRaSAkFPfwslWU2LXt0mM+oap3iFwGbdD8kuAWIeOAxqSiamOcM3Dw==} engines: {node: '>=18.0.0'} - '@smithy/md5-js@4.2.14': - resolution: {integrity: sha512-V2v0vx+h0iUSNG1Alt+GNBMSLGCrl9iVsdd+Ap67HPM9PN479x12V8LkuMoKImNZxn3MXeuyUjls+/7ZACZghA==} + '@smithy/middleware-content-length@4.3.3': + resolution: {integrity: sha512-Up1XAYnj6oxFBypWpkhNpgX+yReQxkKAV/iLaeP0KVLb2oTkmA9X+UJuGBVvEA9uZIN06y0irDi7sBMuTZMVJg==} engines: {node: '>=18.0.0'} - '@smithy/middleware-content-length@4.2.14': - resolution: {integrity: sha512-xhHq7fX4/3lv5NHxLUk3OeEvl0xZ+Ek3qIbWaCL4f9JwgDZEclPBElljaZCAItdGPQl/kSM4LPMOpy1MYgprpw==} + '@smithy/middleware-endpoint@4.5.3': + resolution: {integrity: sha512-p60HGFflWsJC6V9GAYeFgbfORn+9ILx8FqgMa/8PzA0rhIUxF57EKoOR4Irs6oe1oy8RLzhjhcGS8CBtPv/t+Q==} engines: {node: '>=18.0.0'} - '@smithy/middleware-endpoint@4.4.32': - resolution: {integrity: sha512-ZZkgyjnJppiZbIm6Qbx92pbXYi1uzenIvGhBSCDlc7NwuAkiqSgS75j1czAD25ZLs2FjMjYy1q7gyRVWG6JA0Q==} + '@smithy/middleware-retry@4.6.3': + resolution: {integrity: sha512-MnfYnJs3cBXK3ZBqbPzXRPHIp+QtgpkX5NogcUOWHPU5GbgTAQSIfPLi91lTcEbkFDcH2YbgjLPQjWeyQ689rA==} engines: {node: '>=18.0.0'} - '@smithy/middleware-retry@4.5.7': - resolution: {integrity: sha512-bRt6ZImqVSeTk39Nm81K20ObIiAZ3WefY7G6+iz/0tZjs4dgRRjvRX2sgsH+zi6iDCRR/aQvQofLKxxz4rPBZg==} + '@smithy/middleware-serde@4.3.3': + resolution: {integrity: sha512-RUVCZgn92izDAARs5OJSM2+KWSfTRvQWwN9t0MmiybT3pquRgDx9vD9t/YZjd/5lwcFbsNuPojJSddYQEZGeWw==} engines: {node: '>=18.0.0'} - '@smithy/middleware-serde@4.2.20': - resolution: {integrity: sha512-Lx9JMO9vArPtiChE3wbEZ5akMIDQpWQtlu90lhACQmNOXcGXRbaDywMHDzuDZ2OkZzP+9wQfZi3YJT9F67zTQQ==} + '@smithy/middleware-stack@4.3.3': + resolution: {integrity: sha512-+BPabWluqxo3EfMMvOgnAmPtWnCSzj+gf5mJ27wTZUbvS0hpdUIU1g80R01bEGKZx4JCi8P58jAXD9FUGMjhwA==} engines: {node: '>=18.0.0'} - '@smithy/middleware-stack@4.2.14': - resolution: {integrity: sha512-2dvkUKLuFdKsCRmOE4Mn63co0Djtsm+JMh0bYZQupN1pJwMeE8FmQmRLLzzEMN0dnNi7CDCYYH8F0EVwWiPBeA==} + '@smithy/node-config-provider@4.4.3': + resolution: {integrity: sha512-vDtz5OuytrjP4o9GtAOz1JloN003p94utJIQeO0WAjorhpafFFjpbDOrP6btPoCN3UxaU/U84OIEt5dM7ZRRLA==} engines: {node: '>=18.0.0'} - '@smithy/node-config-provider@4.3.14': - resolution: {integrity: sha512-S+gFjyo/weSVL0P1b9Ts8C/CwIfNCgUPikk3sl6QVsfE/uUuO+QsF+NsE/JkpvWqqyz1wg7HFdiaZuj5CoBMRg==} + '@smithy/node-http-handler@4.7.3': + resolution: {integrity: sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA==} engines: {node: '>=18.0.0'} - '@smithy/node-http-handler@4.6.1': - resolution: {integrity: sha512-iB+orM4x3xrr57X3YaXazfKnntl0LHlZB1kcXSGzMV1Tt0+YwEjGlbjk/44qEGtBzXAz6yFDzkYTKSV6Pj2HUg==} + '@smithy/protocol-http@5.4.3': + resolution: {integrity: sha512-P16TBD/d8ZcD9MHQ0ubQ9BbOYSd5HZKbHOLsyFWxKk2oBEoghbRFPfGOoqToZX1yrfLITXRylL16EyPP4IzLPg==} engines: {node: '>=18.0.0'} - '@smithy/property-provider@4.2.14': - resolution: {integrity: sha512-WuM31CgfsnQ/10i7NYr0PyxqknD72Y5uMfUMVSniPjbEPceiTErb4eIqJQ+pdxNEAUEWrewrGjIRjVbVHsxZiQ==} + '@smithy/signature-v4@5.4.3': + resolution: {integrity: sha512-53+75QuPl6DL+ct6vVEB51FDO5oulXr20TPV46VvJZg76lIlXNWfxi8j+G2V/t0I2qxCBOa3vX/8bmjrpFVo9g==} engines: {node: '>=18.0.0'} - '@smithy/protocol-http@5.3.14': - resolution: {integrity: sha512-dN5F8kHx8RNU0r+pCwNmFZyz6ChjMkzShy/zup6MtkRmmix4vZzJdW+di7x//b1LiynIev88FM18ie+wwPcQtQ==} + '@smithy/smithy-client@4.13.3': + resolution: {integrity: sha512-Z8mQ+YryjP5krDadV6unnp5035L4S1brafXpTiRmjPweKSaQ6X9CYDYWvmEggXjDIa1oufX/2a/bdwu8EIz/lw==} engines: {node: '>=18.0.0'} - '@smithy/querystring-builder@4.2.14': - resolution: {integrity: sha512-XYA5Z0IqTeF+5XDdh4BBmSA0HvbgVZIyv4cmOoUheDNR57K1HgBp9ukUMx3Cr3XpDHHpLBnexPE3LAtDsZkj2A==} + '@smithy/types@4.14.2': + resolution: {integrity: sha512-P+otAxbV4CqBybp7EkcJCrig63yE2E7PuNVOmilVMRcx/O+QDzGULTrKsq4DV13gSfak9ObPrWaHl/9bL5YcWw==} engines: {node: '>=18.0.0'} - '@smithy/querystring-parser@4.2.14': - resolution: {integrity: sha512-hr+YyqBD23GVvRxGGrcc/oOeNlK3PzT5Fu4dzrDXxzS1LpFiuL2PQQqKPs87M79aW7ziMs+nvB3qdw77SqE7Lw==} + '@smithy/url-parser@4.3.3': + resolution: {integrity: sha512-TsMTAOnjuMOv1zJBw8cfYGWhopyc3og8tZX/KuyCPjg7V3ji3f4YjFOVu843UjBmrfS/+X6kwFv5ZKg7sSm1bQ==} engines: {node: '>=18.0.0'} - '@smithy/service-error-classification@4.3.1': - resolution: {integrity: sha512-aUQuDGh760ts/8MU+APjIZhlLPKhIIfqyzZaJikLEIMrdxFvxuLYD0WxWzaYWpmLbQlXDe9p7EWM3HsBe0K6Gw==} + '@smithy/util-base64@4.4.3': + resolution: {integrity: sha512-91lxjhFpAktA9yPBxniqVR/NSH9zyjMjLmoa+jbQHQFR9WiJA+n61T7HBrfh5APdEoAledJwGq8l4cS+ZJFUnQ==} engines: {node: '>=18.0.0'} - '@smithy/shared-ini-file-loader@4.4.9': - resolution: {integrity: sha512-495/V2I15SHgedSJoDPD23JuSfKAp726ZI1V0wtjB07Wh7q/0tri/0e0DLefZCHgxZonrGKt/OCTpAtP1wE1kQ==} + '@smithy/util-body-length-browser@4.3.3': + resolution: {integrity: sha512-/M6Ya1Fjq8hg3rYjiwwqTen6s1bAa3U3g/2eicBaBQfaoa4ymLUke/x4T8mwb9dSq/L8TQ4YgndS0MaB9ShgmA==} engines: {node: '>=18.0.0'} - '@smithy/signature-v4@5.3.14': - resolution: {integrity: sha512-1D9Y/nmlVjCeSivCbhZ7hgEpmHyY1h0GvpSZt3l0xcD9JjmjVC1CHOozS6+Gh+/ldMH8JuJ6cujObQqfayAVFA==} - engines: {node: '>=18.0.0'} - - '@smithy/smithy-client@4.12.13': - resolution: {integrity: sha512-y/Pcj1V9+qG98gyu1gvftHB7rDpdh+7kIBIggs55yGm3JdtBV8GT8IFF3a1qxZ79QnaJHX9GXzvBG6tAd+czJA==} - engines: {node: '>=18.0.0'} - - '@smithy/types@4.14.1': - resolution: {integrity: sha512-59b5HtSVrVR/eYNei3BUj3DCPKD/G7EtDDe7OEJE7i7FtQFugYo6MxbotS8mVJkLNVf8gYaAlEBwwtJ9HzhWSg==} - engines: {node: '>=18.0.0'} - - '@smithy/url-parser@4.2.14': - resolution: {integrity: sha512-p06BiBigJ8bTA3MgnOfCtDUWnAMY0YfedO/GRpmc7p+wg3KW8vbXy1xwSu5ASy0wV7rRYtlfZOIKH4XqfhjSQQ==} - engines: {node: '>=18.0.0'} - - '@smithy/util-base64@4.3.2': - resolution: {integrity: sha512-XRH6b0H/5A3SgblmMa5ErXQ2XKhfbQB+Fm/oyLZ2O2kCUrwgg55bU0RekmzAhuwOjA9qdN5VU2BprOvGGUkOOQ==} - engines: {node: '>=18.0.0'} - - '@smithy/util-body-length-browser@4.2.2': - resolution: {integrity: sha512-JKCrLNOup3OOgmzeaKQwi4ZCTWlYR5H4Gm1r2uTMVBXoemo1UEghk5vtMi1xSu2ymgKVGW631e2fp9/R610ZjQ==} - engines: {node: '>=18.0.0'} - - '@smithy/util-body-length-node@4.2.3': - resolution: {integrity: sha512-ZkJGvqBzMHVHE7r/hcuCxlTY8pQr1kMtdsVPs7ex4mMU+EAbcXppfo5NmyxMYi2XU49eqaz56j2gsk4dHHPG/g==} + '@smithy/util-body-length-node@4.3.3': + resolution: {integrity: sha512-M+zdSrevWj0grtZx2RBULPUyjTq1aB+n+13Hrm9owiGpow6DqY/WqiSj6sHVQy/rKp0j7NzV3TNf2LrwDel8JQ==} engines: {node: '>=18.0.0'} '@smithy/util-buffer-from@2.2.0': resolution: {integrity: sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==} engines: {node: '>=14.0.0'} - '@smithy/util-buffer-from@4.2.2': - resolution: {integrity: sha512-FDXD7cvUoFWwN6vtQfEta540Y/YBe5JneK3SoZg9bThSoOAC/eGeYEua6RkBgKjGa/sz6Y+DuBZj3+YEY21y4Q==} + '@smithy/util-defaults-mode-browser@4.4.3': + resolution: {integrity: sha512-Q60hxKkMEkmBsOEzxlMWEymBWov0dtWGgoJhOUs6mE8k2FDPjK8NlsRdMkmO80n2pwzreHtrYcX5jiRP7ZkP3w==} engines: {node: '>=18.0.0'} - '@smithy/util-config-provider@4.2.2': - resolution: {integrity: sha512-dWU03V3XUprJwaUIFVv4iOnS1FC9HnMHDfUrlNDSh4315v0cWyaIErP8KiqGVbf5z+JupoVpNM7ZB3jFiTejvQ==} + '@smithy/util-defaults-mode-node@4.3.3': + resolution: {integrity: sha512-RYj+8gr95WiiBqvVghoRvL12NS9ryvLyufp7FOs7EzKwGX0W5gOVlXdCrFkJScSf8gxdjQMRyIZ3Y82/MvXQ3Q==} engines: {node: '>=18.0.0'} - '@smithy/util-defaults-mode-browser@4.3.49': - resolution: {integrity: sha512-a5bNrdiONYB/qE2BuKegvUMd/+ZDwdg4vsNuuSzYE8qs2EYAdK9CynL+Rzn29PbPiUqoz/cbpRbcLzD5lEevHw==} + '@smithy/util-endpoints@3.5.3': + resolution: {integrity: sha512-2JqSmzQtKDKqBckLl/9NXTL1fY+zQBU5fNGMpud7AT65vql0tVFhb2UEZNZmLSHayLeD+X/Qzn84oXw5KS+KSQ==} engines: {node: '>=18.0.0'} - '@smithy/util-defaults-mode-node@4.2.54': - resolution: {integrity: sha512-g1cvrJvOnzeJgEdf7AE4luI7gp6L8weE0y9a9wQUSGtjb8QRHDbCJYuE4Sy0SD9N8RrnNPFsPltAz/OSoBR9Zw==} + '@smithy/util-middleware@4.3.3': + resolution: {integrity: sha512-8NZwlQ+nyAIWn9YZxH14FC8ca0i6ZGW1aJyPjD+zMZz3k9jOhXXKhdCSRvjmcSYLW42uhbrxavXqMkrTKHyY3A==} engines: {node: '>=18.0.0'} - '@smithy/util-endpoints@3.4.2': - resolution: {integrity: sha512-a55Tr+3OKld4TTtnT+RhKOQHyPxm3j/xL4OR83WBUhLJaKDS9dnJ7arRMOp3t31dcLhApwG9bgvrRXBHlLdIkg==} + '@smithy/util-retry@4.4.3': + resolution: {integrity: sha512-8RJXeU5lEhdNfXm4XAuHlf6VtNzd279Z2FJZSR7VaELYCR46ffgjJBSjc+3UAy7V1YqBOLV0G9gWhLB/nA44nA==} engines: {node: '>=18.0.0'} - '@smithy/util-hex-encoding@4.2.2': - resolution: {integrity: sha512-Qcz3W5vuHK4sLQdyT93k/rfrUwdJ8/HZ+nMUOyGdpeGA1Wxt65zYwi3oEl9kOM+RswvYq90fzkNDahPS8K0OIg==} - engines: {node: '>=18.0.0'} - - '@smithy/util-middleware@4.2.14': - resolution: {integrity: sha512-1Su2vj9RYNDEv/V+2E+jXkkwGsgR7dc4sfHn9Z7ruzQHJIEni9zzw5CauvRXlFJfmgcqYP8fWa0dkh2Q2YaQyw==} - engines: {node: '>=18.0.0'} - - '@smithy/util-retry@4.3.6': - resolution: {integrity: sha512-p6/FO1n2KxMeQyna067i0uJ6TSbb165ZhnRtCpWh4Foxqbfc6oW+XITaL8QkFJj3KFnDe2URt4gOhgU06EP9ew==} - engines: {node: '>=18.0.0'} - deprecated: '@smithy/util-retry v4.3.6 contains a bug in Adaptive Retry, see https://github.com/smithy-lang/smithy-typescript/issues/1993. Upgrade to 4.3.7+' - - '@smithy/util-stream@4.5.25': - resolution: {integrity: sha512-/PFpG4k8Ze8Ei+mMKj3oiPICYekthuzePZMgZbCqMiXIHHf4n2aZ4Ps0aSRShycFTGuj/J6XldmC0x0DwednIA==} - engines: {node: '>=18.0.0'} - - '@smithy/util-uri-escape@4.2.2': - resolution: {integrity: sha512-2kAStBlvq+lTXHyAZYfJRb/DfS3rsinLiwb+69SstC9Vb0s9vNWkRwpnj918Pfi85mzi42sOqdV72OLxWAISnw==} + '@smithy/util-stream@4.6.3': + resolution: {integrity: sha512-DSpJpPg0rQwjZk9/CSlOTplD6xSUu+bz8eDJQkq/Fmy9JlSD4ZGhXG/qFl0aRHmouDbBF75tnZ00lPxiL/sgRQ==} engines: {node: '>=18.0.0'} '@smithy/util-utf8@2.3.0': resolution: {integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==} engines: {node: '>=14.0.0'} - '@smithy/util-utf8@4.2.2': - resolution: {integrity: sha512-75MeYpjdWRe8M5E3AW0O4Cx3UadweS+cwdXjwYGBW5h/gxxnbeZ877sLPX/ZJA9GVTlL/qG0dXP29JWFCD1Ayw==} + '@smithy/util-utf8@4.3.3': + resolution: {integrity: sha512-c1QpRBn3aMsoqE64dd4Imgjy8Pynfw+eR7GkjElquxUFSnezwYVaOFm8JcYa+Bo/5ssbEyPKcT3+4bmrWYh6eQ==} engines: {node: '>=18.0.0'} - '@smithy/util-waiter@4.3.0': - resolution: {integrity: sha512-JyjYmLAfS+pdxF92o4yLgEoy0zhayKTw73FU1aofLWwLcJw7iSqIY2exGmMTrl/lmZugP5p/zxdFSippJDfKWA==} - engines: {node: '>=18.0.0'} - - '@smithy/uuid@1.1.2': - resolution: {integrity: sha512-O/IEdcCUKkubz60tFbGA7ceITTAJsty+lBjNoorP4Z6XRqaFb/OjQjZODophEcuq68nKm6/0r+6/lLQ+XVpk8g==} + '@smithy/util-waiter@4.4.3': + resolution: {integrity: sha512-WSHSF865zDGFGtJdMmYPI2Blq/MbUrn5CB4bLDg4ARbQ9z7oA87ZZ/FSiwNZbQrU/EiVyl9lpINswALgI4lZXA==} engines: {node: '>=18.0.0'} '@so-ric/colorspace@1.1.6': resolution: {integrity: sha512-/KiKkpHNOBgkFJwu9sh48LkHSMYGyuTcSFK/qMBdnOAlrRJzRSXAOFB5qwzaVQuDl8wAvHVMkaASQDReTahxuw==} + '@stablelib/base64@1.0.1': + resolution: {integrity: sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -2432,8 +2433,8 @@ packages: '@tediousjs/connection-string@1.1.0': resolution: {integrity: sha512-z9ZBWEG+8pIB5V1zYzlRPXx0oRJ5H7coPnMQK8EZOw03UTPI9Umn6viL36f5w+CuqkKsnCM50RVStpjZmR0Bng==} - '@tybys/wasm-util@0.10.1': - resolution: {integrity: sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==} + '@tybys/wasm-util@0.10.2': + resolution: {integrity: sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==} '@types/better-sqlite3@7.6.13': resolution: {integrity: sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==} @@ -2468,8 +2469,8 @@ packages: '@types/estree-jsx@1.0.5': resolution: {integrity: sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==} - '@types/estree@1.0.8': - resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} '@types/hast@3.0.4': resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==} @@ -2486,8 +2487,8 @@ packages: '@types/mssql@12.3.0': resolution: {integrity: sha512-+jy+AJtfuTDI5+nhh0hDNcir1p8P+pf+qsHXpUpYvg7EikxUUePBe+a+Kr6j/Xs89o4EbHlVzrh0HOxbqWM31Q==} - '@types/node@24.12.2': - resolution: {integrity: sha512-A1sre26ke7HDIuY/M23nd9gfB+nrmhtYyMINbjI1zHJxYteKR6qSMX56FsmjMcDb3SMcjJg5BiRRgOCC/yBD0g==} + '@types/node@24.12.4': + resolution: {integrity: sha512-GUUEShf+PBCGW2KaXwcIt3Yk+e3pkKwWKb9GSyM9WQVE+ep2jzmHdGsHzu4wgcZy5fN9FBdVzjpBQsYlpfpgLA==} '@types/normalize-package-data@2.4.4': resolution: {integrity: sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==} @@ -2500,12 +2501,15 @@ packages: peerDependencies: '@types/react': ^19.2.0 - '@types/react@19.2.14': - resolution: {integrity: sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==} + '@types/react@19.2.15': + resolution: {integrity: sha512-eRwcGNHve+E8qtEQSSRl6urh+rFop4v8gm6O8rGv25CodbvFdLjA1vVQ1KkiFE0w0UPOnb8tDiFKL5lp0rtY5Q==} '@types/readable-stream@4.0.23': resolution: {integrity: sha512-wwXrtQvbMHxCbBgjHaMGEmImFTQxxpfMOR/ZoQnXxB1woqkUbdLGFDgauo00Py9IudiaqSeiBiulSV9i6XIPig==} + '@types/semver@7.7.1': + resolution: {integrity: sha512-FmgJfu+MOcQ370SD0ev7EI8TlCAfKYU+B4m5T3yXc1CiRN94g/SZPtsCkk506aUDtlMnFZvasDwHHUcZUEaYuA==} + '@types/triple-beam@1.3.5': resolution: {integrity: sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==} @@ -2526,20 +2530,20 @@ packages: resolution: {integrity: sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==} engines: {node: '>= 20'} - '@vitest/coverage-v8@4.1.6': - resolution: {integrity: sha512-36l628fQ/9a/8ihy97eOtEnvWQEdqULQOJtcaxtoNq0G1w3Mxd4szSahOaMM9/NGyZ+hyKcMtIW/WIxq0XQViQ==} + '@vitest/coverage-v8@4.1.7': + resolution: {integrity: sha512-qsYPeXc5Q9dFLd1i8Ap+Bx8sQgcp+rFVQo4R0dDsWNBzl26ldVF1qOO+RL24K7FDrR6pA+50XedRLSoSG24bVQ==} peerDependencies: - '@vitest/browser': 4.1.6 - vitest: 4.1.6 + '@vitest/browser': 4.1.7 + vitest: 4.1.7 peerDependenciesMeta: '@vitest/browser': optional: true - '@vitest/expect@4.1.6': - resolution: {integrity: sha512-7EHDquPthALSV0jhhjgEW8FXaviMx7rSqu8W6oqCoAuOhKov814P99QDV1pxMA3QPv21YudvJngIhjrNI4opLg==} + '@vitest/expect@4.1.7': + resolution: {integrity: sha512-1R+tw0ortHEbZDGMymm+pN7/AFQ/RkFFdtd7EN+VBpynKmLbP8A3rpEXdshBJ7+8hQ9zBJh/i1s0yKNtxAnU7w==} - '@vitest/mocker@4.1.6': - resolution: {integrity: sha512-MCFc63czMjEInOlcY2cpQCvCN+KgbAn+60xu9cMgP4sKaLC5JNAKw7JH8QdAnoAC88hW1IiSNZ+GgVXlN1UcMQ==} + '@vitest/mocker@4.1.7': + resolution: {integrity: sha512-vY7nuamKgfvpA1Koa3oYIw/k7D6kZnpGyNMZW8loow2bsBYla1TFdqTaXncWdRn4pgwNs+90RhnXhJScDwQeJA==} peerDependencies: msw: ^2.4.9 vite: ^6.0.0 || ^7.0.0 || ^8.0.0 @@ -2549,20 +2553,20 @@ packages: vite: optional: true - '@vitest/pretty-format@4.1.6': - resolution: {integrity: sha512-h5SxD/IzNhZYnrSZRsUZQIC+vD0GY8cUvq0iwsmkFKixRCKLLWqCXa/FIQ4S1R+sI+PGoojkHsdNrbZiM9Qpgw==} + '@vitest/pretty-format@4.1.7': + resolution: {integrity: sha512-umgCarTOYQWIaDMvGDRZij+6b9oVeLIyJzfN+AS88e0ZOU3QTgNNSTtjQOpcvWr3np1N0j4WgZj+sb3oYBDscw==} - '@vitest/runner@4.1.6': - resolution: {integrity: sha512-nOPCmn2+yD0ZNmKdsXGv/UxMMWbMuKeD6GyYncNwdkYDxpQvrPSKYj2rWuDjC2Y4b6w6hjip5dBKFzEUuZe3vA==} + '@vitest/runner@4.1.7': + resolution: {integrity: sha512-BapjmAQ2aI78WdMEfeUWivnfVzB+VPGwWRQcJE0OUq7qEeEcBsCSf+0T5iREBNE5nBb4wA5Ya0W6IA+sghdEFw==} - '@vitest/snapshot@4.1.6': - resolution: {integrity: sha512-YhsdE6xAVfTDmzjxL2ZDUvjj+ZsgyOKe+TdQzqkD72wIOmHka8NuGQ6NpTNZv9D2Z63fbwWKJPeVpEw4EQgYxw==} + '@vitest/snapshot@4.1.7': + resolution: {integrity: sha512-ZacLzja+TmJeZ1h14xW2FB/WpeimUD3haBXQPyJqxvo8jQTmfeA8zv58mtjN2C7EHXZDYVcVYdYmAxjkWVvKCw==} - '@vitest/spy@4.1.6': - resolution: {integrity: sha512-JFKxMx6udhwKh/Ldo270e17QX710vgunMkuPAvXjHSvC6oqLWAHhVhjg/I71q0u0CBSErIODV1Kjv0FQNSWjdg==} + '@vitest/spy@4.1.7': + resolution: {integrity: sha512-kbkI5LMWakyuTIvs6fUJ5qdIVb1XVKsYJAT4OJ938cHMROYMSfmoQdZy0aaAnjbbc8F61vkoTqz/Az+/HiIu5Q==} - '@vitest/utils@4.1.6': - resolution: {integrity: sha512-FxIY+U81R3LGKCxaHHFRQ5+g6/iRgGLmeHWdp2Amj4ljQRrEIWHmZyDfDYBRZlpyqA7qKxtS9DD1dhk8RnRIVQ==} + '@vitest/utils@4.1.7': + resolution: {integrity: sha512-T532WBu791cBxJlCl6SO+J14l81DQx6uQHm1bQbmCDY7nqlEIgkza/UFnSBNaUtSf41unldDFjdOBYEQC4b5Hw==} '@xyflow/react@12.10.2': resolution: {integrity: sha512-CgIi6HwlcHXwlkTpr0fxLv/0sRVNZ8IdwKLzzeCscaYBwpvfcH1QFOCeaTCuEn1FQEs/B8CjnTSjhs8udgmBgQ==} @@ -2591,6 +2595,10 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + agent-base@6.0.2: + resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} + engines: {node: '>= 6.0.0'} + agent-base@7.1.4: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} @@ -2607,8 +2615,8 @@ packages: resolution: {integrity: sha512-gOsf2YwSlleG6IjRYG2A7k0HmBMEo6qVNk9Bp/EaLgAJT5ngH6PXbqa4ItvnEwCm/velL5jAnQgsHsWnjhGmvw==} engines: {node: '>=18'} - ai@6.0.180: - resolution: {integrity: sha512-tOyRbwD0PEjMZKGvYQcTsv95K2zktwwNhQ49QOUAh0g8MNprO7ELIO1SgANMuPc0BFtkP6Ny6OAdjq3TtxLCbQ==} + ai@6.0.188: + resolution: {integrity: sha512-kNwIl1MM4ESzeOPDYuN+FidJ2QY5kGWHLtTMru6HHPW/JJ6nPuvHRhJ8tMX/Y2Tijx9DCiv2W7y5IBouuB712g==} engines: {node: '>=18'} peerDependencies: zod: ^3.25.76 || ^4.1.8 @@ -2708,8 +2716,8 @@ packages: resolution: {integrity: sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==} engines: {node: '>= 6.0.0'} - axios@1.15.2: - resolution: {integrity: sha512-wLrXxPtcrPTsNlJmKjkPnNPK2Ihe0hn0wGSaTEiHRPxwjvJwT3hKmXF4dpqxmPO9SoNb2FsYXj/xEo0gHN+D5A==} + axios@1.16.1: + resolution: {integrity: sha512-caYkukvroVPO8KrzuJEb50Hm07KwfBZPEC3VeFHTsqWHvKTsy54hjJz9BS/cdaypROE2rH6xvm9mHX4fgWkr3A==} bail@2.0.2: resolution: {integrity: sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==} @@ -2724,8 +2732,8 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - baseline-browser-mapping@2.10.29: - resolution: {integrity: sha512-Asa2krT+XTPZINCS+2QcyS8WTkObE77RwkydwF7h6DmnKqbvlalz93m/dnphUyCa6SWSP51VgtEUf2FN+gelFQ==} + baseline-browser-mapping@2.10.31: + resolution: {integrity: sha512-MujYO3eP72uvmSE0i4wltsodRfIpZATP3jvzRNRGGxgzId7aVocVJJV3nf01qnzzKFGxQVC9bpWxl5cjxTr/7Q==} engines: {node: '>=6.0.0'} hasBin: true @@ -2811,8 +2819,8 @@ packages: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} - caniuse-lite@1.0.30001792: - resolution: {integrity: sha512-hVLMUZFgR4JJ6ACt1uEESvQN1/dBVqPAKY0hgrV70eN3391K6juAfTjKZLKvOMsx8PxA7gsY1/tLMMTcfFLLpw==} + caniuse-lite@1.0.30001793: + resolution: {integrity: sha512-iwSsYWaCOoh26cV8NwNRViHlrfUvYsHDfRVcbtmw0Kg6PJIZZXwMkj1442FYLBGkeUf1juAsU3DTfxW579mrPA==} ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} @@ -3191,8 +3199,8 @@ packages: end-of-stream@1.4.5: resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} - enhanced-resolve@5.21.2: - resolution: {integrity: sha512-xe9vQb5kReirPUxgQrXA3ihgbCqssmTiM7cOZ+Gzu+VeGWgpV98lLZvp0dl4yriyAePcewxGUs9UpKD8PET9KQ==} + enhanced-resolve@5.21.6: + resolution: {integrity: sha512-aNnGCvbJ/RIyWo1IuhNdVjnNF+EjH9wpzpNHt+ci/m9He9LJvUN8wrCcXjp9cWsGNAuvSpVFTx/vraAFQ8qGjQ==} engines: {node: '>=10.13.0'} entities@6.0.1: @@ -3334,8 +3342,8 @@ packages: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} - express-rate-limit@8.4.1: - resolution: {integrity: sha512-NGVYwQSAyEQgzxX1iCM978PP9AdO/hW93gMcF6ZwQCm+rFvLsBH6w4xcXWTcliS8La5EPRN3p9wzItqBwJrfNw==} + express-rate-limit@8.5.2: + resolution: {integrity: sha512-5Kb34ipNX694DH48vN9irak1Qx30nb0PLYHXfJgw4YEjiC3ZEmZJhwOp+VfiCYwFzvFTdB9QkArYS5kXa2cx2A==} engines: {node: '>= 16'} peerDependencies: express: '>= 4.11' @@ -3353,6 +3361,9 @@ packages: fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + fast-sha256@1.3.0: + resolution: {integrity: sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==} + fast-string-truncated-width@3.0.3: resolution: {integrity: sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==} @@ -3362,14 +3373,18 @@ packages: fast-uri@3.1.2: resolution: {integrity: sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==} - fast-wrap-ansi@0.2.0: - resolution: {integrity: sha512-rLV8JHxTyhVmFYhBJuMujcrHqOT2cnO5Zxj37qROj23CP39GXubJRBUFF0z8KFK77Uc0SukZUf7JZhsVEQ6n8w==} + fast-wrap-ansi@0.2.2: + resolution: {integrity: sha512-7F2Fl+TjRSenLqlU3UjSH0iyqopqoZIu7eZVpEirP2g1GtWa2G/ecEmBdgz31+Mxr+ELclgg6sokpSFIQiZ02Q==} fast-xml-builder@1.1.7: resolution: {integrity: sha512-Yh7/7rQuMXICNr0oMYDR2yHP6oUvmQsTToFeOWj/kIDhAwQ+c4Ol/lbcwOmEM5OHYQmh6S6EQSQ1sljCKP36bQ==} - fast-xml-parser@5.7.2: - resolution: {integrity: sha512-P7oW7tLbYnhOLQk/Gv7cZgzgMPP/XN03K02/Jy6Y/NHzyIAIpxuZIM/YqAkfiXFPxA2CTm7NtCijK9EDu09u2w==} + fast-xml-parser@5.7.3: + resolution: {integrity: sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg==} + hasBin: true + + fast-xml-parser@5.8.0: + resolution: {integrity: sha512-6bIM7fsJxeo3uXv7OncQYsBAMPJ7V16Slahl/6M98C/i2q+vB1+4a0MtrvYwDFEUrwDSbAmeLDRXsOBwrL7yAg==} hasBin: true fastest-levenshtein@1.0.16: @@ -3395,8 +3410,8 @@ packages: resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} engines: {node: ^12.20 || >= 14.13} - fflate@0.8.2: - resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==} + fflate@0.8.3: + resolution: {integrity: sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==} figures@2.0.0: resolution: {integrity: sha512-Oa2M9atig69ZkfwiApY8F2Yy+tzMbazyvqv21R0NsSC8floSOC09BbT1ITWAdoMGQvJ/aZnR1KMwdx9tvHnTNA==} @@ -3458,8 +3473,8 @@ packages: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} - framer-motion@12.38.0: - resolution: {integrity: sha512-rFYkY/pigbcswl1XQSb7q424kSTQ8q6eAC+YUsSKooHQYuLdzdHjrt6uxUC+PRAO++q5IS7+TamgIw1AphxR+g==} + framer-motion@12.40.0: + resolution: {integrity: sha512-uaBd3qC1v3KQqBEjwTUd183K6PbS+j0yR9w9VmEOLWA/tnUcSn8Xa3uck7t4dgpDoUss8xQTcj8W2L07lrnLFg==} peerDependencies: '@emotion/is-prop-valid': '*' react: ^18.0.0 || ^19.0.0 @@ -3550,8 +3565,8 @@ packages: zod: optional: true - fumadocs-mdx@15.0.4: - resolution: {integrity: sha512-swJ7VB9x8fPjVAcH4xMg7qplgm7m6Hy3woB5s/yssBWVPkvET7wGOHoYZ05iW9TDIWruk5vyqwEDrwq3t4PZUw==} + fumadocs-mdx@15.0.7: + resolution: {integrity: sha512-3ffG5th20eL3CvEH9YSYfmn0MPeaJkdnu1MMXNsR1zduhn6tW5ieQ8YX+EC4+BfnpwcB3y2wwAlGM0PWl/YPNg==} hasBin: true peerDependencies: '@types/mdast': '*' @@ -3627,8 +3642,8 @@ packages: resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} engines: {node: 6.* || 8.* || >= 10.*} - get-east-asian-width@1.5.0: - resolution: {integrity: sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==} + get-east-asian-width@1.6.0: + resolution: {integrity: sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==} engines: {node: '>=18'} get-intrinsic@1.3.0: @@ -3770,6 +3785,9 @@ packages: html-escaper@2.0.2: resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} + html-to-image@1.11.11: + resolution: {integrity: sha512-9gux8QhvjRO/erSnDPv28noDZcPZmYE7e1vFsBLKLlRlKDSqNJYebj6Qz1TGd5lsRV+X+xYyjCKjuZdABinWjA==} + html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} @@ -3785,6 +3803,10 @@ packages: resolution: {integrity: sha512-FcF8VhXYLQcxWCnt/cCpT2apKsRDUGeVEeMqGu4HSTu29U8Yw0TLOjdYIlDsYk3IkUh+taX4IDWpPcCqKDhCjA==} engines: {node: '>= 20'} + https-proxy-agent@5.0.1: + resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} + engines: {node: '>= 6'} + https-proxy-agent@7.0.6: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} @@ -3858,8 +3880,8 @@ packages: '@types/react': optional: true - ink@7.0.2: - resolution: {integrity: sha512-cnkE2SsDC/gieJ+BD8+gWpXrZPMInv7agBYN5gcKVlQZYp+IKa/FKM5bp1OIuJFp3ZIuRK7ZNxY4MZR3tUzyfQ==} + ink@7.0.3: + resolution: {integrity: sha512-5kxHkIj9+RuqCU3zyvP4qvYWNOSHP2TW/SHayHGHOmk87KwfVcZwvJGemi9ch+ci2gXUqerK/Eh2DGEDt5q45g==} engines: {node: '>=22'} peerDependencies: '@types/react': '>=19.2.0' @@ -3997,8 +4019,8 @@ packages: resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} hasBin: true - jose@6.2.2: - resolution: {integrity: sha512-d7kPDd34KO/YnzaDOlikGpOurfF0ByC2sEV4cANCtdqLlTfBlw2p14O/5d/zv40gJPbIQxfES3nSx1/oYNyuZQ==} + jose@6.2.3: + resolution: {integrity: sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==} js-md4@0.3.2: resolution: {integrity: sha512-/GDnfQYsltsjRswQhN9fhv3EMw2sCpUdrdxyWDOUK7eyD++r3gRhzgiQgc/x4MAv2i1iuQ4lxO5mvqM3vj4bwA==} @@ -4051,8 +4073,8 @@ packages: jws@4.0.1: resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} - knip@6.12.2: - resolution: {integrity: sha512-RcZpT1sVziKZgDk1F0hAcp+bq71VJAF8vg1Y9ZLXc1+UXQaMm1rjiUqpJQTIj+lqwmiBQT19/u7ikgazs23cvA==} + knip@6.14.1: + resolution: {integrity: sha512-SN3Ly0ixzj5CQkY/rc4OPHpWrCC0XRIIjgdP76G9Cni5k72ur5jBYOyvJuF5oPTM14v8eHcMUgPbElHa+lnR0g==} engines: {node: ^20.19.0 || >=22.12.0} hasBin: true @@ -4202,24 +4224,24 @@ packages: lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} - lru-cache@11.3.6: - resolution: {integrity: sha512-Gf/KoL3C/MlI7Bt0PGI9I+TeTC/I6r/csU58N4BSNc4lppLBeKsOdFYkK+dX0ABDUMJNfCHTyPpzwwO21Awd3A==} + lru-cache@11.5.0: + resolution: {integrity: sha512-5YgH9UJd7wVb9hIouI2adWpgqrrICkt070Dnj8EUY1+B4B2P9eRLPAkAAo6NICA7CEhOIeBHl46u9zSNpNu7zA==} engines: {node: 20 || >=22} lru.min@1.1.4: resolution: {integrity: sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA==} engines: {bun: '>=1.0.0', deno: '>=1.30.0', node: '>=8.0.0'} - lucide-react@1.14.0: - resolution: {integrity: sha512-+1mdWcfSJVUsaTIjN9zoezmUhfXo5l0vP7ekBMPo3jcS/aIkxHnXqAPsByszMZx/Y8oQBRJxJx5xg+RH3urzxA==} + lucide-react@1.16.0: + resolution: {integrity: sha512-dYwyPzb4MEKpGUmNYk3WKWPnMrHs3FKM+q94kAnJrcDIqqn1hq2xY8scaS2ovsOCM5D51ey2gaRG3PBb1vgoYQ==} peerDependencies: react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} - magicast@0.5.2: - resolution: {integrity: sha512-E3ZJh4J3S9KfwdjZhe2afj6R9lGIN5Pher1pF39UGrXRqq/VDaGVIGN13BjHd2u8B61hArAGOnso7nBOouW3TQ==} + magicast@0.5.3: + resolution: {integrity: sha512-pVKE4UdSQ7DvHzivsCIFx2BJn1mHG6KsyrFcaxFx6tONdneEuThrDx0Cj3AMg58KyN4pzYT+LHOotxDQDjNvkw==} make-asynchronous@1.1.0: resolution: {integrity: sha512-ayF7iT+44LXdxJLTrTd3TLQpFDDvPCBxXxbv+pMUSuHA5Q8zyAfwkRP6aHHwNVFBUFWtxAHqwNJxF8vMZLAbVg==} @@ -4478,14 +4500,14 @@ packages: moment@2.30.1: resolution: {integrity: sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==} - motion-dom@12.38.0: - resolution: {integrity: sha512-pdkHLD8QYRp8VfiNLb8xIBJis1byQ9gPT3Jnh2jqfFtAsWUA3dEepDlsWe/xMpO8McV+VdpKVcp+E+TGJEtOoA==} + motion-dom@12.40.0: + resolution: {integrity: sha512-HxU3ZaBwNPVQUBQf1xxgq+7JrPNZvjLVxgbpEZL7RrWJnsxOf0/OM+yrHG9ogLQ31Do/r57Oz2gQWPK+6q62mg==} - motion-utils@12.36.0: - resolution: {integrity: sha512-eHWisygbiwVvf6PZ1vhaHCLamvkSbPIeAYxWUuL3a2PD/TROgE7FvfHWTIH4vMl798QLfMw15nRqIaRDXTlYRg==} + motion-utils@12.39.0: + resolution: {integrity: sha512-8nadJAJjTtqRkmRF36FoJTrywK9nnFmnPwnSMyxaOCU7GDjN9RTMJIxx9De8ErM+vpPhMccr/6fo5WciyQLnMQ==} - motion@12.38.0: - resolution: {integrity: sha512-uYfXzeHlgThchzwz5Te47dlv5JOUC7OB4rjJ/7XTUgtBZD8CchMN8qEJ4ZVsUmTyYA44zjV0fBwsiktRuFnn+w==} + motion@12.40.0: + resolution: {integrity: sha512-yjrHUrBFW6kQvjJwRsoiPSAhC5tRwRqNGJWmiJ4CrGnbKp0V88AdzkhBmDoqIsIPfarOe0Uddd37Xq43/gIocA==} peerDependencies: '@emotion/is-prop-valid': '*' react: ^18.0.0 || ^19.0.0 @@ -4501,8 +4523,8 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - mssql@12.5.2: - resolution: {integrity: sha512-rPXZdvYGCayb4+pRmqZ3oymDJB4ZrMpjnZfs3/EZYg8cXfYMWHZo8Kh5zVxny0GyRfPOCslcGuEEMUIyCWRRxg==} + mssql@12.5.4: + resolution: {integrity: sha512-f8UzhpO1STCYhxBybEgT4kaPa2Pda+nucQcMMad7RqOmmTZu3tjkvpPeI9h0RdSK//eua4ybRsLcalz/ttagwQ==} engines: {node: '>=18.19.0'} hasBin: true @@ -4519,8 +4541,8 @@ packages: resolution: {integrity: sha512-Tz09sEL2EEuv5fFowm419c1+a/jSMiBjI9gHxVLrVdbUkkNUUfjsVYs9pVZu5oCon/kmRh9TfLEObFtkVxmY0w==} engines: {node: '>=8.0.0'} - nanoid@3.3.11: - resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} + nanoid@3.3.12: + resolution: {integrity: sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true @@ -4567,8 +4589,8 @@ packages: sass: optional: true - node-abi@3.89.0: - resolution: {integrity: sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA==} + node-abi@3.92.0: + resolution: {integrity: sha512-KdHvFWZjEKDf0cakgFjebl371GPsISX2oZHcuyKqM7DtogIsHrqKeLTo8wBHxaXRAQlY2PsPlZmfo+9ZCxEREQ==} engines: {node: '>=10'} node-domexception@1.0.0: @@ -4592,8 +4614,8 @@ packages: resolution: {integrity: sha512-RWk+PI433eESQ7ounYxIp67CYuVsS1uYSonX3kA6ps/3LWfjVQa/ptEg6Y3T6uAMq1mWpX9PQ+qx+QaHpsc7gQ==} engines: {node: ^20.17.0 || >=22.9.0} - normalize-url@9.0.0: - resolution: {integrity: sha512-z9nC87iaZXXySbWWtTHfCFJyFvKaUAW6lODhikG7ILSbVgmwuFjUqkgnheHvAUcGedO29e2QGBRXMUD64aurqQ==} + normalize-url@9.0.1: + resolution: {integrity: sha512-ARftfC5HdUNu9jJeL8pHj8debUIHA2b91FizCoMzY4lG6dDX13jdvTK0TBe24IBDRf2HvJSzzwEPvmbkQWHRSg==} engines: {node: '>=20'} npm-run-path@4.0.1: @@ -4608,8 +4630,8 @@ packages: resolution: {integrity: sha512-9qny7Z9DsQU8Ou39ERsPU4OZQlSTP47ShQzuKZ6PRXpYLtIFgl/DEBYEXKlvcEa+9tHVcK8CF81Y2V72qaZhWA==} engines: {node: '>=18'} - npm@11.14.1: - resolution: {integrity: sha512-aopNZ0eEl6LbxoFcrXLmTEPzNBNxfiQnVgR9RmJBqzm+5h5pFoOmRljpRJbsXxocBeSl7GLcx3MoDf2UlEOjZw==} + npm@11.15.0: + resolution: {integrity: sha512-+k0tk7lRnpMUPnC7kTuU/yrV/mnFoPhJQ75VfLtZ6fwbzOVXaPsTE/Il9Pn1DHi482byMyqkHv/XsQ76mNjXLw==} engines: {node: ^20.17.0 || >=22.9.0} hasBin: true bundledDependencies: @@ -4725,8 +4747,8 @@ packages: resolution: {integrity: sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==} engines: {node: '>=8'} - openai@6.37.0: - resolution: {integrity: sha512-0H5dEGFmmLv6KSd0W1w2nyL8WsLkX6yoLeQpU+dZAOuGcany5qkYQMmj35ZrKgb6yiyYqpUzFOpR8mZQkgqeEQ==} + openai@6.38.0: + resolution: {integrity: sha512-AoMplt2UalrpgUDMh3L09QWjNRlgJPipclQvA6sYAaeF6nHNBMgmikAZGmcYLn8on4d9sQY9Q8bOLfrBS7Lc8g==} hasBin: true peerDependencies: ws: 8.20.1 @@ -4737,8 +4759,8 @@ packages: zod: optional: true - oxc-parser@0.128.0: - resolution: {integrity: sha512-XkOw3eiIxAgQ19WRew/Bq9wc5Ga/guaWIzDBzq80z1PyuDNGvWBpPby9k6YGwV8A8uMw+Nlq3xqlzuDYmUFYUw==} + oxc-parser@0.130.0: + resolution: {integrity: sha512-X0PJ+NmOok8qP3vK9uaW431ngkdM9UPEK7KG466urtIL2+EYTEgbZK2yqe2MWKJKBjRlFweP/pJPx0x9muMEVw==} engines: {node: ^20.19.0 || >=22.12.0} oxc-resolver@11.19.1: @@ -4870,30 +4892,30 @@ packages: engines: {node: '>=0.10'} hasBin: true - pg-cloudflare@1.3.0: - resolution: {integrity: sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==} + pg-cloudflare@1.4.0: + resolution: {integrity: sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==} - pg-connection-string@2.12.0: - resolution: {integrity: sha512-U7qg+bpswf3Cs5xLzRqbXbQl85ng0mfSV/J0nnA31MCLgvEaAo7CIhmeyrmJpOr7o+zm0rXK+hNnT5l9RHkCkQ==} + pg-connection-string@2.13.0: + resolution: {integrity: sha512-EMnU9E2fSULdsbErBbMaXJvFeD9B4+nPcM3f+4lsiCR0BHLPrLVjv3DbyM2hgQQviKJaTWIRRTjKjWlHg3p2ig==} pg-int8@1.0.1: resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} engines: {node: '>=4.0.0'} - pg-pool@3.13.0: - resolution: {integrity: sha512-gB+R+Xud1gLFuRD/QgOIgGOBE2KCQPaPwkzBBGC9oG69pHTkhQeIuejVIk3/cnDyX39av2AxomQiyPT13WKHQA==} + pg-pool@3.14.0: + resolution: {integrity: sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==} peerDependencies: pg: '>=8.0' - pg-protocol@1.13.0: - resolution: {integrity: sha512-zzdvXfS6v89r6v7OcFCHfHlyG/wvry1ALxZo4LqgUoy7W9xhBDMaqOuMiF3qEV45VqsN6rdlcehHrfDtlCPc8w==} + pg-protocol@1.14.0: + resolution: {integrity: sha512-n5taZ1kO3s9ngDTVxsEznOqCyToTgz0FLuPq0B33COy5pPpuWJpY3/2oRBVETuOgzdqRXfWpM9HIhp2LBBT1BA==} pg-types@2.2.0: resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} engines: {node: '>=4'} - pg@8.20.0: - resolution: {integrity: sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==} + pg@8.21.0: + resolution: {integrity: sha512-AUP1EYJuHraQGsVoCQVIcM7TEJVGtDzxWtGFZd8rds9d+CCXlU5Js1rYgfLNvxy9iJrpHjGrRjoi/3BT9fRyiA==} engines: {node: '>= 16.0.0'} peerDependencies: pg-native: '>=3.0.1' @@ -4947,9 +4969,14 @@ packages: resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} engines: {node: '>=0.10.0'} - posthog-node@5.0.0: - resolution: {integrity: sha512-gontigBt1pGHGXZme3+ojDdCYL66h/vvo+6KaQ6A51xqUOYgRvyzCLkS9Xv816jNBesRO8ouRjG428SDb2fFkg==} - engines: {node: '>=20'} + posthog-node@5.34.9: + resolution: {integrity: sha512-vOH+71q/Cb68ILXj58M2fV3GcE2sHimVFn2JQJYpe2wxAdZjWyt7sGhY1NI2/87DVjJ0zisEboVCX9/WXxXlMg==} + engines: {node: ^20.20.0 || >=22.22.0} + peerDependencies: + rxjs: ^7.0.0 + peerDependenciesMeta: + rxjs: + optional: true prebuild-install@7.1.3: resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} @@ -4985,8 +5012,8 @@ packages: pump@3.0.4: resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} - qs@6.15.1: - resolution: {integrity: sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg==} + qs@6.15.2: + resolution: {integrity: sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==} engines: {node: '>=0.6'} range-parser@1.2.1: @@ -5155,8 +5182,8 @@ packages: resolution: {integrity: sha512-JzFPAfklk1kjR1w76f0QOIhoDkNkSqW8wYKT08n9yysTmZfB+RQ2QoXoTAeOi1HD9ZipTyTAZg3c4pM/jeqgSw==} engines: {node: '>=18'} - rolldown@1.0.0-rc.17: - resolution: {integrity: sha512-ZrT53oAKrtA4+YtBWPQbtPOxIbVDbxT0orcYERKd63VJTF13zPcgXTvD4843L8pcsI7M6MErt8QtON6lrB9tyA==} + rolldown@1.0.2: + resolution: {integrity: sha512-oZx5zVDtVB44AW3eaifgDml1gWRDZGvjcfdxonE4swNPG98PrrXjaO/KrnUjzlMnztCCRVlUueA1kCXhARGk6g==} engines: {node: ^20.19.0 || >=22.12.0} hasBin: true @@ -5196,8 +5223,13 @@ packages: resolution: {integrity: sha512-hunMQrEy1T6Jr2uEVjrAIqjwWcQTgOAcIM52C8MY1EZSD3DDNft04XzvYKPqjED65bNVVko0YI38nYeEHCX3yw==} engines: {node: '>=12'} - semver@7.7.4: - resolution: {integrity: sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==} + semver@7.8.0: + resolution: {integrity: sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==} + engines: {node: '>=10'} + hasBin: true + + semver@7.8.1: + resolution: {integrity: sha512-rkVq3IXh+4FDGch+KwzX3aV9W3kO54GyEgpvBzSyctDA6Xtd7RJQV1xmXbeQp5v7+VzLOfVqiutSE6GICgPFvg==} engines: {node: '>=10'} hasBin: true @@ -5224,8 +5256,8 @@ packages: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} - shiki@4.0.2: - resolution: {integrity: sha512-eAVKTMedR5ckPo4xne/PjYQYrU3qx78gtJZ+sHlXEg5IHhhoQhMfZVzetTYuaJS0L2Ef3AcCRzCHV8T0WI6nIQ==} + shiki@4.1.0: + resolution: {integrity: sha512-l/ABZPUR5v70jI10EzqfMS/I96vjSGv2y0ihUV+WYFzv0EfvW4s54m0Lg8wCrrL+2IkwBzFTuxkZjPf8b2NX9Q==} engines: {node: '>=20'} side-channel-list@1.0.1: @@ -5285,8 +5317,8 @@ packages: resolution: {integrity: sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==} engines: {node: '>= 18'} - snowflake-sdk@2.4.1: - resolution: {integrity: sha512-JIdqz9ed2FzkU8oEstf06hTJRoX9+PRRG9LJT1vfGTXN3A52kGxhGoWzmK0GtFTUnxTMxMoMYgD5QdoQbckyag==} + snowflake-sdk@2.4.2: + resolution: {integrity: sha512-sN9683tRetlGC1rFGLUSkwpUVaVwRbATho7DcHUwft76rg2EHq8ooiMs6iHHtXLSG6IGmly51oajAyc7/nOOhg==} engines: {node: '>=18'} peerDependencies: asn1.js: ^5.4.1 @@ -5345,6 +5377,9 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + standardwebhooks@1.0.0: + resolution: {integrity: sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==} + statuses@2.0.2: resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} engines: {node: '>= 0.8'} @@ -5386,8 +5421,8 @@ packages: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} - strip-ansi@7.1.2: - resolution: {integrity: sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==} + strip-ansi@7.2.0: + resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} engines: {node: '>=12'} strip-bom@3.0.0: @@ -5414,8 +5449,8 @@ packages: resolution: {integrity: sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==} engines: {node: '>=14.16'} - strnum@2.2.3: - resolution: {integrity: sha512-oKx6RUCuHfT3oyVjtnrmn19H1SiCqgJSg+54XqURKp5aCMbrXrhLjRN9TjuwMjiYstZ0MzDrHqkGZ5dFTKd+zg==} + strnum@2.3.0: + resolution: {integrity: sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q==} stubs@3.0.0: resolution: {integrity: sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==} @@ -5520,10 +5555,6 @@ packages: tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} - tinyexec@1.1.1: - resolution: {integrity: sha512-VKS/ZaQhhkKFMANmAOhhXVoIfBXblQxGX1myCQ2faQrfmobMftXeJPcZGp0gS07ocvGJWDLZGyOZDadDBqYIJg==} - engines: {node: '>=18'} - tinyexec@1.1.2: resolution: {integrity: sha512-dAqSqE/RabpBKI8+h26GfLq6Vb3JVXs30XYQjdMjaj/c2tS8IYYMbIzP599KtRj7c57/wYApb3QjgRgXmrCukA==} engines: {node: '>=18'} @@ -5590,9 +5621,9 @@ packages: resolution: {integrity: sha512-8ZiHFm91orbSAe2PSAiSVBVko18pbhbiB3U9GglSzF/zCGkR+rxpHx6sEMCUm4kxY4LjDIUGgCfUMtwfZfjfUA==} engines: {node: '>=20'} - type-is@2.0.1: - resolution: {integrity: sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==} - engines: {node: '>= 0.6'} + type-is@2.1.0: + resolution: {integrity: sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==} + engines: {node: '>= 18'} typescript@6.0.3: resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} @@ -5722,13 +5753,13 @@ packages: vfile@6.0.3: resolution: {integrity: sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==} - vite@8.0.10: - resolution: {integrity: sha512-rZuUu9j6J5uotLDs+cAA4O5H4K1SfPliUlQwqa6YEwSrWDZzP4rhm00oJR5snMewjxF5V/K3D4kctsUTsIU9Mw==} + vite@8.0.14: + resolution: {integrity: sha512-s4BJJ+5y1pYL6Otw51FHhVJQhPnuRinKig64g/1+EUNaJsd3gCKdD31IPFvswUgW9/60QT9oFHbZHbQK5imcxw==} engines: {node: ^20.19.0 || >=22.12.0} hasBin: true peerDependencies: '@types/node': ^24.3.0 - '@vitejs/devtools': ^0.1.0 + '@vitejs/devtools': ^0.1.18 esbuild: ^0.27.0 || ^0.28.0 jiti: '>=1.21.0' less: ^4.0.0 @@ -5765,20 +5796,20 @@ packages: yaml: optional: true - vitest@4.1.6: - resolution: {integrity: sha512-6lvjbS3p9b4CrdCmguzbh2/4uoXhGE2q71R4OX5sqF9R1bo9Xd6fGrMAfvp5wnCzlBnFVdCOp6onuTQVbo8iUQ==} + vitest@4.1.7: + resolution: {integrity: sha512-flYyaFd2CgoCoU+0UKt3pxksgC+S02iTDN0n3LtqaMeXsI9SBcdNujc2k0DeFLzUn/0k538yNjOSdwgCqcrwJA==} engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} hasBin: true peerDependencies: '@edge-runtime/vm': '*' '@opentelemetry/api': ^1.9.0 '@types/node': ^24.3.0 - '@vitest/browser-playwright': 4.1.6 - '@vitest/browser-preview': 4.1.6 - '@vitest/browser-webdriverio': 4.1.6 - '@vitest/coverage-istanbul': 4.1.6 - '@vitest/coverage-v8': 4.1.6 - '@vitest/ui': 4.1.6 + '@vitest/browser-playwright': 4.1.7 + '@vitest/browser-preview': 4.1.7 + '@vitest/browser-webdriverio': 4.1.7 + '@vitest/coverage-istanbul': 4.1.7 + '@vitest/coverage-v8': 4.1.7 + '@vitest/ui': 4.1.7 happy-dom: '*' jsdom: '*' vite: ^6.0.0 || ^7.0.0 || ^8.0.0 @@ -5876,6 +5907,10 @@ packages: resolution: {integrity: sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==} engines: {node: '>=18'} + xml-naming@0.1.0: + resolution: {integrity: sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==} + engines: {node: '>=16.0.0'} + xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} @@ -5960,29 +5995,29 @@ snapshots: '@actions/io@3.0.2': {} - '@ai-sdk/anthropic@3.0.77(zod@4.4.3)': + '@ai-sdk/anthropic@3.0.78(zod@4.4.3)': dependencies: '@ai-sdk/provider': 3.0.10 '@ai-sdk/provider-utils': 4.0.27(zod@4.4.3) zod: 4.4.3 - '@ai-sdk/devtools@0.0.17': + '@ai-sdk/devtools@0.0.18': dependencies: '@ai-sdk/provider': 3.0.10 '@hono/node-server': 1.19.14(hono@4.12.18) hono: 4.12.18 - '@ai-sdk/gateway@3.0.114(zod@4.4.3)': + '@ai-sdk/gateway@3.0.118(zod@4.4.3)': dependencies: '@ai-sdk/provider': 3.0.10 '@ai-sdk/provider-utils': 4.0.27(zod@4.4.3) '@vercel/oidc': 3.2.0 zod: 4.4.3 - '@ai-sdk/google-vertex@4.0.128(zod@4.4.3)': + '@ai-sdk/google-vertex@4.0.134(zod@4.4.3)': dependencies: - '@ai-sdk/anthropic': 3.0.77(zod@4.4.3) - '@ai-sdk/google': 3.0.73(zod@4.4.3) + '@ai-sdk/anthropic': 3.0.78(zod@4.4.3) + '@ai-sdk/google': 3.0.78(zod@4.4.3) '@ai-sdk/openai-compatible': 2.0.47(zod@4.4.3) '@ai-sdk/provider': 3.0.10 '@ai-sdk/provider-utils': 4.0.27(zod@4.4.3) @@ -5991,7 +6026,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@ai-sdk/google@3.0.73(zod@4.4.3)': + '@ai-sdk/google@3.0.78(zod@4.4.3)': dependencies: '@ai-sdk/provider': 3.0.10 '@ai-sdk/provider-utils': 4.0.27(zod@4.4.3) @@ -6021,51 +6056,49 @@ snapshots: '@alloc/quick-lru@5.2.0': {} - '@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.142': + '@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.146': optional: true - '@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.142': + '@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.146': optional: true - '@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.142': + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.146': optional: true - '@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.142': + '@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.146': optional: true - '@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.142': + '@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.146': optional: true - '@anthropic-ai/claude-agent-sdk-linux-x64@0.3.142': + '@anthropic-ai/claude-agent-sdk-linux-x64@0.3.146': optional: true - '@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.142': + '@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.146': optional: true - '@anthropic-ai/claude-agent-sdk-win32-x64@0.3.142': + '@anthropic-ai/claude-agent-sdk-win32-x64@0.3.146': optional: true - '@anthropic-ai/claude-agent-sdk@0.3.142(zod@4.4.3)': + '@anthropic-ai/claude-agent-sdk@0.3.146(@anthropic-ai/sdk@0.97.1(zod@4.4.3))(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(zod@4.4.3)': dependencies: - '@anthropic-ai/sdk': 0.93.0(zod@4.4.3) + '@anthropic-ai/sdk': 0.97.1(zod@4.4.3) '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) zod: 4.4.3 optionalDependencies: - '@anthropic-ai/claude-agent-sdk-darwin-arm64': 0.3.142 - '@anthropic-ai/claude-agent-sdk-darwin-x64': 0.3.142 - '@anthropic-ai/claude-agent-sdk-linux-arm64': 0.3.142 - '@anthropic-ai/claude-agent-sdk-linux-arm64-musl': 0.3.142 - '@anthropic-ai/claude-agent-sdk-linux-x64': 0.3.142 - '@anthropic-ai/claude-agent-sdk-linux-x64-musl': 0.3.142 - '@anthropic-ai/claude-agent-sdk-win32-arm64': 0.3.142 - '@anthropic-ai/claude-agent-sdk-win32-x64': 0.3.142 - transitivePeerDependencies: - - '@cfworker/json-schema' - - supports-color + '@anthropic-ai/claude-agent-sdk-darwin-arm64': 0.3.146 + '@anthropic-ai/claude-agent-sdk-darwin-x64': 0.3.146 + '@anthropic-ai/claude-agent-sdk-linux-arm64': 0.3.146 + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl': 0.3.146 + '@anthropic-ai/claude-agent-sdk-linux-x64': 0.3.146 + '@anthropic-ai/claude-agent-sdk-linux-x64-musl': 0.3.146 + '@anthropic-ai/claude-agent-sdk-win32-arm64': 0.3.146 + '@anthropic-ai/claude-agent-sdk-win32-x64': 0.3.146 - '@anthropic-ai/sdk@0.93.0(zod@4.4.3)': + '@anthropic-ai/sdk@0.97.1(zod@4.4.3)': dependencies: json-schema-to-ts: 3.1.1 + standardwebhooks: 1.0.0 optionalDependencies: zod: 4.4.3 @@ -6121,452 +6154,346 @@ snapshots: '@aws-crypto/sha1-browser': 5.2.0 '@aws-crypto/sha256-browser': 5.2.0 '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/core': 3.974.8 - '@aws-sdk/credential-provider-node': 3.972.39 - '@aws-sdk/middleware-bucket-endpoint': 3.972.10 - '@aws-sdk/middleware-expect-continue': 3.972.10 - '@aws-sdk/middleware-flexible-checksums': 3.974.16 - '@aws-sdk/middleware-host-header': 3.972.10 + '@aws-sdk/core': 3.974.12 + '@aws-sdk/credential-provider-node': 3.972.43 + '@aws-sdk/middleware-bucket-endpoint': 3.972.14 + '@aws-sdk/middleware-expect-continue': 3.972.12 + '@aws-sdk/middleware-flexible-checksums': 3.974.20 + '@aws-sdk/middleware-host-header': 3.972.13 '@aws-sdk/middleware-location-constraint': 3.972.10 - '@aws-sdk/middleware-logger': 3.972.10 - '@aws-sdk/middleware-recursion-detection': 3.972.11 - '@aws-sdk/middleware-sdk-s3': 3.972.37 + '@aws-sdk/middleware-logger': 3.972.12 + '@aws-sdk/middleware-recursion-detection': 3.972.14 + '@aws-sdk/middleware-sdk-s3': 3.972.41 '@aws-sdk/middleware-ssec': 3.972.10 - '@aws-sdk/middleware-user-agent': 3.972.38 - '@aws-sdk/region-config-resolver': 3.972.13 - '@aws-sdk/signature-v4-multi-region': 3.996.25 + '@aws-sdk/middleware-user-agent': 3.972.42 + '@aws-sdk/region-config-resolver': 3.972.16 + '@aws-sdk/signature-v4-multi-region': 3.996.27 '@aws-sdk/types': 3.973.8 - '@aws-sdk/util-endpoints': 3.996.8 - '@aws-sdk/util-user-agent-browser': 3.972.10 - '@aws-sdk/util-user-agent-node': 3.973.24 - '@smithy/config-resolver': 4.4.17 - '@smithy/core': 3.23.17 - '@smithy/eventstream-serde-browser': 4.2.14 - '@smithy/eventstream-serde-config-resolver': 4.3.14 - '@smithy/eventstream-serde-node': 4.2.14 - '@smithy/fetch-http-handler': 5.3.17 - '@smithy/hash-blob-browser': 4.2.15 - '@smithy/hash-node': 4.2.14 - '@smithy/hash-stream-node': 4.2.14 - '@smithy/invalid-dependency': 4.2.14 - '@smithy/md5-js': 4.2.14 - '@smithy/middleware-content-length': 4.2.14 - '@smithy/middleware-endpoint': 4.4.32 - '@smithy/middleware-retry': 4.5.7 - '@smithy/middleware-serde': 4.2.20 - '@smithy/middleware-stack': 4.2.14 - '@smithy/node-config-provider': 4.3.14 - '@smithy/node-http-handler': 4.6.1 - '@smithy/protocol-http': 5.3.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - '@smithy/url-parser': 4.2.14 - '@smithy/util-base64': 4.3.2 - '@smithy/util-body-length-browser': 4.2.2 - '@smithy/util-body-length-node': 4.2.3 - '@smithy/util-defaults-mode-browser': 4.3.49 - '@smithy/util-defaults-mode-node': 4.2.54 - '@smithy/util-endpoints': 3.4.2 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-retry': 4.3.6 - '@smithy/util-stream': 4.5.25 - '@smithy/util-utf8': 4.2.2 - '@smithy/util-waiter': 4.3.0 + '@aws-sdk/util-endpoints': 3.996.11 + '@aws-sdk/util-user-agent-browser': 3.972.13 + '@aws-sdk/util-user-agent-node': 3.973.28 + '@smithy/config-resolver': 4.5.3 + '@smithy/core': 3.24.3 + '@smithy/eventstream-serde-browser': 4.3.3 + '@smithy/eventstream-serde-config-resolver': 4.4.3 + '@smithy/eventstream-serde-node': 4.3.3 + '@smithy/fetch-http-handler': 5.4.3 + '@smithy/hash-blob-browser': 4.3.3 + '@smithy/hash-node': 4.3.3 + '@smithy/hash-stream-node': 4.3.3 + '@smithy/invalid-dependency': 4.3.3 + '@smithy/md5-js': 4.3.3 + '@smithy/middleware-content-length': 4.3.3 + '@smithy/middleware-endpoint': 4.5.3 + '@smithy/middleware-retry': 4.6.3 + '@smithy/middleware-serde': 4.3.3 + '@smithy/middleware-stack': 4.3.3 + '@smithy/node-config-provider': 4.4.3 + '@smithy/node-http-handler': 4.7.3 + '@smithy/protocol-http': 5.4.3 + '@smithy/smithy-client': 4.13.3 + '@smithy/types': 4.14.2 + '@smithy/url-parser': 4.3.3 + '@smithy/util-base64': 4.4.3 + '@smithy/util-body-length-browser': 4.3.3 + '@smithy/util-body-length-node': 4.3.3 + '@smithy/util-defaults-mode-browser': 4.4.3 + '@smithy/util-defaults-mode-node': 4.3.3 + '@smithy/util-endpoints': 3.5.3 + '@smithy/util-middleware': 4.3.3 + '@smithy/util-retry': 4.4.3 + '@smithy/util-stream': 4.6.3 + '@smithy/util-utf8': 4.3.3 + '@smithy/util-waiter': 4.4.3 tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt '@aws-sdk/client-sts@3.1045.0': dependencies: '@aws-crypto/sha256-browser': 5.2.0 '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/core': 3.974.8 - '@aws-sdk/credential-provider-node': 3.972.39 - '@aws-sdk/middleware-host-header': 3.972.10 - '@aws-sdk/middleware-logger': 3.972.10 - '@aws-sdk/middleware-recursion-detection': 3.972.11 - '@aws-sdk/middleware-user-agent': 3.972.38 - '@aws-sdk/region-config-resolver': 3.972.13 - '@aws-sdk/signature-v4-multi-region': 3.996.25 + '@aws-sdk/core': 3.974.12 + '@aws-sdk/credential-provider-node': 3.972.43 + '@aws-sdk/middleware-host-header': 3.972.13 + '@aws-sdk/middleware-logger': 3.972.12 + '@aws-sdk/middleware-recursion-detection': 3.972.14 + '@aws-sdk/middleware-user-agent': 3.972.42 + '@aws-sdk/region-config-resolver': 3.972.16 + '@aws-sdk/signature-v4-multi-region': 3.996.27 '@aws-sdk/types': 3.973.8 - '@aws-sdk/util-endpoints': 3.996.8 - '@aws-sdk/util-user-agent-browser': 3.972.10 - '@aws-sdk/util-user-agent-node': 3.973.24 - '@smithy/config-resolver': 4.4.17 - '@smithy/core': 3.23.17 - '@smithy/fetch-http-handler': 5.3.17 - '@smithy/hash-node': 4.2.14 - '@smithy/invalid-dependency': 4.2.14 - '@smithy/middleware-content-length': 4.2.14 - '@smithy/middleware-endpoint': 4.4.32 - '@smithy/middleware-retry': 4.5.7 - '@smithy/middleware-serde': 4.2.20 - '@smithy/middleware-stack': 4.2.14 - '@smithy/node-config-provider': 4.3.14 - '@smithy/node-http-handler': 4.6.1 - '@smithy/protocol-http': 5.3.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - '@smithy/url-parser': 4.2.14 - '@smithy/util-base64': 4.3.2 - '@smithy/util-body-length-browser': 4.2.2 - '@smithy/util-body-length-node': 4.2.3 - '@smithy/util-defaults-mode-browser': 4.3.49 - '@smithy/util-defaults-mode-node': 4.2.54 - '@smithy/util-endpoints': 3.4.2 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-retry': 4.3.6 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - - '@aws-sdk/core@3.974.8': - dependencies: - '@aws-sdk/types': 3.973.8 - '@aws-sdk/xml-builder': 3.972.22 - '@smithy/core': 3.23.17 - '@smithy/node-config-provider': 4.3.14 - '@smithy/property-provider': 4.2.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/signature-v4': 5.3.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - '@smithy/util-base64': 4.3.2 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-retry': 4.3.6 - '@smithy/util-utf8': 4.2.2 + '@aws-sdk/util-endpoints': 3.996.11 + '@aws-sdk/util-user-agent-browser': 3.972.13 + '@aws-sdk/util-user-agent-node': 3.973.28 + '@smithy/config-resolver': 4.5.3 + '@smithy/core': 3.24.3 + '@smithy/fetch-http-handler': 5.4.3 + '@smithy/hash-node': 4.3.3 + '@smithy/invalid-dependency': 4.3.3 + '@smithy/middleware-content-length': 4.3.3 + '@smithy/middleware-endpoint': 4.5.3 + '@smithy/middleware-retry': 4.6.3 + '@smithy/middleware-serde': 4.3.3 + '@smithy/middleware-stack': 4.3.3 + '@smithy/node-config-provider': 4.4.3 + '@smithy/node-http-handler': 4.7.3 + '@smithy/protocol-http': 5.4.3 + '@smithy/smithy-client': 4.13.3 + '@smithy/types': 4.14.2 + '@smithy/url-parser': 4.3.3 + '@smithy/util-base64': 4.4.3 + '@smithy/util-body-length-browser': 4.3.3 + '@smithy/util-body-length-node': 4.3.3 + '@smithy/util-defaults-mode-browser': 4.4.3 + '@smithy/util-defaults-mode-node': 4.3.3 + '@smithy/util-endpoints': 3.5.3 + '@smithy/util-middleware': 4.3.3 + '@smithy/util-retry': 4.4.3 + '@smithy/util-utf8': 4.3.3 tslib: 2.8.1 - '@aws-sdk/crc64-nvme@3.972.7': + '@aws-sdk/core@3.974.12': dependencies: - '@smithy/types': 4.14.1 + '@aws-sdk/types': 3.973.8 + '@aws-sdk/xml-builder': 3.972.24 + '@aws/lambda-invoke-store': 0.2.4 + '@smithy/core': 3.24.3 + '@smithy/signature-v4': 5.4.3 + '@smithy/types': 4.14.2 + bowser: 2.14.1 tslib: 2.8.1 - '@aws-sdk/credential-provider-env@3.972.34': + '@aws-sdk/crc64-nvme@3.972.8': dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/types': 4.14.1 + '@smithy/types': 4.14.2 tslib: 2.8.1 - '@aws-sdk/credential-provider-http@3.972.36': + '@aws-sdk/credential-provider-env@3.972.38': dependencies: - '@aws-sdk/core': 3.974.8 + '@aws-sdk/core': 3.974.12 '@aws-sdk/types': 3.973.8 - '@smithy/fetch-http-handler': 5.3.17 - '@smithy/node-http-handler': 4.6.1 - '@smithy/property-provider': 4.2.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - '@smithy/util-stream': 4.5.25 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 tslib: 2.8.1 - '@aws-sdk/credential-provider-ini@3.972.38': + '@aws-sdk/credential-provider-http@3.972.40': dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/credential-provider-env': 3.972.34 - '@aws-sdk/credential-provider-http': 3.972.36 - '@aws-sdk/credential-provider-login': 3.972.38 - '@aws-sdk/credential-provider-process': 3.972.34 - '@aws-sdk/credential-provider-sso': 3.972.38 - '@aws-sdk/credential-provider-web-identity': 3.972.38 - '@aws-sdk/nested-clients': 3.997.6 + '@aws-sdk/core': 3.974.12 '@aws-sdk/types': 3.973.8 - '@smithy/credential-provider-imds': 4.2.14 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - - '@aws-sdk/credential-provider-login@3.972.38': - dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/nested-clients': 3.997.6 - '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - - '@aws-sdk/credential-provider-node@3.972.39': - dependencies: - '@aws-sdk/credential-provider-env': 3.972.34 - '@aws-sdk/credential-provider-http': 3.972.36 - '@aws-sdk/credential-provider-ini': 3.972.38 - '@aws-sdk/credential-provider-process': 3.972.34 - '@aws-sdk/credential-provider-sso': 3.972.38 - '@aws-sdk/credential-provider-web-identity': 3.972.38 - '@aws-sdk/types': 3.973.8 - '@smithy/credential-provider-imds': 4.2.14 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - - '@aws-sdk/credential-provider-process@3.972.34': - dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 + '@smithy/core': 3.24.3 + '@smithy/fetch-http-handler': 5.4.3 + '@smithy/node-http-handler': 4.7.3 + '@smithy/types': 4.14.2 tslib: 2.8.1 - '@aws-sdk/credential-provider-sso@3.972.38': + '@aws-sdk/credential-provider-ini@3.972.42': dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/nested-clients': 3.997.6 - '@aws-sdk/token-providers': 3.1041.0 + '@aws-sdk/core': 3.974.12 + '@aws-sdk/credential-provider-env': 3.972.38 + '@aws-sdk/credential-provider-http': 3.972.40 + '@aws-sdk/credential-provider-login': 3.972.42 + '@aws-sdk/credential-provider-process': 3.972.38 + '@aws-sdk/credential-provider-sso': 3.972.42 + '@aws-sdk/credential-provider-web-identity': 3.972.42 + '@aws-sdk/nested-clients': 3.997.10 '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 + '@smithy/core': 3.24.3 + '@smithy/credential-provider-imds': 4.3.3 + '@smithy/types': 4.14.2 tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - '@aws-sdk/credential-provider-web-identity@3.972.38': + '@aws-sdk/credential-provider-login@3.972.42': dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/nested-clients': 3.997.6 + '@aws-sdk/core': 3.974.12 + '@aws-sdk/nested-clients': 3.997.10 '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-node@3.972.43': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.38 + '@aws-sdk/credential-provider-http': 3.972.40 + '@aws-sdk/credential-provider-ini': 3.972.42 + '@aws-sdk/credential-provider-process': 3.972.38 + '@aws-sdk/credential-provider-sso': 3.972.42 + '@aws-sdk/credential-provider-web-identity': 3.972.42 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/credential-provider-imds': 4.3.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-process@3.972.38': + dependencies: + '@aws-sdk/core': 3.974.12 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-sso@3.972.42': + dependencies: + '@aws-sdk/core': 3.974.12 + '@aws-sdk/nested-clients': 3.997.10 + '@aws-sdk/token-providers': 3.1049.0 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-web-identity@3.972.42': + dependencies: + '@aws-sdk/core': 3.974.12 + '@aws-sdk/nested-clients': 3.997.10 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt '@aws-sdk/ec2-metadata-service@3.1045.0': dependencies: '@aws-sdk/types': 3.973.8 - '@smithy/node-config-provider': 4.3.14 - '@smithy/node-http-handler': 4.6.1 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - '@smithy/util-stream': 4.5.25 + '@smithy/node-config-provider': 4.4.3 + '@smithy/node-http-handler': 4.7.3 + '@smithy/protocol-http': 5.4.3 + '@smithy/types': 4.14.2 + '@smithy/util-stream': 4.6.3 tslib: 2.8.1 - '@aws-sdk/middleware-bucket-endpoint@3.972.10': + '@aws-sdk/middleware-bucket-endpoint@3.972.14': + dependencies: + '@aws-sdk/core': 3.974.12 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/middleware-expect-continue@3.972.12': dependencies: '@aws-sdk/types': 3.973.8 - '@aws-sdk/util-arn-parser': 3.972.3 - '@smithy/node-config-provider': 4.3.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - '@smithy/util-config-provider': 4.2.2 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 tslib: 2.8.1 - '@aws-sdk/middleware-expect-continue@3.972.10': - dependencies: - '@aws-sdk/types': 3.973.8 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@aws-sdk/middleware-flexible-checksums@3.974.16': + '@aws-sdk/middleware-flexible-checksums@3.974.20': dependencies: '@aws-crypto/crc32': 5.2.0 '@aws-crypto/crc32c': 5.2.0 '@aws-crypto/util': 5.2.0 - '@aws-sdk/core': 3.974.8 - '@aws-sdk/crc64-nvme': 3.972.7 + '@aws-sdk/core': 3.974.12 + '@aws-sdk/crc64-nvme': 3.972.8 '@aws-sdk/types': 3.973.8 - '@smithy/is-array-buffer': 4.2.2 - '@smithy/node-config-provider': 4.3.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-stream': 4.5.25 - '@smithy/util-utf8': 4.2.2 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 tslib: 2.8.1 - '@aws-sdk/middleware-host-header@3.972.10': + '@aws-sdk/middleware-host-header@3.972.13': dependencies: - '@aws-sdk/types': 3.973.8 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 + '@aws-sdk/core': 3.974.12 tslib: 2.8.1 '@aws-sdk/middleware-location-constraint@3.972.10': dependencies: '@aws-sdk/types': 3.973.8 - '@smithy/types': 4.14.1 + '@smithy/types': 4.14.2 tslib: 2.8.1 - '@aws-sdk/middleware-logger@3.972.10': + '@aws-sdk/middleware-logger@3.972.12': dependencies: - '@aws-sdk/types': 3.973.8 - '@smithy/types': 4.14.1 + '@aws-sdk/core': 3.974.12 tslib: 2.8.1 - '@aws-sdk/middleware-recursion-detection@3.972.11': + '@aws-sdk/middleware-recursion-detection@3.972.14': dependencies: - '@aws-sdk/types': 3.973.8 - '@aws/lambda-invoke-store': 0.2.4 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 + '@aws-sdk/core': 3.974.12 tslib: 2.8.1 - '@aws-sdk/middleware-sdk-s3@3.972.37': + '@aws-sdk/middleware-sdk-s3@3.972.41': dependencies: - '@aws-sdk/core': 3.974.8 + '@aws-sdk/core': 3.974.12 + '@aws-sdk/signature-v4-multi-region': 3.996.27 '@aws-sdk/types': 3.973.8 - '@aws-sdk/util-arn-parser': 3.972.3 - '@smithy/core': 3.23.17 - '@smithy/node-config-provider': 4.3.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/signature-v4': 5.3.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - '@smithy/util-config-provider': 4.2.2 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-stream': 4.5.25 - '@smithy/util-utf8': 4.2.2 + '@smithy/core': 3.24.3 + '@smithy/signature-v4': 5.4.3 + '@smithy/types': 4.14.2 tslib: 2.8.1 '@aws-sdk/middleware-ssec@3.972.10': dependencies: '@aws-sdk/types': 3.973.8 - '@smithy/types': 4.14.1 + '@smithy/types': 4.14.2 tslib: 2.8.1 - '@aws-sdk/middleware-user-agent@3.972.38': + '@aws-sdk/middleware-user-agent@3.972.42': dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/types': 3.973.8 - '@aws-sdk/util-endpoints': 3.996.8 - '@smithy/core': 3.23.17 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - '@smithy/util-retry': 4.3.6 + '@aws-sdk/core': 3.974.12 tslib: 2.8.1 - '@aws-sdk/nested-clients@3.997.6': + '@aws-sdk/nested-clients@3.997.10': dependencies: '@aws-crypto/sha256-browser': 5.2.0 '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/core': 3.974.8 - '@aws-sdk/middleware-host-header': 3.972.10 - '@aws-sdk/middleware-logger': 3.972.10 - '@aws-sdk/middleware-recursion-detection': 3.972.11 - '@aws-sdk/middleware-user-agent': 3.972.38 - '@aws-sdk/region-config-resolver': 3.972.13 - '@aws-sdk/signature-v4-multi-region': 3.996.25 + '@aws-sdk/core': 3.974.12 + '@aws-sdk/signature-v4-multi-region': 3.996.27 '@aws-sdk/types': 3.973.8 - '@aws-sdk/util-endpoints': 3.996.8 - '@aws-sdk/util-user-agent-browser': 3.972.10 - '@aws-sdk/util-user-agent-node': 3.973.24 - '@smithy/config-resolver': 4.4.17 - '@smithy/core': 3.23.17 - '@smithy/fetch-http-handler': 5.3.17 - '@smithy/hash-node': 4.2.14 - '@smithy/invalid-dependency': 4.2.14 - '@smithy/middleware-content-length': 4.2.14 - '@smithy/middleware-endpoint': 4.4.32 - '@smithy/middleware-retry': 4.5.7 - '@smithy/middleware-serde': 4.2.20 - '@smithy/middleware-stack': 4.2.14 - '@smithy/node-config-provider': 4.3.14 - '@smithy/node-http-handler': 4.6.1 - '@smithy/protocol-http': 5.3.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - '@smithy/url-parser': 4.2.14 - '@smithy/util-base64': 4.3.2 - '@smithy/util-body-length-browser': 4.2.2 - '@smithy/util-body-length-node': 4.2.3 - '@smithy/util-defaults-mode-browser': 4.3.49 - '@smithy/util-defaults-mode-node': 4.2.54 - '@smithy/util-endpoints': 3.4.2 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-retry': 4.3.6 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - - '@aws-sdk/region-config-resolver@3.972.13': - dependencies: - '@aws-sdk/types': 3.973.8 - '@smithy/config-resolver': 4.4.17 - '@smithy/node-config-provider': 4.3.14 - '@smithy/types': 4.14.1 + '@smithy/core': 3.24.3 + '@smithy/fetch-http-handler': 5.4.3 + '@smithy/node-http-handler': 4.7.3 + '@smithy/types': 4.14.2 tslib: 2.8.1 - '@aws-sdk/signature-v4-multi-region@3.996.25': + '@aws-sdk/region-config-resolver@3.972.16': dependencies: - '@aws-sdk/middleware-sdk-s3': 3.972.37 - '@aws-sdk/types': 3.973.8 - '@smithy/protocol-http': 5.3.14 - '@smithy/signature-v4': 5.3.14 - '@smithy/types': 4.14.1 + '@aws-sdk/core': 3.974.12 tslib: 2.8.1 - '@aws-sdk/token-providers@3.1041.0': + '@aws-sdk/signature-v4-multi-region@3.996.27': dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/nested-clients': 3.997.6 '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 + '@smithy/core': 3.24.3 + '@smithy/signature-v4': 5.4.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/token-providers@3.1049.0': + dependencies: + '@aws-sdk/core': 3.974.12 + '@aws-sdk/nested-clients': 3.997.10 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt '@aws-sdk/types@3.973.8': dependencies: - '@smithy/types': 4.14.1 + '@smithy/types': 4.14.2 tslib: 2.8.1 - '@aws-sdk/util-arn-parser@3.972.3': + '@aws-sdk/util-endpoints@3.996.11': dependencies: - tslib: 2.8.1 - - '@aws-sdk/util-endpoints@3.996.8': - dependencies: - '@aws-sdk/types': 3.973.8 - '@smithy/types': 4.14.1 - '@smithy/url-parser': 4.2.14 - '@smithy/util-endpoints': 3.4.2 + '@aws-sdk/core': 3.974.12 + '@smithy/core': 3.24.3 tslib: 2.8.1 '@aws-sdk/util-locate-window@3.965.5': dependencies: tslib: 2.8.1 - '@aws-sdk/util-user-agent-browser@3.972.10': + '@aws-sdk/util-user-agent-browser@3.972.13': dependencies: - '@aws-sdk/types': 3.973.8 - '@smithy/types': 4.14.1 - bowser: 2.14.1 + '@aws-sdk/core': 3.974.12 tslib: 2.8.1 - '@aws-sdk/util-user-agent-node@3.973.24': + '@aws-sdk/util-user-agent-node@3.973.28': dependencies: - '@aws-sdk/middleware-user-agent': 3.972.38 - '@aws-sdk/types': 3.973.8 - '@smithy/node-config-provider': 4.3.14 - '@smithy/types': 4.14.1 - '@smithy/util-config-provider': 4.2.2 + '@aws-sdk/core': 3.974.12 tslib: 2.8.1 - '@aws-sdk/xml-builder@3.972.22': + '@aws-sdk/xml-builder@3.972.24': dependencies: '@nodable/entities': 2.1.0 - '@smithy/types': 4.14.1 - fast-xml-parser: 5.7.2 + '@smithy/types': 4.14.2 + fast-xml-parser: 5.7.3 tslib: 2.8.1 '@aws/lambda-invoke-store@0.2.4': {} @@ -6651,7 +6578,7 @@ snapshots: '@azure/core-xml@1.5.1': dependencies: - fast-xml-parser: 5.7.2 + fast-xml-parser: 5.8.0 tslib: 2.8.1 '@azure/identity@4.13.1': @@ -6663,8 +6590,8 @@ snapshots: '@azure/core-tracing': 1.3.1 '@azure/core-util': 1.13.1 '@azure/logger': 1.3.0 - '@azure/msal-browser': 5.9.0 - '@azure/msal-node': 5.1.5 + '@azure/msal-browser': 5.11.0 + '@azure/msal-node': 5.2.2 open: 10.2.0 tslib: 2.8.1 transitivePeerDependencies: @@ -6708,15 +6635,15 @@ snapshots: transitivePeerDependencies: - supports-color - '@azure/msal-browser@5.9.0': + '@azure/msal-browser@5.11.0': dependencies: - '@azure/msal-common': 16.5.2 + '@azure/msal-common': 16.6.2 - '@azure/msal-common@16.5.2': {} + '@azure/msal-common@16.6.2': {} - '@azure/msal-node@5.1.5': + '@azure/msal-node@5.2.2': dependencies: - '@azure/msal-common': 16.5.2 + '@azure/msal-common': 16.6.2 jsonwebtoken: 9.0.3 '@azure/storage-blob@12.26.0': @@ -6797,21 +6724,21 @@ snapshots: '@clack/core@1.3.1': dependencies: - fast-wrap-ansi: 0.2.0 + fast-wrap-ansi: 0.2.2 sisteransi: 1.0.5 '@clack/prompts@1.4.0': dependencies: '@clack/core': 1.3.1 fast-string-width: 3.0.2 - fast-wrap-ansi: 0.2.0 + fast-wrap-ansi: 0.2.2 sisteransi: 1.0.5 - '@clickhouse/client-common@1.18.4': {} + '@clickhouse/client-common@1.18.5': {} - '@clickhouse/client@1.18.4': + '@clickhouse/client@1.18.5': dependencies: - '@clickhouse/client-common': 1.18.4 + '@clickhouse/client-common': 1.18.5 '@colors/colors@1.5.0': optional: true @@ -7135,7 +7062,7 @@ snapshots: '@mdx-js/mdx@3.1.1': dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 '@types/mdx': 2.0.13 @@ -7174,9 +7101,9 @@ snapshots: eventsource: 3.0.7 eventsource-parser: 3.0.8 express: 5.2.1 - express-rate-limit: 8.4.1(express@5.2.1) + express-rate-limit: 8.5.2(express@5.2.1) hono: 4.12.18 - jose: 6.2.2 + jose: 6.2.3 json-schema-typed: 8.0.2 pkce-challenge: 5.0.1 raw-body: 3.0.2 @@ -7189,7 +7116,7 @@ snapshots: dependencies: '@emnapi/core': 1.10.0 '@emnapi/runtime': 1.10.0 - '@tybys/wasm-util': 0.10.1 + '@tybys/wasm-util': 0.10.2 optional: true '@next/env@16.2.6': {} @@ -7220,7 +7147,7 @@ snapshots: '@nodable/entities@2.1.0': {} - '@notionhq/client@5.21.0': {} + '@notionhq/client@5.22.0': {} '@octokit/auth-token@6.0.0': {} @@ -7283,77 +7210,108 @@ snapshots: dependencies: '@octokit/openapi-types': 27.0.0 - '@opentelemetry/api@1.9.0': {} + '@openai/codex-sdk@0.133.0': + dependencies: + '@openai/codex': 0.133.0 + + '@openai/codex@0.133.0': + optionalDependencies: + '@openai/codex-darwin-arm64': '@openai/codex@0.133.0-darwin-arm64' + '@openai/codex-darwin-x64': '@openai/codex@0.133.0-darwin-x64' + '@openai/codex-linux-arm64': '@openai/codex@0.133.0-linux-arm64' + '@openai/codex-linux-x64': '@openai/codex@0.133.0-linux-x64' + '@openai/codex-win32-arm64': '@openai/codex@0.133.0-win32-arm64' + '@openai/codex-win32-x64': '@openai/codex@0.133.0-win32-x64' + + '@openai/codex@0.133.0-darwin-arm64': + optional: true + + '@openai/codex@0.133.0-darwin-x64': + optional: true + + '@openai/codex@0.133.0-linux-arm64': + optional: true + + '@openai/codex@0.133.0-linux-x64': + optional: true + + '@openai/codex@0.133.0-win32-arm64': + optional: true + + '@openai/codex@0.133.0-win32-x64': + optional: true + + '@opentelemetry/api@1.9.1': {} '@orama/orama@3.1.18': {} - '@oxc-parser/binding-android-arm-eabi@0.128.0': + '@oxc-parser/binding-android-arm-eabi@0.130.0': optional: true - '@oxc-parser/binding-android-arm64@0.128.0': + '@oxc-parser/binding-android-arm64@0.130.0': optional: true - '@oxc-parser/binding-darwin-arm64@0.128.0': + '@oxc-parser/binding-darwin-arm64@0.130.0': optional: true - '@oxc-parser/binding-darwin-x64@0.128.0': + '@oxc-parser/binding-darwin-x64@0.130.0': optional: true - '@oxc-parser/binding-freebsd-x64@0.128.0': + '@oxc-parser/binding-freebsd-x64@0.130.0': optional: true - '@oxc-parser/binding-linux-arm-gnueabihf@0.128.0': + '@oxc-parser/binding-linux-arm-gnueabihf@0.130.0': optional: true - '@oxc-parser/binding-linux-arm-musleabihf@0.128.0': + '@oxc-parser/binding-linux-arm-musleabihf@0.130.0': optional: true - '@oxc-parser/binding-linux-arm64-gnu@0.128.0': + '@oxc-parser/binding-linux-arm64-gnu@0.130.0': optional: true - '@oxc-parser/binding-linux-arm64-musl@0.128.0': + '@oxc-parser/binding-linux-arm64-musl@0.130.0': optional: true - '@oxc-parser/binding-linux-ppc64-gnu@0.128.0': + '@oxc-parser/binding-linux-ppc64-gnu@0.130.0': optional: true - '@oxc-parser/binding-linux-riscv64-gnu@0.128.0': + '@oxc-parser/binding-linux-riscv64-gnu@0.130.0': optional: true - '@oxc-parser/binding-linux-riscv64-musl@0.128.0': + '@oxc-parser/binding-linux-riscv64-musl@0.130.0': optional: true - '@oxc-parser/binding-linux-s390x-gnu@0.128.0': + '@oxc-parser/binding-linux-s390x-gnu@0.130.0': optional: true - '@oxc-parser/binding-linux-x64-gnu@0.128.0': + '@oxc-parser/binding-linux-x64-gnu@0.130.0': optional: true - '@oxc-parser/binding-linux-x64-musl@0.128.0': + '@oxc-parser/binding-linux-x64-musl@0.130.0': optional: true - '@oxc-parser/binding-openharmony-arm64@0.128.0': + '@oxc-parser/binding-openharmony-arm64@0.130.0': optional: true - '@oxc-parser/binding-wasm32-wasi@0.128.0': + '@oxc-parser/binding-wasm32-wasi@0.130.0': dependencies: '@emnapi/core': 1.10.0 '@emnapi/runtime': 1.10.0 '@napi-rs/wasm-runtime': 1.1.4(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) optional: true - '@oxc-parser/binding-win32-arm64-msvc@0.128.0': + '@oxc-parser/binding-win32-arm64-msvc@0.130.0': optional: true - '@oxc-parser/binding-win32-ia32-msvc@0.128.0': + '@oxc-parser/binding-win32-ia32-msvc@0.130.0': optional: true - '@oxc-parser/binding-win32-x64-msvc@0.128.0': + '@oxc-parser/binding-win32-x64-msvc@0.130.0': optional: true - '@oxc-project/types@0.127.0': {} + '@oxc-project/types@0.130.0': {} - '@oxc-project/types@0.128.0': {} + '@oxc-project/types@0.132.0': {} '@oxc-resolver/binding-android-arm-eabi@11.19.1': optional: true @@ -7432,412 +7390,418 @@ snapshots: '@pnpm/network.ca-file': 1.0.2 config-chain: 1.1.13 + '@posthog/core@1.29.7': + dependencies: + '@posthog/types': 1.374.4 + + '@posthog/types@1.374.4': {} + '@radix-ui/number@1.1.1': {} '@radix-ui/primitive@1.1.3': {} - '@radix-ui/react-accordion@1.2.12(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@radix-ui/react-accordion@1.2.12(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: '@radix-ui/primitive': 1.1.3 - '@radix-ui/react-collapsible': 1.1.12(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-collection': 1.1.7(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-context': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-direction': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-id': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-collapsible': 1.1.12(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-collection': 1.1.7(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-context': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-direction': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-id': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 - '@types/react-dom': 19.2.3(@types/react@19.2.14) + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) - '@radix-ui/react-arrow@1.1.7(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@radix-ui/react-arrow@1.1.7(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: - '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 - '@types/react-dom': 19.2.3(@types/react@19.2.14) + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) - '@radix-ui/react-collapsible@1.1.12(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@radix-ui/react-collapsible@1.1.12(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: '@radix-ui/primitive': 1.1.3 - '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-context': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-id': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-presence': 1.1.5(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-context': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-id': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-presence': 1.1.5(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 - '@types/react-dom': 19.2.3(@types/react@19.2.14) + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) - '@radix-ui/react-collection@1.1.7(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@radix-ui/react-collection@1.1.7(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: - '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-context': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-slot': 1.2.3(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-context': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-slot': 1.2.3(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 - '@types/react-dom': 19.2.3(@types/react@19.2.14) + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) - '@radix-ui/react-compose-refs@1.1.2(@types/react@19.2.14)(react@19.2.6)': + '@radix-ui/react-compose-refs@1.1.2(@types/react@19.2.15)(react@19.2.6)': dependencies: react: 19.2.6 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - '@radix-ui/react-context@1.1.2(@types/react@19.2.14)(react@19.2.6)': + '@radix-ui/react-context@1.1.2(@types/react@19.2.15)(react@19.2.6)': dependencies: react: 19.2.6 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - '@radix-ui/react-dialog@1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@radix-ui/react-dialog@1.1.15(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: '@radix-ui/primitive': 1.1.3 - '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-context': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-dismissable-layer': 1.1.11(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-focus-guards': 1.1.3(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-focus-scope': 1.1.7(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-id': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-portal': 1.1.9(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-presence': 1.1.5(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-slot': 1.2.3(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-context': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-dismissable-layer': 1.1.11(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-focus-guards': 1.1.3(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-focus-scope': 1.1.7(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-id': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-portal': 1.1.9(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-presence': 1.1.5(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-slot': 1.2.3(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.15)(react@19.2.6) aria-hidden: 1.2.6 react: 19.2.6 react-dom: 19.2.6(react@19.2.6) - react-remove-scroll: 2.7.2(@types/react@19.2.14)(react@19.2.6) + react-remove-scroll: 2.7.2(@types/react@19.2.15)(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 - '@types/react-dom': 19.2.3(@types/react@19.2.14) + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) - '@radix-ui/react-direction@1.1.1(@types/react@19.2.14)(react@19.2.6)': + '@radix-ui/react-direction@1.1.1(@types/react@19.2.15)(react@19.2.6)': dependencies: react: 19.2.6 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - '@radix-ui/react-dismissable-layer@1.1.11(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@radix-ui/react-dismissable-layer@1.1.11(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: '@radix-ui/primitive': 1.1.3 - '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-use-escape-keydown': 1.1.1(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-use-escape-keydown': 1.1.1(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 - '@types/react-dom': 19.2.3(@types/react@19.2.14) + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) - '@radix-ui/react-focus-guards@1.1.3(@types/react@19.2.14)(react@19.2.6)': + '@radix-ui/react-focus-guards@1.1.3(@types/react@19.2.15)(react@19.2.6)': dependencies: react: 19.2.6 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - '@radix-ui/react-focus-scope@1.1.7(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@radix-ui/react-focus-scope@1.1.7(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: - '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 - '@types/react-dom': 19.2.3(@types/react@19.2.14) + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) - '@radix-ui/react-id@1.1.1(@types/react@19.2.14)(react@19.2.6)': + '@radix-ui/react-id@1.1.1(@types/react@19.2.15)(react@19.2.6)': dependencies: - '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - '@radix-ui/react-navigation-menu@1.2.14(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@radix-ui/react-navigation-menu@1.2.14(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: '@radix-ui/primitive': 1.1.3 - '@radix-ui/react-collection': 1.1.7(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-context': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-direction': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-dismissable-layer': 1.1.11(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-id': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-presence': 1.1.5(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-use-previous': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-visually-hidden': 1.2.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-collection': 1.1.7(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-context': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-direction': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-dismissable-layer': 1.1.11(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-id': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-presence': 1.1.5(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-use-previous': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-visually-hidden': 1.2.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 - '@types/react-dom': 19.2.3(@types/react@19.2.14) + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) - '@radix-ui/react-popover@1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@radix-ui/react-popover@1.1.15(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: '@radix-ui/primitive': 1.1.3 - '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-context': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-dismissable-layer': 1.1.11(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-focus-guards': 1.1.3(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-focus-scope': 1.1.7(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-id': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-popper': 1.2.8(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-portal': 1.1.9(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-presence': 1.1.5(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-slot': 1.2.3(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-context': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-dismissable-layer': 1.1.11(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-focus-guards': 1.1.3(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-focus-scope': 1.1.7(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-id': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-popper': 1.2.8(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-portal': 1.1.9(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-presence': 1.1.5(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-slot': 1.2.3(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.15)(react@19.2.6) aria-hidden: 1.2.6 react: 19.2.6 react-dom: 19.2.6(react@19.2.6) - react-remove-scroll: 2.7.2(@types/react@19.2.14)(react@19.2.6) + react-remove-scroll: 2.7.2(@types/react@19.2.15)(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 - '@types/react-dom': 19.2.3(@types/react@19.2.14) + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) - '@radix-ui/react-popper@1.2.8(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@radix-ui/react-popper@1.2.8(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: '@floating-ui/react-dom': 2.1.8(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-arrow': 1.1.7(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-context': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-use-rect': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-use-size': 1.1.1(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-arrow': 1.1.7(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-context': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-use-rect': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-use-size': 1.1.1(@types/react@19.2.15)(react@19.2.6) '@radix-ui/rect': 1.1.1 react: 19.2.6 react-dom: 19.2.6(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 - '@types/react-dom': 19.2.3(@types/react@19.2.14) + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) - '@radix-ui/react-portal@1.1.9(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@radix-ui/react-portal@1.1.9(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: - '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 - '@types/react-dom': 19.2.3(@types/react@19.2.14) + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) - '@radix-ui/react-presence@1.1.5(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@radix-ui/react-presence@1.1.5(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: - '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 - '@types/react-dom': 19.2.3(@types/react@19.2.14) + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) - '@radix-ui/react-primitive@2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@radix-ui/react-primitive@2.1.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: - '@radix-ui/react-slot': 1.2.3(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-slot': 1.2.3(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 - '@types/react-dom': 19.2.3(@types/react@19.2.14) + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) - '@radix-ui/react-roving-focus@1.1.11(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@radix-ui/react-roving-focus@1.1.11(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: '@radix-ui/primitive': 1.1.3 - '@radix-ui/react-collection': 1.1.7(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-context': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-direction': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-id': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-collection': 1.1.7(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-context': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-direction': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-id': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 - '@types/react-dom': 19.2.3(@types/react@19.2.14) + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) - '@radix-ui/react-scroll-area@1.2.10(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@radix-ui/react-scroll-area@1.2.10(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: '@radix-ui/number': 1.1.1 '@radix-ui/primitive': 1.1.3 - '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-context': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-direction': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-presence': 1.1.5(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-context': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-direction': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-presence': 1.1.5(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 - '@types/react-dom': 19.2.3(@types/react@19.2.14) + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) - '@radix-ui/react-slot@1.2.3(@types/react@19.2.14)(react@19.2.6)': + '@radix-ui/react-slot@1.2.3(@types/react@19.2.15)(react@19.2.6)': dependencies: - '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - '@radix-ui/react-slot@1.2.4(@types/react@19.2.14)(react@19.2.6)': + '@radix-ui/react-slot@1.2.4(@types/react@19.2.15)(react@19.2.6)': dependencies: - '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - '@radix-ui/react-tabs@1.1.13(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@radix-ui/react-tabs@1.1.13(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: '@radix-ui/primitive': 1.1.3 - '@radix-ui/react-context': 1.1.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-direction': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-id': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-presence': 1.1.5(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-roving-focus': 1.1.11(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-context': 1.1.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-direction': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-id': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-presence': 1.1.5(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-roving-focus': 1.1.11(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 - '@types/react-dom': 19.2.3(@types/react@19.2.14) + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) - '@radix-ui/react-use-callback-ref@1.1.1(@types/react@19.2.14)(react@19.2.6)': + '@radix-ui/react-use-callback-ref@1.1.1(@types/react@19.2.15)(react@19.2.6)': dependencies: react: 19.2.6 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - '@radix-ui/react-use-controllable-state@1.2.2(@types/react@19.2.14)(react@19.2.6)': + '@radix-ui/react-use-controllable-state@1.2.2(@types/react@19.2.15)(react@19.2.6)': dependencies: - '@radix-ui/react-use-effect-event': 0.0.2(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-use-effect-event': 0.0.2(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - '@radix-ui/react-use-effect-event@0.0.2(@types/react@19.2.14)(react@19.2.6)': + '@radix-ui/react-use-effect-event@0.0.2(@types/react@19.2.15)(react@19.2.6)': dependencies: - '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - '@radix-ui/react-use-escape-keydown@1.1.1(@types/react@19.2.14)(react@19.2.6)': + '@radix-ui/react-use-escape-keydown@1.1.1(@types/react@19.2.15)(react@19.2.6)': dependencies: - '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - '@radix-ui/react-use-layout-effect@1.1.1(@types/react@19.2.14)(react@19.2.6)': + '@radix-ui/react-use-layout-effect@1.1.1(@types/react@19.2.15)(react@19.2.6)': dependencies: react: 19.2.6 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - '@radix-ui/react-use-previous@1.1.1(@types/react@19.2.14)(react@19.2.6)': + '@radix-ui/react-use-previous@1.1.1(@types/react@19.2.15)(react@19.2.6)': dependencies: react: 19.2.6 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - '@radix-ui/react-use-rect@1.1.1(@types/react@19.2.14)(react@19.2.6)': + '@radix-ui/react-use-rect@1.1.1(@types/react@19.2.15)(react@19.2.6)': dependencies: '@radix-ui/rect': 1.1.1 react: 19.2.6 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - '@radix-ui/react-use-size@1.1.1(@types/react@19.2.14)(react@19.2.6)': + '@radix-ui/react-use-size@1.1.1(@types/react@19.2.15)(react@19.2.6)': dependencies: - '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.14)(react@19.2.6) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.15)(react@19.2.6) react: 19.2.6 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - '@radix-ui/react-visually-hidden@1.2.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@radix-ui/react-visually-hidden@1.2.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: - '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 - '@types/react-dom': 19.2.3(@types/react@19.2.14) + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) '@radix-ui/rect@1.1.1': {} - '@rolldown/binding-android-arm64@1.0.0-rc.17': + '@rolldown/binding-android-arm64@1.0.2': optional: true - '@rolldown/binding-darwin-arm64@1.0.0-rc.17': + '@rolldown/binding-darwin-arm64@1.0.2': optional: true - '@rolldown/binding-darwin-x64@1.0.0-rc.17': + '@rolldown/binding-darwin-x64@1.0.2': optional: true - '@rolldown/binding-freebsd-x64@1.0.0-rc.17': + '@rolldown/binding-freebsd-x64@1.0.2': optional: true - '@rolldown/binding-linux-arm-gnueabihf@1.0.0-rc.17': + '@rolldown/binding-linux-arm-gnueabihf@1.0.2': optional: true - '@rolldown/binding-linux-arm64-gnu@1.0.0-rc.17': + '@rolldown/binding-linux-arm64-gnu@1.0.2': optional: true - '@rolldown/binding-linux-arm64-musl@1.0.0-rc.17': + '@rolldown/binding-linux-arm64-musl@1.0.2': optional: true - '@rolldown/binding-linux-ppc64-gnu@1.0.0-rc.17': + '@rolldown/binding-linux-ppc64-gnu@1.0.2': optional: true - '@rolldown/binding-linux-s390x-gnu@1.0.0-rc.17': + '@rolldown/binding-linux-s390x-gnu@1.0.2': optional: true - '@rolldown/binding-linux-x64-gnu@1.0.0-rc.17': + '@rolldown/binding-linux-x64-gnu@1.0.2': optional: true - '@rolldown/binding-linux-x64-musl@1.0.0-rc.17': + '@rolldown/binding-linux-x64-musl@1.0.2': optional: true - '@rolldown/binding-openharmony-arm64@1.0.0-rc.17': + '@rolldown/binding-openharmony-arm64@1.0.2': optional: true - '@rolldown/binding-wasm32-wasi@1.0.0-rc.17': + '@rolldown/binding-wasm32-wasi@1.0.2': dependencies: '@emnapi/core': 1.10.0 '@emnapi/runtime': 1.10.0 '@napi-rs/wasm-runtime': 1.1.4(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) optional: true - '@rolldown/binding-win32-arm64-msvc@1.0.0-rc.17': + '@rolldown/binding-win32-arm64-msvc@1.0.2': optional: true - '@rolldown/binding-win32-x64-msvc@1.0.0-rc.17': + '@rolldown/binding-win32-x64-msvc@1.0.2': optional: true - '@rolldown/pluginutils@1.0.0-rc.17': {} + '@rolldown/pluginutils@1.0.1': {} '@sec-ant/readable-stream@0.4.1': {} @@ -7918,13 +7882,13 @@ snapshots: fs-extra: 11.3.5 lodash-es: 4.18.1 nerf-dart: 1.0.0 - normalize-url: 9.0.0 - npm: 11.14.1 + normalize-url: 9.0.1 + npm: 11.15.0 rc: 1.2.8 read-pkg: 10.1.0 registry-auth-token: 5.1.1 semantic-release: 25.0.3(typescript@6.0.3) - semver: 7.7.4 + semver: 7.8.0 tempy: 3.2.0 '@semantic-release/release-notes-generator@14.1.1(semantic-release@25.0.3(typescript@6.0.3))': @@ -7941,40 +7905,40 @@ snapshots: transitivePeerDependencies: - supports-color - '@shikijs/core@4.0.2': + '@shikijs/core@4.1.0': dependencies: - '@shikijs/primitive': 4.0.2 - '@shikijs/types': 4.0.2 + '@shikijs/primitive': 4.1.0 + '@shikijs/types': 4.1.0 '@shikijs/vscode-textmate': 10.0.2 '@types/hast': 3.0.4 hast-util-to-html: 9.0.5 - '@shikijs/engine-javascript@4.0.2': + '@shikijs/engine-javascript@4.1.0': dependencies: - '@shikijs/types': 4.0.2 + '@shikijs/types': 4.1.0 '@shikijs/vscode-textmate': 10.0.2 oniguruma-to-es: 4.3.6 - '@shikijs/engine-oniguruma@4.0.2': + '@shikijs/engine-oniguruma@4.1.0': dependencies: - '@shikijs/types': 4.0.2 + '@shikijs/types': 4.1.0 '@shikijs/vscode-textmate': 10.0.2 - '@shikijs/langs@4.0.2': + '@shikijs/langs@4.1.0': dependencies: - '@shikijs/types': 4.0.2 + '@shikijs/types': 4.1.0 - '@shikijs/primitive@4.0.2': + '@shikijs/primitive@4.1.0': dependencies: - '@shikijs/types': 4.0.2 + '@shikijs/types': 4.1.0 '@shikijs/vscode-textmate': 10.0.2 '@types/hast': 3.0.4 - '@shikijs/themes@4.0.2': + '@shikijs/themes@4.1.0': dependencies: - '@shikijs/types': 4.0.2 + '@shikijs/types': 4.1.0 - '@shikijs/types@4.0.2': + '@shikijs/types@4.1.0': dependencies: '@shikijs/vscode-textmate': 10.0.2 '@types/hast': 3.0.4 @@ -7993,251 +7957,148 @@ snapshots: '@sindresorhus/merge-streams@4.0.0': {} - '@smithy/chunked-blob-reader-native@4.2.3': + '@smithy/config-resolver@4.5.3': dependencies: - '@smithy/util-base64': 4.3.2 + '@smithy/core': 3.24.3 tslib: 2.8.1 - '@smithy/chunked-blob-reader@5.2.2': - dependencies: - tslib: 2.8.1 - - '@smithy/config-resolver@4.4.17': - dependencies: - '@smithy/node-config-provider': 4.3.14 - '@smithy/types': 4.14.1 - '@smithy/util-config-provider': 4.2.2 - '@smithy/util-endpoints': 3.4.2 - '@smithy/util-middleware': 4.2.14 - tslib: 2.8.1 - - '@smithy/core@3.23.17': - dependencies: - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - '@smithy/url-parser': 4.2.14 - '@smithy/util-base64': 4.3.2 - '@smithy/util-body-length-browser': 4.2.2 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-stream': 4.5.25 - '@smithy/util-utf8': 4.2.2 - '@smithy/uuid': 1.1.2 - tslib: 2.8.1 - - '@smithy/credential-provider-imds@4.2.14': - dependencies: - '@smithy/node-config-provider': 4.3.14 - '@smithy/property-provider': 4.2.14 - '@smithy/types': 4.14.1 - '@smithy/url-parser': 4.2.14 - tslib: 2.8.1 - - '@smithy/eventstream-codec@4.2.14': + '@smithy/core@3.24.3': dependencies: '@aws-crypto/crc32': 5.2.0 - '@smithy/types': 4.14.1 - '@smithy/util-hex-encoding': 4.2.2 + '@smithy/types': 4.14.2 tslib: 2.8.1 - '@smithy/eventstream-serde-browser@4.2.14': + '@smithy/credential-provider-imds@4.3.3': dependencies: - '@smithy/eventstream-serde-universal': 4.2.14 - '@smithy/types': 4.14.1 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 tslib: 2.8.1 - '@smithy/eventstream-serde-config-resolver@4.3.14': + '@smithy/eventstream-serde-browser@4.3.3': dependencies: - '@smithy/types': 4.14.1 + '@smithy/core': 3.24.3 tslib: 2.8.1 - '@smithy/eventstream-serde-node@4.2.14': + '@smithy/eventstream-serde-config-resolver@4.4.3': dependencies: - '@smithy/eventstream-serde-universal': 4.2.14 - '@smithy/types': 4.14.1 + '@smithy/core': 3.24.3 tslib: 2.8.1 - '@smithy/eventstream-serde-universal@4.2.14': + '@smithy/eventstream-serde-node@4.3.3': dependencies: - '@smithy/eventstream-codec': 4.2.14 - '@smithy/types': 4.14.1 + '@smithy/core': 3.24.3 tslib: 2.8.1 - '@smithy/fetch-http-handler@5.3.17': + '@smithy/fetch-http-handler@5.4.3': dependencies: - '@smithy/protocol-http': 5.3.14 - '@smithy/querystring-builder': 4.2.14 - '@smithy/types': 4.14.1 - '@smithy/util-base64': 4.3.2 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 tslib: 2.8.1 - '@smithy/hash-blob-browser@4.2.15': + '@smithy/hash-blob-browser@4.3.3': dependencies: - '@smithy/chunked-blob-reader': 5.2.2 - '@smithy/chunked-blob-reader-native': 4.2.3 - '@smithy/types': 4.14.1 + '@smithy/core': 3.24.3 tslib: 2.8.1 - '@smithy/hash-node@4.2.14': + '@smithy/hash-node@4.3.3': dependencies: - '@smithy/types': 4.14.1 - '@smithy/util-buffer-from': 4.2.2 - '@smithy/util-utf8': 4.2.2 + '@smithy/core': 3.24.3 tslib: 2.8.1 - '@smithy/hash-stream-node@4.2.14': + '@smithy/hash-stream-node@4.3.3': dependencies: - '@smithy/types': 4.14.1 - '@smithy/util-utf8': 4.2.2 + '@smithy/core': 3.24.3 tslib: 2.8.1 - '@smithy/invalid-dependency@4.2.14': + '@smithy/invalid-dependency@4.3.3': dependencies: - '@smithy/types': 4.14.1 + '@smithy/core': 3.24.3 tslib: 2.8.1 '@smithy/is-array-buffer@2.2.0': dependencies: tslib: 2.8.1 - '@smithy/is-array-buffer@4.2.2': + '@smithy/md5-js@4.3.3': + dependencies: + '@smithy/core': 3.24.3 + tslib: 2.8.1 + + '@smithy/middleware-content-length@4.3.3': + dependencies: + '@smithy/core': 3.24.3 + tslib: 2.8.1 + + '@smithy/middleware-endpoint@4.5.3': + dependencies: + '@smithy/core': 3.24.3 + tslib: 2.8.1 + + '@smithy/middleware-retry@4.6.3': + dependencies: + '@smithy/core': 3.24.3 + tslib: 2.8.1 + + '@smithy/middleware-serde@4.3.3': + dependencies: + '@smithy/core': 3.24.3 + tslib: 2.8.1 + + '@smithy/middleware-stack@4.3.3': + dependencies: + '@smithy/core': 3.24.3 + tslib: 2.8.1 + + '@smithy/node-config-provider@4.4.3': + dependencies: + '@smithy/core': 3.24.3 + tslib: 2.8.1 + + '@smithy/node-http-handler@4.7.3': + dependencies: + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@smithy/protocol-http@5.4.3': + dependencies: + '@smithy/core': 3.24.3 + tslib: 2.8.1 + + '@smithy/signature-v4@5.4.3': + dependencies: + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@smithy/smithy-client@4.13.3': + dependencies: + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@smithy/types@4.14.2': dependencies: tslib: 2.8.1 - '@smithy/md5-js@4.2.14': + '@smithy/url-parser@4.3.3': dependencies: - '@smithy/types': 4.14.1 - '@smithy/util-utf8': 4.2.2 + '@smithy/core': 3.24.3 tslib: 2.8.1 - '@smithy/middleware-content-length@4.2.14': + '@smithy/util-base64@4.4.3': dependencies: - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 + '@smithy/core': 3.24.3 tslib: 2.8.1 - '@smithy/middleware-endpoint@4.4.32': + '@smithy/util-body-length-browser@4.3.3': dependencies: - '@smithy/core': 3.23.17 - '@smithy/middleware-serde': 4.2.20 - '@smithy/node-config-provider': 4.3.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - '@smithy/url-parser': 4.2.14 - '@smithy/util-middleware': 4.2.14 + '@smithy/core': 3.24.3 tslib: 2.8.1 - '@smithy/middleware-retry@4.5.7': - dependencies: - '@smithy/core': 3.23.17 - '@smithy/node-config-provider': 4.3.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/service-error-classification': 4.3.1 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-retry': 4.3.6 - '@smithy/uuid': 1.1.2 - tslib: 2.8.1 - - '@smithy/middleware-serde@4.2.20': - dependencies: - '@smithy/core': 3.23.17 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/middleware-stack@4.2.14': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/node-config-provider@4.3.14': - dependencies: - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/node-http-handler@4.6.1': - dependencies: - '@smithy/protocol-http': 5.3.14 - '@smithy/querystring-builder': 4.2.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/property-provider@4.2.14': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/protocol-http@5.3.14': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/querystring-builder@4.2.14': - dependencies: - '@smithy/types': 4.14.1 - '@smithy/util-uri-escape': 4.2.2 - tslib: 2.8.1 - - '@smithy/querystring-parser@4.2.14': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/service-error-classification@4.3.1': - dependencies: - '@smithy/types': 4.14.1 - - '@smithy/shared-ini-file-loader@4.4.9': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/signature-v4@5.3.14': - dependencies: - '@smithy/is-array-buffer': 4.2.2 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - '@smithy/util-hex-encoding': 4.2.2 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-uri-escape': 4.2.2 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - - '@smithy/smithy-client@4.12.13': - dependencies: - '@smithy/core': 3.23.17 - '@smithy/middleware-endpoint': 4.4.32 - '@smithy/middleware-stack': 4.2.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - '@smithy/util-stream': 4.5.25 - tslib: 2.8.1 - - '@smithy/types@4.14.1': - dependencies: - tslib: 2.8.1 - - '@smithy/url-parser@4.2.14': - dependencies: - '@smithy/querystring-parser': 4.2.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/util-base64@4.3.2': - dependencies: - '@smithy/util-buffer-from': 4.2.2 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - - '@smithy/util-body-length-browser@4.2.2': - dependencies: - tslib: 2.8.1 - - '@smithy/util-body-length-node@4.2.3': + '@smithy/util-body-length-node@4.3.3': dependencies: + '@smithy/core': 3.24.3 tslib: 2.8.1 '@smithy/util-buffer-from@2.2.0': @@ -8245,66 +8106,34 @@ snapshots: '@smithy/is-array-buffer': 2.2.0 tslib: 2.8.1 - '@smithy/util-buffer-from@4.2.2': + '@smithy/util-defaults-mode-browser@4.4.3': dependencies: - '@smithy/is-array-buffer': 4.2.2 + '@smithy/core': 3.24.3 tslib: 2.8.1 - '@smithy/util-config-provider@4.2.2': + '@smithy/util-defaults-mode-node@4.3.3': dependencies: + '@smithy/core': 3.24.3 tslib: 2.8.1 - '@smithy/util-defaults-mode-browser@4.3.49': + '@smithy/util-endpoints@3.5.3': dependencies: - '@smithy/property-provider': 4.2.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 + '@smithy/core': 3.24.3 tslib: 2.8.1 - '@smithy/util-defaults-mode-node@4.2.54': + '@smithy/util-middleware@4.3.3': dependencies: - '@smithy/config-resolver': 4.4.17 - '@smithy/credential-provider-imds': 4.2.14 - '@smithy/node-config-provider': 4.3.14 - '@smithy/property-provider': 4.2.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 + '@smithy/core': 3.24.3 tslib: 2.8.1 - '@smithy/util-endpoints@3.4.2': + '@smithy/util-retry@4.4.3': dependencies: - '@smithy/node-config-provider': 4.3.14 - '@smithy/types': 4.14.1 + '@smithy/core': 3.24.3 tslib: 2.8.1 - '@smithy/util-hex-encoding@4.2.2': - dependencies: - tslib: 2.8.1 - - '@smithy/util-middleware@4.2.14': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/util-retry@4.3.6': - dependencies: - '@smithy/service-error-classification': 4.3.1 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/util-stream@4.5.25': - dependencies: - '@smithy/fetch-http-handler': 5.3.17 - '@smithy/node-http-handler': 4.6.1 - '@smithy/types': 4.14.1 - '@smithy/util-base64': 4.3.2 - '@smithy/util-buffer-from': 4.2.2 - '@smithy/util-hex-encoding': 4.2.2 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - - '@smithy/util-uri-escape@4.2.2': + '@smithy/util-stream@4.6.3': dependencies: + '@smithy/core': 3.24.3 tslib: 2.8.1 '@smithy/util-utf8@2.3.0': @@ -8312,18 +8141,14 @@ snapshots: '@smithy/util-buffer-from': 2.2.0 tslib: 2.8.1 - '@smithy/util-utf8@4.2.2': + '@smithy/util-utf8@4.3.3': dependencies: - '@smithy/util-buffer-from': 4.2.2 + '@smithy/core': 3.24.3 tslib: 2.8.1 - '@smithy/util-waiter@4.3.0': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/uuid@1.1.2': + '@smithy/util-waiter@4.4.3': dependencies: + '@smithy/core': 3.24.3 tslib: 2.8.1 '@so-ric/colorspace@1.1.6': @@ -8331,6 +8156,8 @@ snapshots: color: 5.0.3 text-hex: 1.0.0 + '@stablelib/base64@1.0.1': {} + '@standard-schema/spec@1.1.0': {} '@swc/helpers@0.5.15': @@ -8340,7 +8167,7 @@ snapshots: '@tailwindcss/node@4.3.0': dependencies: '@jridgewell/remapping': 2.3.5 - enhanced-resolve: 5.21.2 + enhanced-resolve: 5.21.6 jiti: 2.7.0 lightningcss: 1.32.0 magic-string: 0.30.21 @@ -8416,14 +8243,14 @@ snapshots: '@tediousjs/connection-string@1.1.0': {} - '@tybys/wasm-util@0.10.1': + '@tybys/wasm-util@0.10.2': dependencies: tslib: 2.8.1 optional: true '@types/better-sqlite3@7.6.13': dependencies: - '@types/node': 24.12.2 + '@types/node': 24.12.4 '@types/chai@5.2.3': dependencies: @@ -8459,9 +8286,9 @@ snapshots: '@types/estree-jsx@1.0.5': dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 - '@types/estree@1.0.8': {} + '@types/estree@1.0.9': {} '@types/hast@3.0.4': dependencies: @@ -8477,14 +8304,14 @@ snapshots: '@types/mssql@12.3.0(@azure/core-client@1.10.1)': dependencies: - '@types/node': 24.12.2 + '@types/node': 24.12.4 tarn: 3.0.2 tedious: 19.2.1(@azure/core-client@1.10.1) transitivePeerDependencies: - '@azure/core-client' - supports-color - '@types/node@24.12.2': + '@types/node@24.12.4': dependencies: undici-types: 7.16.0 @@ -8492,21 +8319,23 @@ snapshots: '@types/pg@8.20.0': dependencies: - '@types/node': 24.12.2 - pg-protocol: 1.13.0 + '@types/node': 24.12.4 + pg-protocol: 1.14.0 pg-types: 2.2.0 - '@types/react-dom@19.2.3(@types/react@19.2.14)': + '@types/react-dom@19.2.3(@types/react@19.2.15)': dependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - '@types/react@19.2.14': + '@types/react@19.2.15': dependencies: csstype: 3.2.3 '@types/readable-stream@4.0.23': dependencies: - '@types/node': 24.12.2 + '@types/node': 24.12.4 + + '@types/semver@7.7.1': {} '@types/triple-beam@1.3.5': {} @@ -8526,68 +8355,68 @@ snapshots: '@vercel/oidc@3.2.0': {} - '@vitest/coverage-v8@4.1.6(vitest@4.1.6)': + '@vitest/coverage-v8@4.1.7(vitest@4.1.7)': dependencies: '@bcoe/v8-coverage': 1.0.2 - '@vitest/utils': 4.1.6 + '@vitest/utils': 4.1.7 ast-v8-to-istanbul: 1.0.0 istanbul-lib-coverage: 3.2.2 istanbul-lib-report: 3.0.1 istanbul-reports: 3.2.0 - magicast: 0.5.2 + magicast: 0.5.3 obug: 2.1.1 std-env: 4.1.0 tinyrainbow: 3.1.0 - vitest: 4.1.6(@opentelemetry/api@1.9.0)(@types/node@24.12.2)(@vitest/coverage-v8@4.1.6)(vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) + vitest: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.7)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) - '@vitest/expect@4.1.6': + '@vitest/expect@4.1.7': dependencies: '@standard-schema/spec': 1.1.0 '@types/chai': 5.2.3 - '@vitest/spy': 4.1.6 - '@vitest/utils': 4.1.6 + '@vitest/spy': 4.1.7 + '@vitest/utils': 4.1.7 chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.6(vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0))': + '@vitest/mocker@4.1.7(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0))': dependencies: - '@vitest/spy': 4.1.6 + '@vitest/spy': 4.1.7 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0) + vite: 8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0) - '@vitest/pretty-format@4.1.6': + '@vitest/pretty-format@4.1.7': dependencies: tinyrainbow: 3.1.0 - '@vitest/runner@4.1.6': + '@vitest/runner@4.1.7': dependencies: - '@vitest/utils': 4.1.6 + '@vitest/utils': 4.1.7 pathe: 2.0.3 - '@vitest/snapshot@4.1.6': + '@vitest/snapshot@4.1.7': dependencies: - '@vitest/pretty-format': 4.1.6 - '@vitest/utils': 4.1.6 + '@vitest/pretty-format': 4.1.7 + '@vitest/utils': 4.1.7 magic-string: 0.30.21 pathe: 2.0.3 - '@vitest/spy@4.1.6': {} + '@vitest/spy@4.1.7': {} - '@vitest/utils@4.1.6': + '@vitest/utils@4.1.7': dependencies: - '@vitest/pretty-format': 4.1.6 + '@vitest/pretty-format': 4.1.7 convert-source-map: 2.0.0 tinyrainbow: 3.1.0 - '@xyflow/react@12.10.2(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + '@xyflow/react@12.10.2(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: '@xyflow/system': 0.0.76 classcat: 5.0.5 react: 19.2.6 react-dom: 19.2.6(react@19.2.6) - zustand: 4.5.7(@types/react@19.2.14)(react@19.2.6) + zustand: 4.5.7(@types/react@19.2.15)(react@19.2.6) transitivePeerDependencies: - '@types/react' - immer @@ -8619,6 +8448,12 @@ snapshots: acorn@8.16.0: {} + agent-base@6.0.2: + dependencies: + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + agent-base@7.1.4: {} agent-base@9.0.0: {} @@ -8633,12 +8468,12 @@ snapshots: clean-stack: 5.3.0 indent-string: 5.0.0 - ai@6.0.180(zod@4.4.3): + ai@6.0.188(zod@4.4.3): dependencies: - '@ai-sdk/gateway': 3.0.114(zod@4.4.3) + '@ai-sdk/gateway': 3.0.118(zod@4.4.3) '@ai-sdk/provider': 3.0.10 '@ai-sdk/provider-utils': 4.0.27(zod@4.4.3) - '@opentelemetry/api': 1.9.0 + '@opentelemetry/api': 1.9.1 zod: 4.4.3 ajv-formats@3.0.1(ajv@8.20.0): @@ -8720,13 +8555,15 @@ snapshots: aws-ssl-profiles@1.1.2: {} - axios@1.15.2: + axios@1.16.1: dependencies: follow-redirects: 1.16.0 form-data: 4.0.5 + https-proxy-agent: 5.0.1 proxy-from-env: 2.1.0 transitivePeerDependencies: - debug + - supports-color bail@2.0.2: {} @@ -8736,7 +8573,7 @@ snapshots: base64-js@1.5.1: {} - baseline-browser-mapping@2.10.29: {} + baseline-browser-mapping@2.10.31: {} before-after-hook@4.0.0: {} @@ -8780,9 +8617,9 @@ snapshots: http-errors: 2.0.1 iconv-lite: 0.7.2 on-finished: 2.4.1 - qs: 6.15.1 + qs: 6.15.2 raw-body: 3.0.2 - type-is: 2.0.1 + type-is: 2.1.0 transitivePeerDependencies: - supports-color @@ -8833,7 +8670,7 @@ snapshots: callsites@3.1.0: {} - caniuse-lite@1.0.30001792: {} + caniuse-lite@1.0.30001793: {} ccount@2.0.1: {} @@ -8917,7 +8754,7 @@ snapshots: cliui@9.0.1: dependencies: string-width: 7.2.0 - strip-ansi: 7.1.2 + strip-ansi: 7.2.0 wrap-ansi: 9.0.2 clsx@2.1.1: {} @@ -8999,7 +8836,7 @@ snapshots: conventional-commits-filter: 5.0.0 handlebars: 4.7.9 meow: 13.2.0 - semver: 7.7.4 + semver: 7.8.0 conventional-commits-filter@5.0.0: {} @@ -9168,7 +9005,7 @@ snapshots: dependencies: once: 1.4.0 - enhanced-resolve@5.21.2: + enhanced-resolve@5.21.6: dependencies: graceful-fs: 4.2.11 tapable: 2.3.3 @@ -9262,7 +9099,7 @@ snapshots: estree-util-attach-comments@3.0.0: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 estree-util-build-jsx@3.0.1: dependencies: @@ -9275,7 +9112,7 @@ snapshots: estree-util-scope@1.0.0: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 devlop: 1.1.0 estree-util-to-js@2.0.0: @@ -9286,7 +9123,7 @@ snapshots: estree-util-value-to-estree@3.5.0: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 estree-util-visit@2.0.0: dependencies: @@ -9295,7 +9132,7 @@ snapshots: estree-walker@3.0.3: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 etag@1.8.1: {} @@ -9356,7 +9193,7 @@ snapshots: expect-type@1.3.0: {} - express-rate-limit@8.4.1(express@5.2.1): + express-rate-limit@8.5.2(express@5.2.1): dependencies: express: 5.2.1 ip-address: 10.1.1 @@ -9383,13 +9220,13 @@ snapshots: once: 1.4.0 parseurl: 1.3.3 proxy-addr: 2.0.7 - qs: 6.15.1 + qs: 6.15.2 range-parser: 1.2.1 router: 2.2.0 send: 1.2.1 serve-static: 2.2.1 statuses: 2.0.2 - type-is: 2.0.1 + type-is: 2.1.0 vary: 1.1.2 transitivePeerDependencies: - supports-color @@ -9400,6 +9237,8 @@ snapshots: fast-deep-equal@3.1.3: {} + fast-sha256@1.3.0: {} + fast-string-truncated-width@3.0.3: {} fast-string-width@3.0.2: @@ -9408,7 +9247,7 @@ snapshots: fast-uri@3.1.2: {} - fast-wrap-ansi@0.2.0: + fast-wrap-ansi@0.2.2: dependencies: fast-string-width: 3.0.2 @@ -9416,12 +9255,20 @@ snapshots: dependencies: path-expression-matcher: 1.5.0 - fast-xml-parser@5.7.2: + fast-xml-parser@5.7.3: dependencies: '@nodable/entities': 2.1.0 fast-xml-builder: 1.1.7 path-expression-matcher: 1.5.0 - strnum: 2.2.3 + strnum: 2.3.0 + + fast-xml-parser@5.8.0: + dependencies: + '@nodable/entities': 2.1.0 + fast-xml-builder: 1.1.7 + path-expression-matcher: 1.5.0 + strnum: 2.3.0 + xml-naming: 0.1.0 fastest-levenshtein@1.0.16: {} @@ -9440,7 +9287,7 @@ snapshots: node-domexception: 1.0.0 web-streams-polyfill: 3.3.3 - fflate@0.8.2: {} + fflate@0.8.3: {} figures@2.0.0: dependencies: @@ -9500,10 +9347,10 @@ snapshots: forwarded@0.2.0: {} - framer-motion@12.38.0(react-dom@19.2.6(react@19.2.6))(react@19.2.6): + framer-motion@12.40.0(react-dom@19.2.6(react@19.2.6))(react@19.2.6): dependencies: - motion-dom: 12.38.0 - motion-utils: 12.36.0 + motion-dom: 12.40.0 + motion-utils: 12.39.0 tslib: 2.8.1 optionalDependencies: react: 19.2.6 @@ -9524,7 +9371,7 @@ snapshots: fsevents@2.3.3: optional: true - fumadocs-core@16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.14)(lucide-react@1.14.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3): + fumadocs-core@16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.15)(lucide-react@1.16.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3): dependencies: '@orama/orama': 3.1.18 estree-util-value-to-estree: 3.5.0 @@ -9538,7 +9385,7 @@ snapshots: remark-gfm: 4.0.1 remark-rehype: 11.1.2 scroll-into-view-if-needed: 3.1.0 - shiki: 4.0.2 + shiki: 4.1.0 tinyglobby: 0.2.16 unified: 11.0.5 unist-util-visit: 5.1.0 @@ -9548,23 +9395,23 @@ snapshots: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 '@types/mdast': 4.0.4 - '@types/react': 19.2.14 - lucide-react: 1.14.0(react@19.2.6) - next: 16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@types/react': 19.2.15 + lucide-react: 1.16.0(react@19.2.6) + next: 16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) zod: 4.4.3 transitivePeerDependencies: - supports-color - fumadocs-mdx@15.0.4(@types/mdast@4.0.4)(@types/mdx@2.0.13)(@types/react@19.2.14)(fumadocs-core@16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.14)(lucide-react@1.14.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3))(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react@19.2.6)(vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)): + fumadocs-mdx@15.0.7(@types/mdast@4.0.4)(@types/mdx@2.0.13)(@types/react@19.2.15)(fumadocs-core@16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.15)(lucide-react@1.16.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react@19.2.6)(rolldown@1.0.2)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)): dependencies: '@mdx-js/mdx': 3.1.1 '@standard-schema/spec': 1.1.0 chokidar: 5.0.0 esbuild: 0.28.0 estree-util-value-to-estree: 3.5.0 - fumadocs-core: 16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.14)(lucide-react@1.14.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3) + fumadocs-core: 16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.15)(lucide-react@1.16.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3) js-yaml: 4.1.1 mdast-util-mdx: 3.0.0 picocolors: 1.1.1 @@ -9579,43 +9426,44 @@ snapshots: optionalDependencies: '@types/mdast': 4.0.4 '@types/mdx': 2.0.13 - '@types/react': 19.2.14 - next: 16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@types/react': 19.2.15 + next: 16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) react: 19.2.6 - vite: 8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0) + rolldown: 1.0.2 + vite: 8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0) transitivePeerDependencies: - supports-color - fumadocs-ui@16.8.10(@tailwindcss/oxide@4.3.0)(@types/mdx@2.0.13)(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(fumadocs-core@16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.14)(lucide-react@1.14.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3))(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(tailwindcss@4.3.0): + fumadocs-ui@16.8.10(@tailwindcss/oxide@4.3.0)(@types/mdx@2.0.13)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(fumadocs-core@16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.15)(lucide-react@1.16.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(tailwindcss@4.3.0): dependencies: '@fumadocs/tailwind': 0.0.5(@tailwindcss/oxide@4.3.0)(tailwindcss@4.3.0) - '@radix-ui/react-accordion': 1.2.12(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-collapsible': 1.1.12(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-dialog': 1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-direction': 1.1.1(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-navigation-menu': 1.2.14(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-popover': 1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-presence': 1.1.5(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-scroll-area': 1.2.10(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) - '@radix-ui/react-slot': 1.2.4(@types/react@19.2.14)(react@19.2.6) - '@radix-ui/react-tabs': 1.1.13(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-accordion': 1.2.12(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-collapsible': 1.1.12(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-dialog': 1.1.15(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-direction': 1.1.1(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-navigation-menu': 1.2.14(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-popover': 1.1.15(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-presence': 1.1.5(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-scroll-area': 1.2.10(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@radix-ui/react-slot': 1.2.4(@types/react@19.2.15)(react@19.2.6) + '@radix-ui/react-tabs': 1.1.13(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) class-variance-authority: 0.7.1 - fumadocs-core: 16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.14)(lucide-react@1.14.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3) - lucide-react: 1.14.0(react@19.2.6) - motion: 12.38.0(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + fumadocs-core: 16.8.10(@mdx-js/mdx@3.1.1)(@types/estree-jsx@1.0.5)(@types/hast@3.0.4)(@types/mdast@4.0.4)(@types/react@19.2.15)(lucide-react@1.16.0(react@19.2.6))(next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(zod@4.4.3) + lucide-react: 1.16.0(react@19.2.6) + motion: 12.40.0(react-dom@19.2.6(react@19.2.6))(react@19.2.6) next-themes: 0.4.6(react-dom@19.2.6(react@19.2.6))(react@19.2.6) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) - react-remove-scroll: 2.7.2(@types/react@19.2.14)(react@19.2.6) + react-remove-scroll: 2.7.2(@types/react@19.2.15)(react@19.2.6) rehype-raw: 7.0.0 scroll-into-view-if-needed: 3.1.0 - shiki: 4.0.2 + shiki: 4.1.0 tailwind-merge: 3.6.0 unist-util-visit: 5.1.0 optionalDependencies: '@types/mdx': 2.0.13 - '@types/react': 19.2.14 - next: 16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + '@types/react': 19.2.15 + next: 16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) transitivePeerDependencies: - '@emotion/is-prop-valid' - '@tailwindcss/oxide' @@ -9650,7 +9498,7 @@ snapshots: get-caller-file@2.0.5: {} - get-east-asian-width@1.5.0: {} + get-east-asian-width@1.6.0: {} get-intrinsic@1.3.0: dependencies: @@ -9782,7 +9630,7 @@ snapshots: hast-util-to-estree@3.1.3: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 comma-separated-tokens: 2.0.3 @@ -9817,7 +9665,7 @@ snapshots: hast-util-to-jsx-runtime@2.3.6: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 '@types/hast': 3.0.4 '@types/unist': 3.0.3 comma-separated-tokens: 2.0.3 @@ -9873,12 +9721,14 @@ snapshots: hosted-git-info@9.0.3: dependencies: - lru-cache: 11.3.6 + lru-cache: 11.5.0 html-entities@2.6.0: {} html-escaper@2.0.2: {} + html-to-image@1.11.11: {} + html-void-elements@3.0.0: {} http-errors@2.0.1: @@ -9903,6 +9753,13 @@ snapshots: transitivePeerDependencies: - supports-color + https-proxy-agent@5.0.1: + dependencies: + agent-base: 6.0.2 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + https-proxy-agent@7.0.6: dependencies: agent-base: 7.1.4 @@ -9960,11 +9817,11 @@ snapshots: ini@5.0.0: {} - ink-testing-library@4.0.0(@types/react@19.2.14): + ink-testing-library@4.0.0(@types/react@19.2.15): optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - ink@7.0.2(@types/react@19.2.14)(react@19.2.6): + ink@7.0.3(@types/react@19.2.15)(react@19.2.6): dependencies: '@alcalzone/ansi-tokenize': 0.3.0 ansi-escapes: 7.3.0 @@ -9993,7 +9850,7 @@ snapshots: ws: 8.20.1 yoga-layout: 3.2.1 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 transitivePeerDependencies: - bufferutil - utf-8-validate @@ -10023,7 +9880,7 @@ snapshots: is-fullwidth-code-point@5.1.0: dependencies: - get-east-asian-width: 1.5.0 + get-east-asian-width: 1.6.0 is-hexadecimal@2.0.1: {} @@ -10088,7 +9945,7 @@ snapshots: jiti@2.7.0: {} - jose@6.2.2: {} + jose@6.2.3: {} js-md4@0.3.2: {} @@ -10138,7 +9995,7 @@ snapshots: lodash.isstring: 4.0.1 lodash.once: 4.1.1 ms: 2.1.3 - semver: 7.7.4 + semver: 7.8.0 jwa@2.0.1: dependencies: @@ -10151,14 +10008,14 @@ snapshots: jwa: 2.0.1 safe-buffer: 5.2.1 - knip@6.12.2(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0): + knip@6.14.1(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0): dependencies: fdir: 6.5.0(picomatch@4.0.4) formatly: 0.3.0 get-tsconfig: 4.14.0 jiti: 2.7.0 minimist: 1.2.8 - oxc-parser: 0.128.0 + oxc-parser: 0.130.0 oxc-resolver: 11.19.1(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) picomatch: 4.0.4 smol-toml: 1.6.1 @@ -10284,11 +10141,11 @@ snapshots: lru-cache@10.4.3: {} - lru-cache@11.3.6: {} + lru-cache@11.5.0: {} lru.min@1.1.4: {} - lucide-react@1.14.0(react@19.2.6): + lucide-react@1.16.0(react@19.2.6): dependencies: react: 19.2.6 @@ -10296,7 +10153,7 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 - magicast@0.5.2: + magicast@0.5.3: dependencies: '@babel/parser': 7.29.3 '@babel/types': 7.29.0 @@ -10310,7 +10167,7 @@ snapshots: make-dir@4.0.0: dependencies: - semver: 7.7.4 + semver: 7.8.0 markdown-extensions@2.0.0: {} @@ -10581,7 +10438,7 @@ snapshots: micromark-extension-mdx-expression@3.0.1: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 devlop: 1.1.0 micromark-factory-mdx-expression: 2.0.3 micromark-factory-space: 2.0.1 @@ -10592,7 +10449,7 @@ snapshots: micromark-extension-mdx-jsx@3.0.2: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 devlop: 1.1.0 estree-util-is-identifier-name: 3.0.0 micromark-factory-mdx-expression: 2.0.3 @@ -10609,7 +10466,7 @@ snapshots: micromark-extension-mdxjs-esm@3.0.0: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 devlop: 1.1.0 micromark-core-commonmark: 2.0.3 micromark-util-character: 2.1.1 @@ -10645,7 +10502,7 @@ snapshots: micromark-factory-mdx-expression@2.0.3: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 devlop: 1.1.0 micromark-factory-space: 2.0.1 micromark-util-character: 2.1.1 @@ -10709,7 +10566,7 @@ snapshots: micromark-util-events-to-acorn@2.0.3: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 '@types/unist': 3.0.3 devlop: 1.1.0 estree-util-visit: 2.0.0 @@ -10811,15 +10668,15 @@ snapshots: moment@2.30.1: {} - motion-dom@12.38.0: + motion-dom@12.40.0: dependencies: - motion-utils: 12.36.0 + motion-utils: 12.39.0 - motion-utils@12.36.0: {} + motion-utils@12.39.0: {} - motion@12.38.0(react-dom@19.2.6(react@19.2.6))(react@19.2.6): + motion@12.40.0(react-dom@19.2.6(react@19.2.6))(react@19.2.6): dependencies: - framer-motion: 12.38.0(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + framer-motion: 12.40.0(react-dom@19.2.6(react@19.2.6))(react@19.2.6) tslib: 2.8.1 optionalDependencies: react: 19.2.6 @@ -10827,7 +10684,7 @@ snapshots: ms@2.1.3: {} - mssql@12.5.2(@azure/core-client@1.10.1): + mssql@12.5.4(@azure/core-client@1.10.1): dependencies: '@tediousjs/connection-string': 1.1.0 commander: 11.1.0 @@ -10838,9 +10695,9 @@ snapshots: - '@azure/core-client' - supports-color - mysql2@3.22.3(@types/node@24.12.2): + mysql2@3.22.3(@types/node@24.12.4): dependencies: - '@types/node': 24.12.2 + '@types/node': 24.12.4 aws-ssl-profiles: 1.1.2 denque: 2.1.0 generate-function: 2.3.1 @@ -10860,7 +10717,7 @@ snapshots: dependencies: lru.min: 1.1.4 - nanoid@3.3.11: {} + nanoid@3.3.12: {} napi-build-utils@2.0.0: {} @@ -10877,12 +10734,12 @@ snapshots: react: 19.2.6 react-dom: 19.2.6(react@19.2.6) - next@16.2.6(@opentelemetry/api@1.9.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6): + next@16.2.6(@opentelemetry/api@1.9.1)(react-dom@19.2.6(react@19.2.6))(react@19.2.6): dependencies: '@next/env': 16.2.6 '@swc/helpers': 0.5.15 - baseline-browser-mapping: 2.10.29 - caniuse-lite: 1.0.30001792 + baseline-browser-mapping: 2.10.31 + caniuse-lite: 1.0.30001793 postcss: 8.5.10 react: 19.2.6 react-dom: 19.2.6(react@19.2.6) @@ -10896,15 +10753,15 @@ snapshots: '@next/swc-linux-x64-musl': 16.2.6 '@next/swc-win32-arm64-msvc': 16.2.6 '@next/swc-win32-x64-msvc': 16.2.6 - '@opentelemetry/api': 1.9.0 + '@opentelemetry/api': 1.9.1 sharp: 0.34.5 transitivePeerDependencies: - '@babel/core' - babel-plugin-macros - node-abi@3.89.0: + node-abi@3.92.0: dependencies: - semver: 7.7.4 + semver: 7.8.0 node-domexception@1.0.0: {} @@ -10924,16 +10781,16 @@ snapshots: normalize-package-data@6.0.2: dependencies: hosted-git-info: 7.0.2 - semver: 7.7.4 + semver: 7.8.0 validate-npm-package-license: 3.0.4 normalize-package-data@8.0.0: dependencies: hosted-git-info: 9.0.3 - semver: 7.7.4 + semver: 7.8.0 validate-npm-package-license: 3.0.4 - normalize-url@9.0.0: {} + normalize-url@9.0.1: {} npm-run-path@4.0.1: dependencies: @@ -10948,7 +10805,7 @@ snapshots: path-key: 4.0.0 unicorn-magic: 0.3.0 - npm@11.14.1: {} + npm@11.15.0: {} oauth4webapi@3.8.6: {} @@ -10998,35 +10855,35 @@ snapshots: is-docker: 2.2.1 is-wsl: 2.2.0 - openai@6.37.0(ws@8.20.1)(zod@4.4.3): + openai@6.38.0(ws@8.20.1)(zod@4.4.3): optionalDependencies: ws: 8.20.1 zod: 4.4.3 - oxc-parser@0.128.0: + oxc-parser@0.130.0: dependencies: - '@oxc-project/types': 0.128.0 + '@oxc-project/types': 0.130.0 optionalDependencies: - '@oxc-parser/binding-android-arm-eabi': 0.128.0 - '@oxc-parser/binding-android-arm64': 0.128.0 - '@oxc-parser/binding-darwin-arm64': 0.128.0 - '@oxc-parser/binding-darwin-x64': 0.128.0 - '@oxc-parser/binding-freebsd-x64': 0.128.0 - '@oxc-parser/binding-linux-arm-gnueabihf': 0.128.0 - '@oxc-parser/binding-linux-arm-musleabihf': 0.128.0 - '@oxc-parser/binding-linux-arm64-gnu': 0.128.0 - '@oxc-parser/binding-linux-arm64-musl': 0.128.0 - '@oxc-parser/binding-linux-ppc64-gnu': 0.128.0 - '@oxc-parser/binding-linux-riscv64-gnu': 0.128.0 - '@oxc-parser/binding-linux-riscv64-musl': 0.128.0 - '@oxc-parser/binding-linux-s390x-gnu': 0.128.0 - '@oxc-parser/binding-linux-x64-gnu': 0.128.0 - '@oxc-parser/binding-linux-x64-musl': 0.128.0 - '@oxc-parser/binding-openharmony-arm64': 0.128.0 - '@oxc-parser/binding-wasm32-wasi': 0.128.0 - '@oxc-parser/binding-win32-arm64-msvc': 0.128.0 - '@oxc-parser/binding-win32-ia32-msvc': 0.128.0 - '@oxc-parser/binding-win32-x64-msvc': 0.128.0 + '@oxc-parser/binding-android-arm-eabi': 0.130.0 + '@oxc-parser/binding-android-arm64': 0.130.0 + '@oxc-parser/binding-darwin-arm64': 0.130.0 + '@oxc-parser/binding-darwin-x64': 0.130.0 + '@oxc-parser/binding-freebsd-x64': 0.130.0 + '@oxc-parser/binding-linux-arm-gnueabihf': 0.130.0 + '@oxc-parser/binding-linux-arm-musleabihf': 0.130.0 + '@oxc-parser/binding-linux-arm64-gnu': 0.130.0 + '@oxc-parser/binding-linux-arm64-musl': 0.130.0 + '@oxc-parser/binding-linux-ppc64-gnu': 0.130.0 + '@oxc-parser/binding-linux-riscv64-gnu': 0.130.0 + '@oxc-parser/binding-linux-riscv64-musl': 0.130.0 + '@oxc-parser/binding-linux-s390x-gnu': 0.130.0 + '@oxc-parser/binding-linux-x64-gnu': 0.130.0 + '@oxc-parser/binding-linux-x64-musl': 0.130.0 + '@oxc-parser/binding-openharmony-arm64': 0.130.0 + '@oxc-parser/binding-wasm32-wasi': 0.130.0 + '@oxc-parser/binding-win32-arm64-msvc': 0.130.0 + '@oxc-parser/binding-win32-ia32-msvc': 0.130.0 + '@oxc-parser/binding-win32-x64-msvc': 0.130.0 oxc-resolver@11.19.1(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0): optionalDependencies: @@ -11156,18 +11013,18 @@ snapshots: pegjs@0.10.0: {} - pg-cloudflare@1.3.0: + pg-cloudflare@1.4.0: optional: true - pg-connection-string@2.12.0: {} + pg-connection-string@2.13.0: {} pg-int8@1.0.1: {} - pg-pool@3.13.0(pg@8.20.0): + pg-pool@3.14.0(pg@8.21.0): dependencies: - pg: 8.20.0 + pg: 8.21.0 - pg-protocol@1.13.0: {} + pg-protocol@1.14.0: {} pg-types@2.2.0: dependencies: @@ -11177,15 +11034,15 @@ snapshots: postgres-date: 1.0.7 postgres-interval: 1.2.0 - pg@8.20.0: + pg@8.21.0: dependencies: - pg-connection-string: 2.12.0 - pg-pool: 3.13.0(pg@8.20.0) - pg-protocol: 1.13.0 + pg-connection-string: 2.13.0 + pg-pool: 3.14.0(pg@8.21.0) + pg-protocol: 1.14.0 pg-types: 2.2.0 pgpass: 1.0.5 optionalDependencies: - pg-cloudflare: 1.3.0 + pg-cloudflare: 1.4.0 pgpass@1.0.5: dependencies: @@ -11208,7 +11065,7 @@ snapshots: postcss@8.5.10: dependencies: - nanoid: 3.3.11 + nanoid: 3.3.12 picocolors: 1.1.1 source-map-js: 1.2.1 @@ -11222,7 +11079,9 @@ snapshots: dependencies: xtend: 4.0.2 - posthog-node@5.0.0: {} + posthog-node@5.34.9: + dependencies: + '@posthog/core': 1.29.7 prebuild-install@7.1.3: dependencies: @@ -11232,7 +11091,7 @@ snapshots: minimist: 1.2.8 mkdirp-classic: 0.5.3 napi-build-utils: 2.0.0 - node-abi: 3.89.0 + node-abi: 3.92.0 pump: 3.0.4 rc: 1.2.8 simple-get: 4.0.1 @@ -11263,7 +11122,7 @@ snapshots: end-of-stream: 1.4.5 once: 1.4.0 - qs@6.15.1: + qs@6.15.2: dependencies: side-channel: 1.1.0 @@ -11293,32 +11152,32 @@ snapshots: react: 19.2.6 scheduler: 0.27.0 - react-remove-scroll-bar@2.3.8(@types/react@19.2.14)(react@19.2.6): + react-remove-scroll-bar@2.3.8(@types/react@19.2.15)(react@19.2.6): dependencies: react: 19.2.6 - react-style-singleton: 2.2.3(@types/react@19.2.14)(react@19.2.6) + react-style-singleton: 2.2.3(@types/react@19.2.15)(react@19.2.6) tslib: 2.8.1 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - react-remove-scroll@2.7.2(@types/react@19.2.14)(react@19.2.6): + react-remove-scroll@2.7.2(@types/react@19.2.15)(react@19.2.6): dependencies: react: 19.2.6 - react-remove-scroll-bar: 2.3.8(@types/react@19.2.14)(react@19.2.6) - react-style-singleton: 2.2.3(@types/react@19.2.14)(react@19.2.6) + react-remove-scroll-bar: 2.3.8(@types/react@19.2.15)(react@19.2.6) + react-style-singleton: 2.2.3(@types/react@19.2.15)(react@19.2.6) tslib: 2.8.1 - use-callback-ref: 1.3.3(@types/react@19.2.14)(react@19.2.6) - use-sidecar: 1.1.3(@types/react@19.2.14)(react@19.2.6) + use-callback-ref: 1.3.3(@types/react@19.2.15)(react@19.2.6) + use-sidecar: 1.1.3(@types/react@19.2.15)(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - react-style-singleton@2.2.3(@types/react@19.2.14)(react@19.2.6): + react-style-singleton@2.2.3(@types/react@19.2.15)(react@19.2.6): dependencies: get-nonce: 1.0.1 react: 19.2.6 tslib: 2.8.1 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 react@19.2.6: {} @@ -11378,7 +11237,7 @@ snapshots: recma-build-jsx@1.0.0: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 estree-util-build-jsx: 3.0.1 vfile: 6.0.3 @@ -11393,14 +11252,14 @@ snapshots: recma-parse@1.0.0: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 esast-util-from-js: 2.0.1 unified: 11.0.5 vfile: 6.0.3 recma-stringify@1.0.0: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 estree-util-to-js: 2.0.0 unified: 11.0.5 vfile: 6.0.3 @@ -11427,7 +11286,7 @@ snapshots: rehype-recma@1.0.0: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 '@types/hast': 3.0.4 hast-util-to-estree: 3.1.3 transitivePeerDependencies: @@ -11505,26 +11364,26 @@ snapshots: transitivePeerDependencies: - supports-color - rolldown@1.0.0-rc.17: + rolldown@1.0.2: dependencies: - '@oxc-project/types': 0.127.0 - '@rolldown/pluginutils': 1.0.0-rc.17 + '@oxc-project/types': 0.132.0 + '@rolldown/pluginutils': 1.0.1 optionalDependencies: - '@rolldown/binding-android-arm64': 1.0.0-rc.17 - '@rolldown/binding-darwin-arm64': 1.0.0-rc.17 - '@rolldown/binding-darwin-x64': 1.0.0-rc.17 - '@rolldown/binding-freebsd-x64': 1.0.0-rc.17 - '@rolldown/binding-linux-arm-gnueabihf': 1.0.0-rc.17 - '@rolldown/binding-linux-arm64-gnu': 1.0.0-rc.17 - '@rolldown/binding-linux-arm64-musl': 1.0.0-rc.17 - '@rolldown/binding-linux-ppc64-gnu': 1.0.0-rc.17 - '@rolldown/binding-linux-s390x-gnu': 1.0.0-rc.17 - '@rolldown/binding-linux-x64-gnu': 1.0.0-rc.17 - '@rolldown/binding-linux-x64-musl': 1.0.0-rc.17 - '@rolldown/binding-openharmony-arm64': 1.0.0-rc.17 - '@rolldown/binding-wasm32-wasi': 1.0.0-rc.17 - '@rolldown/binding-win32-arm64-msvc': 1.0.0-rc.17 - '@rolldown/binding-win32-x64-msvc': 1.0.0-rc.17 + '@rolldown/binding-android-arm64': 1.0.2 + '@rolldown/binding-darwin-arm64': 1.0.2 + '@rolldown/binding-darwin-x64': 1.0.2 + '@rolldown/binding-freebsd-x64': 1.0.2 + '@rolldown/binding-linux-arm-gnueabihf': 1.0.2 + '@rolldown/binding-linux-arm64-gnu': 1.0.2 + '@rolldown/binding-linux-arm64-musl': 1.0.2 + '@rolldown/binding-linux-ppc64-gnu': 1.0.2 + '@rolldown/binding-linux-s390x-gnu': 1.0.2 + '@rolldown/binding-linux-x64-gnu': 1.0.2 + '@rolldown/binding-linux-x64-musl': 1.0.2 + '@rolldown/binding-openharmony-arm64': 1.0.2 + '@rolldown/binding-wasm32-wasi': 1.0.2 + '@rolldown/binding-win32-arm64-msvc': 1.0.2 + '@rolldown/binding-win32-x64-msvc': 1.0.2 router@2.2.0: dependencies: @@ -11579,7 +11438,7 @@ snapshots: p-reduce: 3.0.0 read-package-up: 12.0.0 resolve-from: 5.0.0 - semver: 7.7.4 + semver: 7.8.0 signale: 1.4.0 yargs: 18.0.0 transitivePeerDependencies: @@ -11588,7 +11447,9 @@ snapshots: semver-regex@4.0.5: {} - semver@7.7.4: {} + semver@7.8.0: {} + + semver@7.8.1: {} send@1.2.1: dependencies: @@ -11621,7 +11482,7 @@ snapshots: dependencies: '@img/colour': 1.1.0 detect-libc: 2.1.2 - semver: 7.7.4 + semver: 7.8.0 optionalDependencies: '@img/sharp-darwin-arm64': 0.34.5 '@img/sharp-darwin-x64': 0.34.5 @@ -11655,14 +11516,14 @@ snapshots: shebang-regex@3.0.0: {} - shiki@4.0.2: + shiki@4.1.0: dependencies: - '@shikijs/core': 4.0.2 - '@shikijs/engine-javascript': 4.0.2 - '@shikijs/engine-oniguruma': 4.0.2 - '@shikijs/langs': 4.0.2 - '@shikijs/themes': 4.0.2 - '@shikijs/types': 4.0.2 + '@shikijs/core': 4.1.0 + '@shikijs/engine-javascript': 4.1.0 + '@shikijs/engine-oniguruma': 4.1.0 + '@shikijs/langs': 4.1.0 + '@shikijs/themes': 4.1.0 + '@shikijs/types': 4.1.0 '@shikijs/vscode-textmate': 10.0.2 '@types/hast': 3.0.4 @@ -11739,27 +11600,27 @@ snapshots: smol-toml@1.6.1: {} - snowflake-sdk@2.4.1(asn1.js@5.4.1): + snowflake-sdk@2.4.2(asn1.js@5.4.1): dependencies: '@aws-crypto/sha256-js': 5.2.0 '@aws-sdk/client-s3': 3.1045.0 '@aws-sdk/client-sts': 3.1045.0 - '@aws-sdk/credential-provider-node': 3.972.39 + '@aws-sdk/credential-provider-node': 3.972.43 '@aws-sdk/ec2-metadata-service': 3.1045.0 '@azure/identity': 4.13.1 '@azure/storage-blob': 12.26.0 - '@smithy/node-http-handler': 4.6.1 - '@smithy/protocol-http': 5.3.14 - '@smithy/signature-v4': 5.3.14 + '@smithy/node-http-handler': 4.7.3 + '@smithy/protocol-http': 5.4.3 + '@smithy/signature-v4': 5.4.3 '@techteamer/ocsp': 1.0.1 asn1.js: 5.4.1 asn1.js-rfc2560: 5.0.1(asn1.js@5.4.1) asn1.js-rfc5280: 3.0.0 - axios: 1.15.2 + axios: 1.16.1 big-integer: 1.6.52 bignumber.js: 9.3.1 expand-tilde: 2.0.2 - fast-xml-parser: 5.7.2 + fast-xml-parser: 5.8.0 fastest-levenshtein: 1.0.16 generic-pool: 3.9.0 google-auth-library: 10.6.2 @@ -11774,7 +11635,6 @@ snapshots: toml: 3.0.0 winston: 3.19.0 transitivePeerDependencies: - - aws-crt - debug - supports-color @@ -11820,6 +11680,11 @@ snapshots: stackback@0.0.2: {} + standardwebhooks@1.0.0: + dependencies: + '@stablelib/base64': 1.0.1 + fast-sha256: 1.3.0 + statuses@2.0.2: {} std-env@4.1.0: {} @@ -11844,13 +11709,13 @@ snapshots: string-width@7.2.0: dependencies: emoji-regex: 10.6.0 - get-east-asian-width: 1.5.0 - strip-ansi: 7.1.2 + get-east-asian-width: 1.6.0 + strip-ansi: 7.2.0 string-width@8.2.1: dependencies: - get-east-asian-width: 1.5.0 - strip-ansi: 7.1.2 + get-east-asian-width: 1.6.0 + strip-ansi: 7.2.0 string_decoder@1.1.1: dependencies: @@ -11869,7 +11734,7 @@ snapshots: dependencies: ansi-regex: 5.0.1 - strip-ansi@7.1.2: + strip-ansi@7.2.0: dependencies: ansi-regex: 6.2.2 @@ -11885,7 +11750,7 @@ snapshots: strip-json-comments@5.0.3: {} - strnum@2.2.3: {} + strnum@2.3.0: {} stubs@3.0.0: {} @@ -11952,7 +11817,7 @@ snapshots: '@azure/identity': 4.13.1 '@azure/keyvault-keys': 4.10.0(@azure/core-client@1.10.1) '@js-joda/core': 5.7.0 - '@types/node': 24.12.2 + '@types/node': 24.12.4 bl: 6.1.6 iconv-lite: 0.7.2 js-md4: 0.3.2 @@ -12003,8 +11868,6 @@ snapshots: tinybench@2.9.0: {} - tinyexec@1.1.1: {} - tinyexec@1.1.2: {} tinyglobby@0.2.16: @@ -12050,9 +11913,9 @@ snapshots: dependencies: tagged-tag: 1.0.0 - type-is@2.0.1: + type-is@2.1.0: dependencies: - content-type: 1.0.5 + content-type: 2.0.0 media-typer: 1.1.0 mime-types: 3.0.2 @@ -12131,20 +11994,20 @@ snapshots: url-join@5.0.0: {} - use-callback-ref@1.3.3(@types/react@19.2.14)(react@19.2.6): + use-callback-ref@1.3.3(@types/react@19.2.15)(react@19.2.6): dependencies: react: 19.2.6 tslib: 2.8.1 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 - use-sidecar@1.1.3(@types/react@19.2.14)(react@19.2.6): + use-sidecar@1.1.3(@types/react@19.2.15)(react@19.2.6): dependencies: detect-node-es: 1.1.0 react: 19.2.6 tslib: 2.8.1 optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 use-sync-external-store@1.6.0(react@19.2.6): dependencies: @@ -12174,29 +12037,29 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.3 - vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0): + vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 postcss: 8.5.10 - rolldown: 1.0.0-rc.17 + rolldown: 1.0.2 tinyglobby: 0.2.16 optionalDependencies: - '@types/node': 24.12.2 + '@types/node': 24.12.4 esbuild: 0.28.0 fsevents: 2.3.3 jiti: 2.7.0 yaml: 2.9.0 - vitest@4.1.6(@opentelemetry/api@1.9.0)(@types/node@24.12.2)(@vitest/coverage-v8@4.1.6)(vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)): + vitest@4.1.7(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.7)(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)): dependencies: - '@vitest/expect': 4.1.6 - '@vitest/mocker': 4.1.6(vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) - '@vitest/pretty-format': 4.1.6 - '@vitest/runner': 4.1.6 - '@vitest/snapshot': 4.1.6 - '@vitest/spy': 4.1.6 - '@vitest/utils': 4.1.6 + '@vitest/expect': 4.1.7 + '@vitest/mocker': 4.1.7(vite@8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) + '@vitest/pretty-format': 4.1.7 + '@vitest/runner': 4.1.7 + '@vitest/snapshot': 4.1.7 + '@vitest/spy': 4.1.7 + '@vitest/utils': 4.1.7 es-module-lexer: 2.1.0 expect-type: 1.3.0 magic-string: 0.30.21 @@ -12205,15 +12068,15 @@ snapshots: picomatch: 4.0.4 std-env: 4.1.0 tinybench: 2.9.0 - tinyexec: 1.1.1 + tinyexec: 1.1.2 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vite: 8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0) + vite: 8.0.14(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0) why-is-node-running: 2.3.0 optionalDependencies: - '@opentelemetry/api': 1.9.0 - '@types/node': 24.12.2 - '@vitest/coverage-v8': 4.1.6(vitest@4.1.6) + '@opentelemetry/api': 1.9.1 + '@types/node': 24.12.4 + '@vitest/coverage-v8': 4.1.7(vitest@4.1.7) transitivePeerDependencies: - msw @@ -12264,7 +12127,7 @@ snapshots: dependencies: ansi-styles: 6.2.3 string-width: 8.2.1 - strip-ansi: 7.1.2 + strip-ansi: 7.2.0 wrap-ansi@7.0.0: dependencies: @@ -12276,7 +12139,7 @@ snapshots: dependencies: ansi-styles: 6.2.3 string-width: 7.2.0 - strip-ansi: 7.1.2 + strip-ansi: 7.2.0 wrappy@1.0.2: {} @@ -12286,6 +12149,8 @@ snapshots: dependencies: is-wsl: 3.1.1 + xml-naming@0.1.0: {} + xtend@4.0.2: {} y18n@5.0.8: {} @@ -12327,11 +12192,11 @@ snapshots: zod@4.4.3: {} - zustand@4.5.7(@types/react@19.2.14)(react@19.2.6): + zustand@4.5.7(@types/react@19.2.15)(react@19.2.6): dependencies: use-sync-external-store: 1.6.0(react@19.2.6) optionalDependencies: - '@types/react': 19.2.14 + '@types/react': 19.2.15 react: 19.2.6 zwitch@2.0.4: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 04eacb8f..ae7b5b37 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -23,3 +23,4 @@ allowBuilds: better-sqlite3: true esbuild: true sharp: true +minimumReleaseAge: 10080 diff --git a/pyproject.toml b/pyproject.toml index 67d2acbb..0395de17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,10 +13,10 @@ Issues = "https://github.com/kaelio/ktx/issues" [dependency-groups] dev = [ - "pre-commit>=4.6.0", - "pytest>=9.0.2", - "pytest-cov>=7.1.0", - "ruff>=0.8.4", + "pre-commit>=4.6.0", + "pytest>=9.0.2", + "pytest-cov>=7.1.0", + "ruff>=0.8.4", ] [tool.uv] @@ -32,14 +32,14 @@ torch = { index = "pytorch-cpu" } [tool.uv.workspace] members = [ - "python/ktx-sl", - "python/ktx-daemon", + "python/ktx-sl", + "python/ktx-daemon", ] [tool.pytest.ini_options] addopts = ["--import-mode=importlib"] pythonpath = ["python/ktx-sl/tests"] testpaths = [ - "python/ktx-sl/tests", - "python/ktx-daemon/tests", + "python/ktx-sl/tests", + "python/ktx-daemon/tests", ] diff --git a/python/ktx-daemon/pyproject.toml b/python/ktx-daemon/pyproject.toml index d7168d01..0fcf8e88 100644 --- a/python/ktx-daemon/pyproject.toml +++ b/python/ktx-daemon/pyproject.toml @@ -1,23 +1,23 @@ [project] name = "ktx-daemon" -version = "0.5.0" +version = "0.9.0" description = "Portable compute package for KTX semantic-layer operations" readme = "README.md" requires-python = ">=3.13" license = "Apache-2.0" dependencies = [ - "fastapi>=0.115.0", - "ktx-sl", - "lkml>=1.3.7", - "numpy>=2.2.6", - "orjson>=3.11.4", - "pandas>=2.2.3", - "posthog>=7.0.0", - "psycopg[binary]>=3.2.0", - "pydantic>=2.9.0", - "requests>=2.32.0", - "sqlglot>=26", - "uvicorn[standard]>=0.32.0", + "fastapi>=0.136.3", + "ktx-sl", + "lkml>=1.3.7", + "numpy>=2.4.6", + "orjson>=3.11.9", + "pandas>=3.0.3", + "posthog>=7.16.1", + "psycopg[binary]>=3.3.4", + "pydantic>=2.13.4", + "requests>=2.34.2", + "sqlglot>=30", + "uvicorn[standard]>=0.48.0", ] [project.scripts] @@ -25,8 +25,8 @@ ktx-daemon = "ktx_daemon.__main__:main" [project.optional-dependencies] local-embeddings = [ - "sentence-transformers>=5.1.1", - "torch>=2.2.0", + "sentence-transformers>=5.1.1", + "torch>=2.2.0", ] [project.urls] @@ -43,8 +43,8 @@ packages = ["src/ktx_daemon"] [dependency-groups] dev = [ - "httpx>=0.28.1", - "pytest>=9.0.2", + "httpx>=0.28.1", + "pytest>=9.0.2", ] [tool.uv.sources] diff --git a/python/ktx-daemon/src/ktx_daemon/__main__.py b/python/ktx-daemon/src/ktx_daemon/__main__.py index 2fc00186..cbc2e228 100644 --- a/python/ktx-daemon/src/ktx_daemon/__main__.py +++ b/python/ktx-daemon/src/ktx_daemon/__main__.py @@ -6,6 +6,8 @@ import argparse import json import sys import time +from collections.abc import Callable +from types import TracebackType from typing import Any from pydantic import ValidationError @@ -90,6 +92,41 @@ def _read_stdin_json() -> dict[str, Any]: return parsed +def install_serve_http_exception_hooks(started_at: float) -> Callable[[], None]: + original_hook = sys.excepthook + + def hook( + exc_type: type[BaseException], + exc: BaseException, + tb: TracebackType | None, + ) -> None: + report_serve_http_crash(exc, started_at=started_at) + original_hook(exc_type, exc, tb) + + sys.excepthook = hook + + def dispose() -> None: + sys.excepthook = original_hook + + return dispose + + +def report_serve_http_crash(error: BaseException, *, started_at: float) -> None: + from ktx_daemon.telemetry import report_exception + from ktx_daemon.telemetry.daemon_lifecycle import emit_daemon_stopped_once + + report_exception( + error, + source="serve-http", + handled=False, + fatal=True, + ) + emit_daemon_stopped_once( + reason="crash", + uptime_ms=max(0, (time.perf_counter() - started_at) * 1000), + ) + + def run_http_server( *, host: str, @@ -102,15 +139,23 @@ def run_http_server( from ktx_daemon.app import create_app started_at = time.perf_counter() - uvicorn.run( - create_app( - enable_code_execution=enable_code_execution, - telemetry_started_at=started_at, - ), - host=host, - port=port, - log_level=log_level, - ) + dispose_hooks = install_serve_http_exception_hooks(started_at) + try: + try: + uvicorn.run( + create_app( + enable_code_execution=enable_code_execution, + telemetry_started_at=started_at, + ), + host=host, + port=port, + log_level=log_level, + ) + except Exception as error: + report_serve_http_crash(error, started_at=started_at) + raise + finally: + dispose_hooks() def main(argv: list[str] | None = None) -> int: @@ -169,6 +214,14 @@ def main(argv: list[str] | None = None) -> int: sys.stderr.write(f"{error}\n") return 1 except Exception as error: + from ktx_daemon.telemetry import report_exception + + report_exception( + error, + source=str(args.command), + handled=True, + fatal=False, + ) sys.stderr.write(f"{type(error).__name__}: {error}\n") return 1 diff --git a/python/ktx-daemon/src/ktx_daemon/app.py b/python/ktx-daemon/src/ktx_daemon/app.py index 7a3fa950..5860c4e4 100644 --- a/python/ktx-daemon/src/ktx_daemon/app.py +++ b/python/ktx-daemon/src/ktx_daemon/app.py @@ -10,8 +10,8 @@ from contextlib import asynccontextmanager from collections.abc import Callable from typing import Any -from fastapi import FastAPI, HTTPException -from fastapi.responses import Response +from fastapi import FastAPI, HTTPException, Request +from fastapi.responses import JSONResponse, Response from ktx_daemon import VERSION from ktx_daemon.code_execution import ( @@ -65,9 +65,11 @@ from ktx_daemon.table_identifier import ( ParseTableIdentifierBatchResponse, parse_table_identifier_response, ) -from ktx_daemon.telemetry import track_telemetry_event +from ktx_daemon.telemetry import report_exception, track_telemetry_event +from ktx_daemon.telemetry.daemon_lifecycle import emit_daemon_stopped_once logger = logging.getLogger(__name__) +CREDENTIAL_KEYS = {"url", "password", "token", "api_key", "apikey", "auth_header"} class NumpyORJSONResponse(Response): @@ -77,6 +79,36 @@ class NumpyORJSONResponse(Response): return dumps_numpy_json(content) +def _route_source(request: Request) -> str: + route = request.scope.get("route") + path = getattr(route, "path", None) + if isinstance(path, str) and path: + return f"app:{path}" + return f"app:{request.url.path}" + + +def _secret_snapshot_from_payload(value: Any) -> list[str]: + secrets: list[str] = [] + if isinstance(value, dict): + for key, child in value.items(): + normalized_key = str(key).lower() + if normalized_key in CREDENTIAL_KEYS and isinstance(child, str) and child: + secrets.append(child) + secrets.extend(_secret_snapshot_from_payload(child)) + elif isinstance(value, list): + for child in value: + secrets.extend(_secret_snapshot_from_payload(child)) + return secrets + + +async def _request_secret_snapshot(request: Request) -> list[str]: + try: + payload = await request.json() + except Exception: + return [] + return _secret_snapshot_from_payload(payload) + + def create_app( *, embedding_provider: EmbeddingProvider | None = None, @@ -104,12 +136,9 @@ def create_app( try: yield finally: - track_telemetry_event( - "daemon_stopped", - { - "reason": "request", - "uptimeMs": max(0, (clock() - started_at) * 1000), - }, + emit_daemon_stopped_once( + reason="request", + uptime_ms=max(0, (clock() - started_at) * 1000), ) app = FastAPI( @@ -119,6 +148,25 @@ def create_app( lifespan=lifespan, ) + @app.middleware("http") + async def report_unhandled_exceptions(request: Request, call_next): + redaction_secrets = await _request_secret_snapshot(request) + try: + return await call_next(request) + except Exception as error: + logger.exception("Unhandled daemon request failed: %s", error) + report_exception( + error, + source=_route_source(request), + handled=True, + fatal=False, + redaction_secrets=redaction_secrets, + ) + return JSONResponse( + status_code=500, + content={"detail": f"Daemon request failed: {error}"}, + ) + @app.get("/health") async def health() -> dict[str, str]: response = {"status": "healthy"} @@ -137,12 +185,6 @@ def create_app( except ValueError as error: logger.warning("Database introspection rejected: %s", error) raise HTTPException(status_code=400, detail=str(error)) from error - except Exception as error: - logger.exception("Database introspection failed: %s", error) - raise HTTPException( - status_code=500, - detail=f"Database introspection failed: {error}", - ) from error @app.post("/embeddings/compute", response_model=ComputeEmbeddingResponse) async def embedding_compute( @@ -156,12 +198,6 @@ def create_app( except ValueError as error: logger.warning("Embedding compute rejected: %s", error) raise HTTPException(status_code=400, detail=str(error)) from error - except Exception as error: - logger.exception("Embedding compute failed: %s", error) - raise HTTPException( - status_code=500, - detail=f"Embedding compute failed: {error}", - ) from error @app.post( "/embeddings/compute-bulk", @@ -178,12 +214,6 @@ def create_app( except ValueError as error: logger.warning("Bulk embedding compute rejected: %s", error) raise HTTPException(status_code=400, detail=str(error)) from error - except Exception as error: - logger.exception("Bulk embedding compute failed: %s", error) - raise HTTPException( - status_code=500, - detail=f"Bulk embedding compute failed: {error}", - ) from error if enable_code_execution: @@ -193,29 +223,15 @@ def create_app( response_class=NumpyORJSONResponse, ) async def code_execute(request: ExecuteCodeRequest) -> ExecuteCodeResponse: - try: - return execute_code_response( - request, - nest_api_url=None, - auth_header=None, - ) - except Exception as error: - logger.exception("Code execution failed: %s", error) - raise HTTPException( - status_code=500, - detail=f"Code execution failed: {error}", - ) from error + return execute_code_response( + request, + nest_api_url=None, + auth_header=None, + ) @app.post("/lookml/parse", response_model=ParseLookMLResponse) async def lookml_parse(request: ParseLookMLRequest) -> ParseLookMLResponse: - try: - return parse_lookml_project(request) - except Exception as error: - logger.exception("LookML parsing failed: %s", error) - raise HTTPException( - status_code=500, - detail=f"LookML parsing failed: {error}", - ) from error + return parse_lookml_project(request) @app.post( "/sql/parse-table-identifier", @@ -224,40 +240,19 @@ def create_app( async def sql_parse_table_identifier( request: ParseTableIdentifierBatchRequest, ) -> ParseTableIdentifierBatchResponse: - try: - return parse_table_identifier_response(request) - except Exception as error: - logger.exception("Table identifier parsing failed: %s", error) - raise HTTPException( - status_code=500, - detail=f"Table identifier parsing failed: {error}", - ) from error + return parse_table_identifier_response(request) @app.post("/sql/validate-read-only", response_model=ValidateReadOnlySqlResponse) async def sql_validate_read_only( request: ValidateReadOnlySqlRequest, ) -> ValidateReadOnlySqlResponse: - try: - return validate_read_only_sql_response(request) - except Exception as error: - logger.exception("SQL read-only validation failed: %s", error) - raise HTTPException( - status_code=500, - detail=f"SQL read-only validation failed: {error}", - ) from error + return validate_read_only_sql_response(request) @app.post("/sql/analyze-batch", response_model=AnalyzeSqlBatchResponse) async def sql_analyze_batch( request: AnalyzeSqlBatchRequest, ) -> AnalyzeSqlBatchResponse: - try: - return analyze_sql_batch_response(request) - except Exception as error: - logger.exception("SQL batch analysis failed: %s", error) - raise HTTPException( - status_code=500, - detail=f"SQL batch analysis failed: {error}", - ) from error + return analyze_sql_batch_response(request) @app.post( "/semantic-layer/generate-sources", response_model=GenerateSourcesResponse @@ -265,14 +260,7 @@ def create_app( async def semantic_generate_sources( request: GenerateSourcesRequest, ) -> GenerateSourcesResponse: - try: - return generate_sources_response(request) - except Exception as error: - logger.exception("Semantic source generation failed: %s", error) - raise HTTPException( - status_code=500, - detail=f"Semantic source generation failed: {error}", - ) from error + return generate_sources_response(request) @app.post("/semantic-layer/query", response_model=SemanticLayerQueryResponse) async def semantic_query( @@ -283,12 +271,6 @@ def create_app( except ValueError as error: logger.warning("Semantic query rejected: %s", error) raise HTTPException(status_code=400, detail=str(error)) from error - except Exception as error: - logger.exception("Semantic query failed: %s", error) - raise HTTPException( - status_code=500, - detail=f"Semantic layer query failed: {error}", - ) from error @app.post("/semantic-layer/validate", response_model=ValidateSourcesResponse) async def semantic_validate( diff --git a/python/ktx-daemon/src/ktx_daemon/database_introspection.py b/python/ktx-daemon/src/ktx_daemon/database_introspection.py index 82058f95..6ba84265 100644 --- a/python/ktx-daemon/src/ktx_daemon/database_introspection.py +++ b/python/ktx-daemon/src/ktx_daemon/database_introspection.py @@ -327,7 +327,7 @@ def introspect_database_response( now: NowProvider | None = None, ) -> DatabaseIntrospectionResponse: driver = _driver_name(request.driver) - if driver not in {"postgres", "postgresql"}: + if driver != "postgres": raise ValueError('database introspection supports only driver "postgres"') rows = (load_rows or _load_postgres_rows)(request) diff --git a/python/ktx-daemon/src/ktx_daemon/semantic_layer.py b/python/ktx-daemon/src/ktx_daemon/semantic_layer.py index e813575e..f58c6e39 100644 --- a/python/ktx-daemon/src/ktx_daemon/semantic_layer.py +++ b/python/ktx-daemon/src/ktx_daemon/semantic_layer.py @@ -5,7 +5,7 @@ from __future__ import annotations import time from typing import Any -from ktx_daemon.telemetry import error_class, track_telemetry_event +from ktx_daemon.telemetry import error_class, report_exception, track_telemetry_event from pydantic import BaseModel, ConfigDict, Field from semantic_layer.duplicate_check import validate_measure_duplicates from semantic_layer.engine import SemanticEngine @@ -13,7 +13,7 @@ from semantic_layer.models import QueryResult, SourceDefinition class SemanticLayerQueryRequest(BaseModel): - model_config = ConfigDict(populate_by_name=True) + model_config = ConfigDict(extra="forbid") sources: list[dict[str, Any]] query: dict[str, Any] @@ -150,6 +150,13 @@ def query_semantic_layer( track_telemetry_event( "sql_gen_completed", sql_fields, project_id=request.project_id ) + report_exception( + error, + source="semantic-query", + handled=True, + fatal=False, + project_id=request.project_id, + ) raise diff --git a/python/ktx-daemon/src/ktx_daemon/sql_analysis.py b/python/ktx-daemon/src/ktx_daemon/sql_analysis.py index ebecf83c..54f3d0e2 100644 --- a/python/ktx-daemon/src/ktx_daemon/sql_analysis.py +++ b/python/ktx-daemon/src/ktx_daemon/sql_analysis.py @@ -2,15 +2,32 @@ from __future__ import annotations import os from concurrent.futures import ProcessPoolExecutor +from dataclasses import dataclass from typing import Literal import sqlglot -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, Field from sqlglot import exp +from sqlglot.optimizer.normalize_identifiers import normalize_identifiers +from sqlglot.optimizer.qualify_tables import qualify_tables SqlAnalysisClause = Literal["select", "where", "join", "groupBy", "having", "orderBy"] +class SqlAnalysisTableRef(BaseModel): + catalog: str | None = None + db: str | None = None + name: str + + +class SqlAnalysisCatalogTable(SqlAnalysisTableRef): + columns: list[str] = Field(default_factory=list) + + +class AnalyzeSqlCatalog(BaseModel): + tables: list[SqlAnalysisCatalogTable] = Field(default_factory=list) + + class AnalyzeSqlBatchItem(BaseModel): id: str sql: str @@ -19,13 +36,12 @@ class AnalyzeSqlBatchItem(BaseModel): class AnalyzeSqlBatchRequest(BaseModel): dialect: str items: list[AnalyzeSqlBatchItem] + catalog: AnalyzeSqlCatalog | None = None max_workers: int | None = Field(default=None, ge=1, le=32) class AnalyzeSqlBatchResult(BaseModel): - model_config = ConfigDict(populate_by_name=True) - - tables_touched: list[str] = Field(default_factory=list) + tables_touched: list[SqlAnalysisTableRef] = Field(default_factory=list) columns_by_clause: dict[SqlAnalysisClause, list[str]] = Field(default_factory=dict) error: str | None = None @@ -84,17 +100,76 @@ def _ordered_unique(values: list[str]) -> list[str]: return result -def _table_ref(table: exp.Table) -> str: - parts: list[str] = [] +def _normalize_identifier(value: str | None, dialect: str) -> str | None: + if value is None: + return None + identifier = exp.to_identifier(value) + identifier.meta["is_table"] = True + normalized = normalize_identifiers(identifier, dialect=dialect) + return str(normalized.name) + + +def _normalized_ref(ref: SqlAnalysisTableRef, dialect: str) -> SqlAnalysisTableRef: + return SqlAnalysisTableRef( + catalog=_normalize_identifier(ref.catalog, dialect), + db=_normalize_identifier(ref.db, dialect), + name=_normalize_identifier(ref.name, dialect) or ref.name, + ) + + +@dataclass(frozen=True) +class _CatalogIndex: + by_full: dict[tuple[str | None, str | None, str], SqlAnalysisTableRef] + by_name: dict[str, list[SqlAnalysisTableRef]] + + +def _catalog_index( + catalog: AnalyzeSqlCatalog | None, dialect: str +) -> _CatalogIndex | None: + if catalog is None or not catalog.tables: + return None + by_full: dict[tuple[str | None, str | None, str], SqlAnalysisTableRef] = {} + by_name: dict[str, list[SqlAnalysisTableRef]] = {} + for table in catalog.tables: + ref = _normalized_ref(table, dialect) + key = (ref.catalog, ref.db, ref.name) + by_full[key] = ref + by_name.setdefault(ref.name, []).append(ref) + return _CatalogIndex(by_full=by_full, by_name=by_name) + + +def _raw_table_ref(table: exp.Table, dialect: str) -> SqlAnalysisTableRef | None: + if not table.name: + return None catalog = table.args.get("catalog") db = table.args.get("db") - if catalog is not None and getattr(catalog, "name", None): - parts.append(str(catalog.name)) - if db is not None and getattr(db, "name", None): - parts.append(str(db.name)) - if table.name: - parts.append(str(table.name)) - return ".".join(parts) + return _normalized_ref( + SqlAnalysisTableRef( + catalog=str(catalog.name) + if catalog is not None and getattr(catalog, "name", None) + else None, + db=str(db.name) if db is not None and getattr(db, "name", None) else None, + name=str(table.name), + ), + dialect, + ) + + +def _resolve_table_refs( + raw: SqlAnalysisTableRef, + catalog: _CatalogIndex | None, +) -> list[SqlAnalysisTableRef]: + if catalog is None: + return [raw] + exact = catalog.by_full.get((raw.catalog, raw.db, raw.name)) + if exact is not None: + return [exact] + if raw.db is not None: + return [raw] + matches = catalog.by_name.get(raw.name, []) + if matches: + return matches + return [SqlAnalysisTableRef(catalog=None, db=None, name=raw.name)] def _column_name(column: exp.Column) -> str: @@ -148,33 +223,48 @@ def _columns_by_clause(tree: exp.Expression) -> dict[SqlAnalysisClause, list[str return result +def _table_refs( + tree: exp.Expression, dialect: str, catalog: _CatalogIndex | None +) -> list[SqlAnalysisTableRef]: + normalized_tree = normalize_identifiers(tree, dialect=dialect) + qualified_tree = qualify_tables(normalized_tree, dialect=dialect) + cte_names = {cte.alias_or_name.lower() for cte in qualified_tree.find_all(exp.CTE)} + refs: list[SqlAnalysisTableRef] = [] + seen: set[tuple[str | None, str | None, str]] = set() + for table in qualified_tree.find_all(exp.Table): + if table.name.lower() in cte_names: + continue + raw = _raw_table_ref(table, dialect) + if raw is None: + continue + for ref in _resolve_table_refs(raw, catalog): + key = (ref.catalog, ref.db, ref.name) + if key not in seen: + seen.add(key) + refs.append(ref) + return refs + + def _analyze_one( - item_id: str, sql: str, dialect: str + item_id: str, sql: str, dialect: str, catalog: _CatalogIndex | None ) -> tuple[str, AnalyzeSqlBatchResult]: try: tree = sqlglot.parse_one(sql, read=dialect) except sqlglot.errors.SqlglotError as exc: return item_id, AnalyzeSqlBatchResult(error=str(exc)) - cte_names = {cte.alias_or_name.lower() for cte in tree.find_all(exp.CTE)} - table_refs = [ - table_ref - for table_ref in (_table_ref(table) for table in tree.find_all(exp.Table)) - if table_ref and table_ref.split(".")[-1].lower() not in cte_names - ] - return item_id, AnalyzeSqlBatchResult( - tables_touched=_ordered_unique(table_refs), + tables_touched=_table_refs(tree, dialect, catalog), columns_by_clause=_columns_by_clause(tree), error=None, ) def _analyze_payload( - payload: tuple[str, str, str], + payload: tuple[str, str, str, _CatalogIndex | None], ) -> tuple[str, AnalyzeSqlBatchResult]: - item_id, sql, dialect = payload - return _analyze_one(item_id, sql, dialect) + item_id, sql, dialect, catalog = payload + return _analyze_one(item_id, sql, dialect, catalog) def validate_read_only_sql_response( @@ -224,7 +314,8 @@ def _worker_count(request: AnalyzeSqlBatchRequest) -> int: def analyze_sql_batch_response( request: AnalyzeSqlBatchRequest, ) -> AnalyzeSqlBatchResponse: - payloads = [(item.id, item.sql, request.dialect) for item in request.items] + catalog = _catalog_index(request.catalog, request.dialect) + payloads = [(item.id, item.sql, request.dialect, catalog) for item in request.items] if _worker_count(request) == 1: analyzed = [_analyze_payload(payload) for payload in payloads] else: diff --git a/python/ktx-daemon/src/ktx_daemon/table_identifier.py b/python/ktx-daemon/src/ktx_daemon/table_identifier.py index 748f2dd8..297c25b4 100644 --- a/python/ktx-daemon/src/ktx_daemon/table_identifier.py +++ b/python/ktx-daemon/src/ktx_daemon/table_identifier.py @@ -1,9 +1,8 @@ from __future__ import annotations -from dataclasses import asdict from typing import Literal -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, Field from semantic_layer.table_identifier_parser import ( ParseTableIdentifierItem as SharedParseTableIdentifierItem, parse_table_identifier_batch, @@ -30,8 +29,6 @@ class ParseTableIdentifierBatchRequest(BaseModel): class ParsedIdentifier(BaseModel): - model_config = ConfigDict(populate_by_name=True) - ok: bool catalog: str | None = None schema_: str | None = Field(default=None, alias="schema") @@ -60,7 +57,15 @@ def parse_table_identifier_response( ) return ParseTableIdentifierBatchResponse( results={ - key: ParsedIdentifier.model_validate(asdict(value)) + key: ParsedIdentifier( + ok=value.ok, + catalog=value.catalog, + schema=value.schema_, + name=value.name, + canonical_table=value.canonical_table, + reason=value.reason, + detail=value.detail, + ) for key, value in shared_results.items() } ) diff --git a/python/ktx-daemon/src/ktx_daemon/telemetry/__init__.py b/python/ktx-daemon/src/ktx_daemon/telemetry/__init__.py index ff9cd07f..bef42338 100644 --- a/python/ktx-daemon/src/ktx_daemon/telemetry/__init__.py +++ b/python/ktx-daemon/src/ktx_daemon/telemetry/__init__.py @@ -1,5 +1,12 @@ from __future__ import annotations +from ktx_daemon.telemetry.daemon_lifecycle import emit_daemon_stopped_once from ktx_daemon.telemetry.emitter import error_class, track_telemetry_event +from ktx_daemon.telemetry.exception import report_exception -__all__ = ["error_class", "track_telemetry_event"] +__all__ = [ + "emit_daemon_stopped_once", + "error_class", + "report_exception", + "track_telemetry_event", +] diff --git a/python/ktx-daemon/src/ktx_daemon/telemetry/daemon_lifecycle.py b/python/ktx-daemon/src/ktx_daemon/telemetry/daemon_lifecycle.py new file mode 100644 index 00000000..dc635601 --- /dev/null +++ b/python/ktx-daemon/src/ktx_daemon/telemetry/daemon_lifecycle.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from typing import Literal + +from ktx_daemon.telemetry.emitter import track_telemetry_event + +StopReason = Literal["signal", "request", "crash"] + +_daemon_stop_emitted = False + + +def emit_daemon_stopped_once(*, reason: StopReason, uptime_ms: float) -> bool: + global _daemon_stop_emitted + if _daemon_stop_emitted: + return False + _daemon_stop_emitted = True + track_telemetry_event( + "daemon_stopped", + { + "reason": reason, + "uptimeMs": max(0, uptime_ms), + }, + ) + return True + + +def reset_daemon_lifecycle_for_tests() -> None: + global _daemon_stop_emitted + _daemon_stop_emitted = False diff --git a/python/ktx-daemon/src/ktx_daemon/telemetry/events.schema.json b/python/ktx-daemon/src/ktx_daemon/telemetry/events.schema.json index 13642c49..c6c3d6f8 100644 --- a/python/ktx-daemon/src/ktx_daemon/telemetry/events.schema.json +++ b/python/ktx-daemon/src/ktx_daemon/telemetry/events.schema.json @@ -26,6 +26,7 @@ "durationMs", "outcome", "errorClass", + "errorDetail", "flagsPresent", "hasProject", "projectGroupAttached" @@ -37,7 +38,8 @@ "fields": [ "step", "outcome", - "durationMs" + "durationMs", + "errorDetail" ] }, { @@ -56,6 +58,7 @@ "isDemoConnection", "outcome", "errorClass", + "errorDetail", "durationMs", "serverVersion" ] @@ -84,7 +87,8 @@ "rowsBucket", "durationMs", "outcome", - "errorClass" + "errorClass", + "errorDetail" ] }, { @@ -98,7 +102,8 @@ "declaredFkCount", "durationMs", "outcome", - "errorClass" + "errorClass", + "errorDetail" ] }, { @@ -157,7 +162,9 @@ "outcome", "durationMs", "errorClass", - "sampleRate" + "sampleRate", + "mcpClientName", + "mcpClientVersion" ] }, { @@ -199,6 +206,17 @@ "errorClass", "durationMs" ] + }, + { + "name": "query_history_filter_completed", + "description": "Emitted after the setup query-history service-account filter picker runs.", + "fields": [ + "dialect", + "consideredRoleCount", + "excludedRoleCount", + "parseFailedCount", + "outcome" + ] } ], "$defs": { @@ -294,6 +312,10 @@ "errorClass": { "type": "string" }, + "errorDetail": { + "type": "string", + "maxLength": 1000 + }, "flagsPresent": { "type": "object", "propertyNames": { @@ -365,7 +387,6 @@ "embeddings", "secrets", "databases", - "database-context-depth", "sources", "context", "agents", @@ -383,6 +404,10 @@ "durationMs": { "type": "number", "minimum": 0 + }, + "errorDetail": { + "type": "string", + "maxLength": 1000 } }, "required": [ @@ -493,6 +518,10 @@ "errorClass": { "type": "string" }, + "errorDetail": { + "type": "string", + "maxLength": 1000 + }, "durationMs": { "type": "number", "minimum": 0 @@ -672,6 +701,10 @@ }, "errorClass": { "type": "string" + }, + "errorDetail": { + "type": "string", + "maxLength": 1000 } }, "required": [ @@ -758,6 +791,10 @@ }, "errorClass": { "type": "string" + }, + "errorDetail": { + "type": "string", + "maxLength": 1000 } }, "required": [ @@ -1132,7 +1169,13 @@ }, "sampleRate": { "type": "number", - "const": 0.1 + "const": 1 + }, + "mcpClientName": { + "type": "string" + }, + "mcpClientVersion": { + "type": "string" } }, "required": [ @@ -1402,6 +1445,77 @@ "durationMs" ], "additionalProperties": false + }, + "query_history_filter_completed": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "cliVersion": { + "type": "string" + }, + "nodeVersion": { + "type": "string" + }, + "osPlatform": { + "type": "string" + }, + "osRelease": { + "type": "string" + }, + "arch": { + "type": "string" + }, + "runtime": { + "type": "string", + "enum": [ + "node", + "daemon-py" + ] + }, + "isCi": { + "type": "boolean" + }, + "dialect": { + "type": "string" + }, + "consideredRoleCount": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "excludedRoleCount": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "parseFailedCount": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "outcome": { + "type": "string", + "enum": [ + "ok", + "error" + ] + } + }, + "required": [ + "cliVersion", + "nodeVersion", + "osPlatform", + "osRelease", + "arch", + "runtime", + "isCi", + "dialect", + "consideredRoleCount", + "excludedRoleCount", + "parseFailedCount", + "outcome" + ], + "additionalProperties": false } } } diff --git a/python/ktx-daemon/src/ktx_daemon/telemetry/exception.py b/python/ktx-daemon/src/ktx_daemon/telemetry/exception.py new file mode 100644 index 00000000..00050d1c --- /dev/null +++ b/python/ktx-daemon/src/ktx_daemon/telemetry/exception.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import json +import os +import re +import sys +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +from ktx_daemon import VERSION +from ktx_daemon.telemetry.emitter import POSTHOG_HOST, POSTHOG_PROJECT_API_KEY +from ktx_daemon.telemetry.events import _common_envelope +from ktx_daemon.telemetry.identity import load_telemetry_identity + +_KTX_REPORTED_ATTR = "__ktx_posthog_exception_reported" + + +def _debug_enabled(env: Mapping[str, str]) -> bool: + return env.get("KTX_TELEMETRY_DEBUG") == "1" + + +def _host(env: Mapping[str, str]) -> str: + return env.get("KTX_TELEMETRY_ENDPOINT") or POSTHOG_HOST + + +def _redact_static(value: str) -> str: + patterns = [ + ( + r"([a-z][a-z0-9+.-]*://[^:\s/@]+:)([^@\s/]+)(@)", + r"\1[redacted]\3", + ), + (r"\b(password|pwd)=([^;&\s]+)", r"\1=[redacted]"), + (r"\bAuthorization\s*:\s*[^\r\n,;]+", "Authorization: [redacted]"), + (r"\bBearer\s+[A-Za-z0-9._~+/=-]+", "Bearer [redacted]"), + (r"\b(api[_-]?key)\s*[:=]\s*([^\s,;]+)", r"\1=[redacted]"), + ( + r"\b(KTX_[A-Z0-9_]*|[A-Z0-9_]*(?:TOKEN|SECRET))\s*[:=]\s*([^\s,;]+)", + r"\1=[redacted]", + ), + (r"([?&](?:X-Amz-Signature|X-Goog-Signature|sig)=)[^&\s]+", r"\1[redacted]"), + ] + redacted = value + for pattern, replacement in patterns: + redacted = re.sub(pattern, replacement, redacted, flags=re.IGNORECASE) + return redacted + + +def _redact_text(value: str, secrets: Sequence[str]) -> str: + redacted = value + for secret in secrets: + if secret: + redacted = redacted.replace(secret, "[redacted]") + return _redact_static(redacted) + + +def _clone_exception(exception: BaseException, secrets: Sequence[str]) -> BaseException: + redacted_args = [_redact_text(str(arg), secrets) for arg in exception.args] + try: + cloned = type(exception)(*redacted_args) + except Exception: + cloned = RuntimeError(_redact_text(str(exception), secrets)) + cloned.__traceback__ = exception.__traceback__ + cloned.__cause__ = ( + _clone_exception(exception.__cause__, secrets) if exception.__cause__ else None + ) + cloned.__context__ = ( + _clone_exception(exception.__context__, secrets) + if exception.__context__ + else None + ) + return cloned + + +def _should_skip_as_reported(exception: BaseException) -> bool: + if getattr(exception, _KTX_REPORTED_ATTR, False): + return True + try: + setattr(exception, _KTX_REPORTED_ATTR, True) + except Exception: + return False + return False + + +def _properties(*, source: str, handled: bool, fatal: bool) -> dict[str, Any]: + return { + **_common_envelope(), + "daemonVersion": os.environ.get("KTX_DAEMON_VERSION", VERSION), + "source": source, + "handled": handled, + "fatal": fatal, + } + + +def report_exception( + exception: BaseException, + *, + source: str, + handled: bool, + fatal: bool, + project_id: str | None = None, + home_dir: Path | None = None, + env: Mapping[str, str] | None = None, + redaction_secrets: Sequence[str] | None = None, +) -> None: + source_env = env if env is not None else os.environ + try: + identity = load_telemetry_identity(home_dir=home_dir, env=source_env) + if not identity.enabled or not identity.install_id: + return + + if _should_skip_as_reported(exception): + return + + properties = _properties(source=source, handled=handled, fatal=fatal) + groups = {"project": project_id} if project_id else None + safe_exception = _clone_exception(exception, redaction_secrets or []) + + if _debug_enabled(source_env): + sys.stderr.write( + "[telemetry-exception] " + + json.dumps( + { + "distinctId": identity.install_id, + "message": str(safe_exception), + "properties": properties, + "groups": groups, + }, + sort_keys=True, + ) + + "\n" + ) + return + + if not POSTHOG_PROJECT_API_KEY.strip() or not _host(source_env).strip(): + return + + from posthog import Posthog + + client = Posthog( + POSTHOG_PROJECT_API_KEY, + host=_host(source_env), + flush_at=1, + flush_interval=0, + sync_mode=True, + timeout=1, + ) + client.capture_exception( + safe_exception, + distinct_id=identity.install_id, + properties=properties, + groups=groups, + ) + client.shutdown() + except Exception: + return diff --git a/python/ktx-daemon/tests/test_app.py b/python/ktx-daemon/tests/test_app.py index 9960daaf..fffc2899 100644 --- a/python/ktx-daemon/tests/test_app.py +++ b/python/ktx-daemon/tests/test_app.py @@ -87,8 +87,10 @@ def test_app_lifespan_emits_daemon_lifecycle_debug_events( monkeypatch, capsys, ) -> None: + from ktx_daemon.telemetry.daemon_lifecycle import reset_daemon_lifecycle_for_tests from ktx_daemon.telemetry.identity import reset_identity_cache + reset_daemon_lifecycle_for_tests() reset_identity_cache() identity_path = tmp_path / ".ktx" / "telemetry.json" identity_path.parent.mkdir(parents=True) @@ -368,7 +370,9 @@ def test_sql_analyze_batch_endpoint_returns_per_item_results() -> None: assert response.status_code == 200 body = response.json() - assert body["results"]["orders"]["tables_touched"] == ["public.orders"] + assert body["results"]["orders"]["tables_touched"] == [ + {"catalog": None, "db": "public", "name": "orders"} + ] assert body["results"]["orders"]["columns_by_clause"] == { "select": ["status"], "where": ["created_at"], diff --git a/python/ktx-daemon/tests/test_database_introspection.py b/python/ktx-daemon/tests/test_database_introspection.py index 0a018046..b0fb7a5b 100644 --- a/python/ktx-daemon/tests/test_database_introspection.py +++ b/python/ktx-daemon/tests/test_database_introspection.py @@ -138,6 +138,18 @@ def test_introspect_database_response_rejects_non_postgres_driver() -> None: ) +def test_introspect_database_response_rejects_legacy_postgresql_driver() -> None: + with pytest.raises(ValueError, match='supports only driver "postgres"'): + introspect_database_response( + DatabaseIntrospectionRequest( + connection_id="warehouse", + driver="postgresql", + url="postgresql://readonly@example.test/warehouse", + ), + load_rows=lambda request: DatabaseIntrospectionRows([], [], []), + ) + + def test_database_introspection_request_rejects_empty_schema_list() -> None: with pytest.raises(ValueError, match="at least one schema"): DatabaseIntrospectionRequest( diff --git a/python/ktx-daemon/tests/test_exception_payload.py b/python/ktx-daemon/tests/test_exception_payload.py new file mode 100644 index 00000000..3198b08f --- /dev/null +++ b/python/ktx-daemon/tests/test_exception_payload.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import gzip +import json +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from typing import Any + +from ktx_daemon.telemetry.identity import reset_identity_cache + + +class CaptureHandler(BaseHTTPRequestHandler): + payloads: list[dict[str, Any]] = [] + + def do_POST(self) -> None: + length = int(self.headers.get("content-length", "0")) + raw = self.rfile.read(length) + if self.headers.get("content-encoding") == "gzip": + raw = gzip.decompress(raw) + self.payloads.append(json.loads(raw.decode("utf-8"))) + self.send_response(200) + self.send_header("content-type", "application/json") + self.end_headers() + self.wfile.write(b"{}") + + def log_message(self, _format: str, *_args: object) -> None: + return + + +def write_identity(home: Path) -> None: + target = home / ".ktx" / "telemetry.json" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text( + json.dumps( + { + "installId": "00000000-0000-4000-8000-000000000000", + "enabled": True, + "createdAt": "2026-06-05T00:00:00.000Z", + } + ) + + "\n", + encoding="utf-8", + ) + + +def find_exception_event(payloads: list[dict[str, Any]]) -> dict[str, Any]: + for payload in payloads: + batch = payload.get("batch") + events = batch if isinstance(batch, list) else [payload] + for event in events: + if isinstance(event, dict) and event.get("event") == "$exception": + return event + raise AssertionError(f"No $exception payload found: {payloads}") + + +def test_prepared_python_exception_payload_groups_and_redacts(tmp_path: Path) -> None: + from ktx_daemon.telemetry.exception import report_exception + + reset_identity_cache() + write_identity(tmp_path) + CaptureHandler.payloads.clear() + server = HTTPServer(("127.0.0.1", 0), CaptureHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + snapshot_secret = "-".join(["plain", "secret", "value"]) + db_password = "-".join(["db", "url", "secret"]) + auth_token = "".join(["abc", "123"]) + report_exception( + RuntimeError( + f"{snapshot_secret} postgres://svc:{db_password}@db.example.test/analytics " + f"Authorization: Basic {auth_token}" + ), + source="database-introspect", + handled=True, + fatal=False, + project_id="a" * 64, + home_dir=tmp_path, + env={"KTX_TELEMETRY_ENDPOINT": f"http://127.0.0.1:{server.server_port}"}, + redaction_secrets=[snapshot_secret], + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + event = find_exception_event(CaptureHandler.payloads) + properties = event["properties"] + assert event.get("$groups") == {"project": "a" * 64} or properties.get( + "$groups" + ) == {"project": "a" * 64} + serialized = json.dumps(properties.get("$exception_list", [])) + assert "[redacted]" in serialized + assert snapshot_secret not in serialized + assert db_password not in serialized + assert auth_token not in serialized + forbidden_keys = { + "argv", + "args", + "env", + "environment", + "sql", + "query", + "prompt", + "mcpArguments", + "tableName", + "schemaName", + "columnName", + "databaseUrl", + "connectionString", + "url", + "password", + "token", + "apiKey", + "authorization", + } + assert forbidden_keys.isdisjoint(properties.keys()) diff --git a/python/ktx-daemon/tests/test_exception_telemetry.py b/python/ktx-daemon/tests/test_exception_telemetry.py new file mode 100644 index 00000000..43da007d --- /dev/null +++ b/python/ktx-daemon/tests/test_exception_telemetry.py @@ -0,0 +1,601 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from ktx_daemon.telemetry.identity import reset_identity_cache + + +class FakePosthog: + captures: list[dict[str, Any]] = [] + shutdowns = 0 + + def __init__(self, *_args: Any, **_kwargs: Any) -> None: + pass + + def capture_exception( + self, + exception: BaseException, + *, + distinct_id: str, + properties: dict[str, Any], + groups: dict[str, str] | None = None, + ) -> None: + self.captures.append( + { + "exception": exception, + "distinct_id": distinct_id, + "properties": properties, + "groups": groups, + } + ) + + def shutdown(self) -> None: + type(self).shutdowns += 1 + + +def write_identity(home: Path, *, enabled: bool = True) -> None: + target = home / ".ktx" / "telemetry.json" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text( + json.dumps( + { + "installId": "00000000-0000-4000-8000-000000000000", + "enabled": enabled, + "createdAt": "2026-06-05T00:00:00.000Z", + } + ) + + "\n", + encoding="utf-8", + ) + + +def test_report_exception_respects_disabled_gate(tmp_path: Path, monkeypatch) -> None: + from ktx_daemon.telemetry.exception import report_exception + + reset_identity_cache() + write_identity(tmp_path) + monkeypatch.setenv("KTX_TELEMETRY_DISABLED", "1") + FakePosthog.captures.clear() + monkeypatch.setattr("posthog.Posthog", FakePosthog) + + report_exception( + RuntimeError("boom"), + source="semantic-query", + handled=True, + fatal=False, + home_dir=tmp_path, + env={"KTX_TELEMETRY_DISABLED": "1"}, + ) + + assert FakePosthog.captures == [] + + +def test_report_exception_sends_groups_and_properties( + tmp_path: Path, monkeypatch +) -> None: + from ktx_daemon.telemetry.exception import report_exception + + reset_identity_cache() + write_identity(tmp_path) + FakePosthog.captures.clear() + monkeypatch.setattr("posthog.Posthog", FakePosthog) + + report_exception( + RuntimeError("boom"), + source="semantic-query", + handled=True, + fatal=False, + project_id="a" * 64, + home_dir=tmp_path, + env={}, + ) + + assert FakePosthog.captures == [ + { + "exception": FakePosthog.captures[0]["exception"], + "distinct_id": "00000000-0000-4000-8000-000000000000", + "properties": FakePosthog.captures[0]["properties"], + "groups": {"project": "a" * 64}, + } + ] + assert FakePosthog.captures[0]["properties"]["source"] == "semantic-query" + assert FakePosthog.captures[0]["properties"]["handled"] is True + assert FakePosthog.captures[0]["properties"]["fatal"] is False + assert FakePosthog.captures[0]["properties"]["runtime"] == "daemon-py" + + +def test_report_exception_debug_prints_without_sending(tmp_path: Path, capsys) -> None: + from ktx_daemon.telemetry.exception import report_exception + + reset_identity_cache() + write_identity(tmp_path) + FakePosthog.captures.clear() + + report_exception( + RuntimeError("debug boom"), + source="app:/health", + handled=True, + fatal=False, + home_dir=tmp_path, + env={"KTX_TELEMETRY_DEBUG": "1"}, + ) + + captured = capsys.readouterr() + assert "[telemetry-exception]" in captured.err + assert '"source": "app:/health"' in captured.err + assert FakePosthog.captures == [] + + +def test_report_exception_redacts_snapshot_and_static_patterns( + tmp_path: Path, monkeypatch +) -> None: + from ktx_daemon.telemetry.exception import report_exception + + reset_identity_cache() + write_identity(tmp_path) + FakePosthog.captures.clear() + monkeypatch.setattr("posthog.Posthog", FakePosthog) + error = RuntimeError("dsn has plain-secret and password=hunter2") + error.__cause__ = ValueError("Authorization: Bearer token-123") + + report_exception( + error, + source="database-introspect", + handled=True, + fatal=False, + home_dir=tmp_path, + env={}, + redaction_secrets=["plain-secret"], + ) + + sent = FakePosthog.captures[0]["exception"] + assert "[redacted]" in str(sent) + assert "plain-secret" not in str(sent) + assert "hunter2" not in str(sent) + assert "token-123" not in str(sent.__cause__) + + +def test_report_exception_does_not_discover_env_values_without_snapshot( + tmp_path: Path, monkeypatch +) -> None: + from ktx_daemon.telemetry.exception import report_exception + + reset_identity_cache() + write_identity(tmp_path) + FakePosthog.captures.clear() + monkeypatch.setenv("KTX_FAKE_SECRET", "plain-secret-without-pattern") + monkeypatch.setattr("posthog.Posthog", FakePosthog) + + report_exception( + RuntimeError("plain-secret-without-pattern"), + source="sys.excepthook", + handled=False, + fatal=True, + home_dir=tmp_path, + env={}, + ) + + assert "plain-secret-without-pattern" in str(FakePosthog.captures[0]["exception"]) + + +def test_route_derived_boundary_reports_new_throwing_route(monkeypatch) -> None: + from fastapi import FastAPI + from fastapi.testclient import TestClient + from ktx_daemon.app import create_app + + reports: list[dict[str, object]] = [] + + def fake_report(exception: BaseException, **kwargs: object) -> None: + reports.append({"exception": exception, **kwargs}) + + monkeypatch.setattr("ktx_daemon.app.report_exception", fake_report) + app: FastAPI = create_app() + + @app.get("/new-throwing-route") + async def new_throwing_route() -> dict[str, str]: + raise RuntimeError("route boom") + + client = TestClient(app, raise_server_exceptions=False) + response = client.get("/new-throwing-route") + + assert response.status_code == 500 + assert reports + assert reports[0]["source"] in {"app:/new-throwing-route", "app:new_throwing_route"} + assert reports[0]["handled"] is True + assert reports[0]["fatal"] is False + + +def test_route_derived_boundary_covers_existing_validate_route(monkeypatch) -> None: + from fastapi.testclient import TestClient + from ktx_daemon import app as app_module + + reports: list[dict[str, object]] = [] + + def fake_report(exception: BaseException, **kwargs: object) -> None: + reports.append({"exception": exception, **kwargs}) + + monkeypatch.setattr( + app_module, + "validate_semantic_layer", + lambda _request: (_ for _ in ()).throw(RuntimeError("validate boom")), + ) + monkeypatch.setattr(app_module, "report_exception", fake_report) + + client = TestClient(app_module.create_app(), raise_server_exceptions=False) + response = client.post("/semantic-layer/validate", json={"sources": []}) + + assert response.status_code == 500 + assert reports + assert reports[0]["source"] in { + "app:/semantic-layer/validate", + "app:semantic_validate", + } + + +def test_daemon_stopped_clean_shutdown_emits_request_once(monkeypatch) -> None: + from ktx_daemon.telemetry.daemon_lifecycle import ( + emit_daemon_stopped_once, + reset_daemon_lifecycle_for_tests, + ) + + events: list[tuple[str, dict[str, object]]] = [] + monkeypatch.setattr( + "ktx_daemon.telemetry.daemon_lifecycle.track_telemetry_event", + lambda name, fields: events.append((name, fields)), + ) + reset_daemon_lifecycle_for_tests() + + emit_daemon_stopped_once(reason="request", uptime_ms=1) + emit_daemon_stopped_once(reason="request", uptime_ms=2) + + assert events == [("daemon_stopped", {"reason": "request", "uptimeMs": 1})] + + +def test_daemon_stopped_crash_wins_over_request(monkeypatch) -> None: + from ktx_daemon.telemetry.daemon_lifecycle import ( + emit_daemon_stopped_once, + reset_daemon_lifecycle_for_tests, + ) + + events: list[tuple[str, dict[str, object]]] = [] + monkeypatch.setattr( + "ktx_daemon.telemetry.daemon_lifecycle.track_telemetry_event", + lambda name, fields: events.append((name, fields)), + ) + reset_daemon_lifecycle_for_tests() + + emit_daemon_stopped_once(reason="crash", uptime_ms=3) + emit_daemon_stopped_once(reason="request", uptime_ms=4) + + assert events == [("daemon_stopped", {"reason": "crash", "uptimeMs": 3})] + + +def test_report_exception_dedupes_same_exception_object( + tmp_path: Path, monkeypatch +) -> None: + from ktx_daemon.telemetry.exception import report_exception + + reset_identity_cache() + write_identity(tmp_path) + FakePosthog.captures.clear() + monkeypatch.setattr("posthog.Posthog", FakePosthog) + error = RuntimeError("same object") + + report_exception( + error, + source="semantic-query", + handled=True, + fatal=False, + home_dir=tmp_path, + env={}, + ) + report_exception( + error, + source="app:/semantic-layer/query", + handled=True, + fatal=False, + home_dir=tmp_path, + env={}, + ) + + assert len(FakePosthog.captures) == 1 + + +def test_report_exception_redacts_url_userinfo_and_authorization( + tmp_path: Path, monkeypatch +) -> None: + from ktx_daemon.telemetry.exception import report_exception + + reset_identity_cache() + write_identity(tmp_path) + FakePosthog.captures.clear() + monkeypatch.setattr("posthog.Posthog", FakePosthog) + + db_password = ["db", "url", "secret"] + auth_token = ["abc", "123"] + report_exception( + RuntimeError( + "connect postgres://svc:" + + "-".join(db_password) + + "@db.example.test/analytics Authorization: Basic " + + "".join(auth_token) + ), + source="database-introspect", + handled=True, + fatal=False, + home_dir=tmp_path, + env={}, + ) + + sent = str(FakePosthog.captures[0]["exception"]) + assert "postgres://svc:[redacted]@db.example.test/analytics" in sent + assert "Authorization: [redacted]" in sent + assert "-".join(db_password) not in sent + assert "".join(auth_token) not in sent + + +def test_report_exception_falls_back_when_exception_type_cannot_be_reconstructed( + tmp_path: Path, monkeypatch +) -> None: + from ktx_daemon.telemetry.exception import report_exception + + class KeywordOnlyException(Exception): + def __init__(self, *, message: str) -> None: + super().__init__(message) + + reset_identity_cache() + write_identity(tmp_path) + FakePosthog.captures.clear() + monkeypatch.setattr("posthog.Posthog", FakePosthog) + + report_exception( + KeywordOnlyException(message="custom secret-value"), + source="app:/custom", + handled=True, + fatal=False, + home_dir=tmp_path, + env={}, + redaction_secrets=["secret-value"], + ) + + assert len(FakePosthog.captures) == 1 + sent = FakePosthog.captures[0]["exception"] + assert "[redacted]" in str(sent) + assert "secret-value" not in str(sent) + + +def test_report_exception_redacts_every_static_pattern_and_leaves_benign_text( + tmp_path: Path, monkeypatch +) -> None: + from ktx_daemon.telemetry.exception import report_exception + + reset_identity_cache() + write_identity(tmp_path) + FakePosthog.captures.clear() + monkeypatch.setattr("posthog.Posthog", FakePosthog) + + cases = [ + ("dsn password=hunter2", "hunter2", "password=[redacted]"), + ("dsn pwd=swordfish", "swordfish", "pwd=[redacted]"), + ("Authorization: Basic abc123", "abc123", "Authorization: [redacted]"), + ("Authorization: Bearer token-123", "token-123", "Authorization: [redacted]"), + ("Bearer standalone-token", "standalone-token", "Bearer [redacted]"), + ("api_key=sk-live-secret", "sk-live-secret", "api_key=[redacted]"), + ("api-key: sk-dash-secret", "sk-dash-secret", "api-key=[redacted]"), + ( + "KTX_PROVIDER_TOKEN=ktx-secret", + "ktx-secret", + "KTX_PROVIDER_TOKEN=[redacted]", + ), + ( + "REFRESH_SECRET: refresh-secret", + "refresh-secret", + "REFRESH_SECRET=[redacted]", + ), + ( + "https://s3.example.test/file?X-Amz-Signature=aws-secret&ok=1", + "aws-secret", + "X-Amz-Signature=[redacted]", + ), + ( + "https://storage.example.test/file?X-Goog-Signature=goog-secret&ok=1", + "goog-secret", + "X-Goog-Signature=[redacted]", + ), + ( + "https://cdn.example.test/file?sig=signed-secret&ok=1", + "signed-secret", + "sig=[redacted]", + ), + ( + "postgres://svc:url-password@db.example.test/analytics", # pragma: allowlist secret + "url-password", + "postgres://svc:[redacted]@db.example.test/analytics", + ), + ] + + for message, leaked, expected in cases: + report_exception( + RuntimeError(message), + source="database-introspect", + handled=True, + fatal=False, + home_dir=tmp_path, + env={}, + ) + sent = str(FakePosthog.captures[-1]["exception"]) + assert expected in sent + assert leaked not in sent + + report_exception( + RuntimeError("token bucket metrics and passwordless auth are benign"), + source="database-introspect", + handled=True, + fatal=False, + home_dir=tmp_path, + env={}, + ) + assert str(FakePosthog.captures[-1]["exception"]) == ( + "token bucket metrics and passwordless auth are benign" + ) + + +def test_route_derived_boundary_covers_existing_health_route(monkeypatch) -> None: + from fastapi.testclient import TestClient + from ktx_daemon import app as app_module + + reports: list[dict[str, object]] = [] + + def fake_report(exception: BaseException, **kwargs: object) -> None: + reports.append({"exception": exception, **kwargs}) + + class BrokenEnviron(dict[str, str]): + def get(self, key: str, default: str | None = None) -> str | None: + if key == "KTX_DAEMON_VERSION": + raise RuntimeError("health boom") + return default + + monkeypatch.setattr(app_module.os, "environ", BrokenEnviron()) + monkeypatch.setattr(app_module, "report_exception", fake_report) + + client = TestClient(app_module.create_app(), raise_server_exceptions=False) + response = client.get("/health") + + assert response.status_code == 500 + assert reports + assert reports[0]["source"] == "app:/health" + assert reports[0]["handled"] is True + assert reports[0]["fatal"] is False + + +def test_route_boundary_passes_request_scoped_database_secrets(monkeypatch) -> None: + from fastapi.testclient import TestClient + from ktx_daemon import app as app_module + + reports: list[dict[str, object]] = [] + + def fake_report(exception: BaseException, **kwargs: object) -> None: + reports.append({"exception": exception, **kwargs}) + + monkeypatch.setattr( + app_module, + "introspect_database_response", + lambda _request: (_ for _ in ()).throw(RuntimeError("db-url-secret")), + ) + monkeypatch.setattr(app_module, "report_exception", fake_report) + + client = TestClient(app_module.create_app(), raise_server_exceptions=False) + response = client.post( + "/database/introspect", + json={ + "connection_id": "warehouse", + "url": "postgres://svc:db-url-secret@db.example.test/analytics", # pragma: allowlist secret + "password": "db-password-secret", # pragma: allowlist secret + }, + ) + + assert response.status_code == 500 + assert reports + assert ( + reports[0]["redaction_secrets"] + == [ + "postgres://svc:db-url-secret@db.example.test/analytics", # pragma: allowlist secret + "db-password-secret", # pragma: allowlist secret + ] + ) + + +def test_serve_http_run_crash_reports_exception_and_crash_stop(monkeypatch) -> None: + import sys + + from ktx_daemon import __main__ as main_module + + reports: list[dict[str, object]] = [] + stops: list[dict[str, object]] = [] + + def fake_report(exception: BaseException, **kwargs: object) -> None: + reports.append({"exception": exception, **kwargs}) + + def fake_stop(*, reason: str, uptime_ms: float) -> bool: + stops.append({"reason": reason, "uptimeMs": uptime_ms}) + return True + + class FakeUvicorn: + @staticmethod + def run(*_args: object, **_kwargs: object) -> None: + raise RuntimeError("uvicorn crash") + + monkeypatch.setitem(sys.modules, "uvicorn", FakeUvicorn) + monkeypatch.setattr("ktx_daemon.telemetry.report_exception", fake_report) + monkeypatch.setattr( + "ktx_daemon.telemetry.daemon_lifecycle.emit_daemon_stopped_once", + fake_stop, + ) + + try: + main_module.run_http_server( + host="127.0.0.1", + port=9999, + log_level="info", + enable_code_execution=False, + ) + except RuntimeError as error: + assert str(error) == "uvicorn crash" + else: + raise AssertionError("run_http_server did not re-raise the crash") + + assert reports + assert reports[0]["source"] == "serve-http" + assert reports[0]["handled"] is False + assert reports[0]["fatal"] is True + assert stops and stops[0]["reason"] == "crash" + + +def test_one_shot_command_reports_without_excepthook_or_daemon_stopped( + monkeypatch, +) -> None: + import sys + + from ktx_daemon import __main__ as daemon_main + + original_hook = sys.excepthook + reports: list[dict[str, object]] = [] + stops: list[dict[str, object]] = [] + + def fake_report(exception: BaseException, **kwargs: object) -> None: + reports.append({"exception": exception, **kwargs}) + + def fake_stop(*, reason: str, uptime_ms: float) -> bool: + stops.append({"reason": reason, "uptimeMs": uptime_ms}) + return True + + monkeypatch.setattr( + daemon_main, + "_read_stdin_json", + lambda: { + "connection_id": "warehouse", + "driver": "postgres", + "url": "postgresql://readonly@example.test/warehouse", + "schemas": ["public"], + }, + ) + monkeypatch.setattr( + daemon_main, + "introspect_database_response", + lambda _request: (_ for _ in ()).throw(RuntimeError("one-shot boom")), + ) + monkeypatch.setattr("ktx_daemon.telemetry.report_exception", fake_report) + monkeypatch.setattr( + "ktx_daemon.telemetry.daemon_lifecycle.emit_daemon_stopped_once", + fake_stop, + ) + + assert daemon_main.main(["database-introspect"]) == 1 + assert sys.excepthook is original_hook + assert stops == [] + assert reports + assert reports[0]["source"] == "database-introspect" + assert reports[0]["handled"] is True + assert reports[0]["fatal"] is False diff --git a/python/ktx-daemon/tests/test_semantic_layer.py b/python/ktx-daemon/tests/test_semantic_layer.py index 8ebb7ad8..72040df9 100644 --- a/python/ktx-daemon/tests/test_semantic_layer.py +++ b/python/ktx-daemon/tests/test_semantic_layer.py @@ -3,6 +3,8 @@ from __future__ import annotations import json from pathlib import Path +import pytest + from ktx_daemon.semantic_layer import ( SemanticLayerQueryRequest, ValidateSourcesRequest, @@ -95,6 +97,43 @@ def test_query_semantic_layer_emits_plan_and_sql_debug_events( assert "public.orders" not in captured.err +def test_query_semantic_layer_reports_exception(monkeypatch) -> None: + from ktx_daemon import semantic_layer as semantic_layer_module + + reports: list[dict[str, object]] = [] + + def fake_report(exception: BaseException, **kwargs: object) -> None: + reports.append({"exception": exception, **kwargs}) + + monkeypatch.setattr(semantic_layer_module, "report_exception", fake_report) + + with pytest.raises(ValueError): + query_semantic_layer( + SemanticLayerQueryRequest( + sources=[ORDERS_SOURCE, ORDERS_SOURCE], + dialect="postgres", + projectId="a" * 64, + query={"measures": ["orders.order_count"]}, + ) + ) + + assert reports + assert reports[0]["source"] == "semantic-query" + assert reports[0]["handled"] is True + assert reports[0]["fatal"] is False + assert reports[0]["project_id"] == "a" * 64 + + +def test_semantic_layer_request_rejects_project_id_field_name() -> None: + with pytest.raises(ValueError): + SemanticLayerQueryRequest( + sources=[], + dialect="postgres", + project_id="a" * 64, + query={"measures": ["orders.order_count"]}, + ) + + def test_validate_semantic_layer_reports_duplicate_measure_names() -> None: invalid_source = { **ORDERS_SOURCE, diff --git a/python/ktx-daemon/tests/test_sql_analysis.py b/python/ktx-daemon/tests/test_sql_analysis.py index 855d16fd..2fb3970a 100644 --- a/python/ktx-daemon/tests/test_sql_analysis.py +++ b/python/ktx-daemon/tests/test_sql_analysis.py @@ -32,7 +32,10 @@ def test_analyze_sql_batch_extracts_tables_and_clause_columns() -> None: result = response.results["orders_by_customer"] assert result.error is None - assert result.tables_touched == ["public.orders", "public.customers"] + assert [item.model_dump() for item in result.tables_touched] == [ + {"catalog": None, "db": "public", "name": "orders"}, + {"catalog": None, "db": "public", "name": "customers"}, + ] assert result.columns_by_clause == { "select": ["status"], "where": ["created_at"], @@ -56,6 +59,114 @@ def test_analyze_sql_batch_returns_per_item_parse_errors() -> None: assert result.error is not None +def test_analyze_sql_batch_qualifies_bare_table_from_catalog() -> None: + response = analyze_sql_batch_response( + AnalyzeSqlBatchRequest( + dialect="postgres", + catalog={ + "tables": [ + { + "catalog": None, + "db": "orbit_raw", + "name": "accounts", + "columns": ["id"], + }, + { + "catalog": None, + "db": "orbit_analytics", + "name": "orders", + "columns": ["id"], + }, + ] + }, + items=[AnalyzeSqlBatchItem(id="bare", sql="select id from accounts")], + max_workers=1, + ) + ) + + assert [item.model_dump() for item in response.results["bare"].tables_touched] == [ + {"catalog": None, "db": "orbit_raw", "name": "accounts"} + ] + + +def test_analyze_sql_batch_returns_all_ambiguous_modeled_matches() -> None: + response = analyze_sql_batch_response( + AnalyzeSqlBatchRequest( + dialect="postgres", + catalog={ + "tables": [ + { + "catalog": None, + "db": "orbit_raw", + "name": "events", + "columns": ["id"], + }, + { + "catalog": None, + "db": "orbit_analytics", + "name": "events", + "columns": ["id"], + }, + ] + }, + items=[AnalyzeSqlBatchItem(id="ambiguous", sql="select id from events")], + max_workers=1, + ) + ) + + assert [ + item.model_dump() for item in response.results["ambiguous"].tables_touched + ] == [ + {"catalog": None, "db": "orbit_raw", "name": "events"}, + {"catalog": None, "db": "orbit_analytics", "name": "events"}, + ] + + +def test_analyze_sql_batch_leaves_unresolved_bare_refs_unqualified() -> None: + response = analyze_sql_batch_response( + AnalyzeSqlBatchRequest( + dialect="postgres", + catalog={ + "tables": [{"catalog": None, "db": "orbit_raw", "name": "accounts"}] + }, + items=[AnalyzeSqlBatchItem(id="missing", sql="select * from invoices")], + max_workers=1, + ) + ) + + assert [ + item.model_dump() for item in response.results["missing"].tables_touched + ] == [{"catalog": None, "db": None, "name": "invoices"}] + + +def test_analyze_sql_batch_returns_bigquery_project_dataset_table_refs() -> None: + response = analyze_sql_batch_response( + AnalyzeSqlBatchRequest( + dialect="bigquery", + catalog={ + "tables": [ + { + "catalog": "demo-project", + "db": "orbit_analytics", + "name": "orders", + } + ] + }, + items=[ + AnalyzeSqlBatchItem( + id="bq", + sql="select * from `demo-project.orbit_analytics.orders`", + ) + ], + max_workers=1, + ) + ) + + assert [item.model_dump() for item in response.results["bq"].tables_touched] == [ + {"catalog": "demo-project", "db": "orbit_analytics", "name": "orders"} + ] + + def test_columns_from_nodes_ignores_non_expression_clause_values() -> None: assert _columns_from_nodes([True, False, None]) == [] diff --git a/python/ktx-daemon/tests/test_telemetry_schema_sync.py b/python/ktx-daemon/tests/test_telemetry_schema_sync.py index 6f2ba634..0cc822f9 100644 --- a/python/ktx-daemon/tests/test_telemetry_schema_sync.py +++ b/python/ktx-daemon/tests/test_telemetry_schema_sync.py @@ -36,4 +36,5 @@ def test_python_schema_copy_matches_node_schema() -> None: "daemon_stopped", "sl_plan_completed", "sql_gen_completed", + "query_history_filter_completed", ] diff --git a/python/ktx-sl/AGENTS.md b/python/ktx-sl/AGENTS.md index b9b54f18..beba3036 100644 --- a/python/ktx-sl/AGENTS.md +++ b/python/ktx-sl/AGENTS.md @@ -59,7 +59,7 @@ uv run python -m semantic_layer.cli --model /tmp/model.yaml \ -q '{"measures":["orders.revenue"],"dimensions":["customers.segment"]}' --suggest ``` -### 3. Test fan-out / chasm traps +### 3. Test fanout / chasm traps Add multiple measure sources that fan out from a shared dimension hub: diff --git a/python/ktx-sl/pyproject.toml b/python/ktx-sl/pyproject.toml index 69dfd2d9..aaf65265 100644 --- a/python/ktx-sl/pyproject.toml +++ b/python/ktx-sl/pyproject.toml @@ -1,14 +1,14 @@ [project] name = "ktx-sl" -version = "0.5.0" +version = "0.9.0" description = "Agent-first semantic layer engine with aggregate locality" readme = "README.md" requires-python = ">=3.13" license = "Apache-2.0" dependencies = [ - "sqlglot>=26", - "pydantic>=2", - "pyyaml>=6", + "sqlglot>=30", + "pydantic>=2", + "pyyaml>=6", ] [project.urls] @@ -18,13 +18,13 @@ Issues = "https://github.com/kaelio/ktx/issues" [project.optional-dependencies] dev = [ - "pytest>=8", - "pytest-cov", - "ruff", - "pre-commit", + "pytest>=8", + "pytest-cov", + "ruff", + "pre-commit", ] tpch = [ - "duckdb>=1.0", + "duckdb>=1.0", ] [tool.pytest.ini_options] @@ -40,9 +40,9 @@ branch = true show_missing = true skip_empty = true exclude_lines = [ - "pragma: no cover", - "if __name__ == .__main__.", - "if TYPE_CHECKING:", + "pragma: no cover", + "if __name__ == .__main__.", + "if TYPE_CHECKING:", ] [build-system] @@ -54,6 +54,6 @@ packages = ["semantic_layer"] [dependency-groups] dev = [ - "pytest>=9.0.2", - "pytest-cov>=7.1.0", + "pytest>=9.0.2", + "pytest-cov>=7.1.0", ] diff --git a/python/ktx-sl/semantic_layer/cli.py b/python/ktx-sl/semantic_layer/cli.py index a2782f38..d1bacaa8 100644 --- a/python/ktx-sl/semantic_layer/cli.py +++ b/python/ktx-sl/semantic_layer/cli.py @@ -160,7 +160,7 @@ def print_plan(plan) -> None: print(" Joins:") for jp in plan.join_paths: print(f" {jp}") - print(f" Fan-out: {plan.fan_out_description}") + print(f" Fanout: {plan.fan_out_description}") if plan.aggregate_locality: print(" Locality:") for al in plan.aggregate_locality: diff --git a/python/ktx-sl/semantic_layer/generator.py b/python/ktx-sl/semantic_layer/generator.py index a5979299..5309018f 100644 --- a/python/ktx-sl/semantic_layer/generator.py +++ b/python/ktx-sl/semantic_layer/generator.py @@ -92,7 +92,7 @@ class SqlGenerator: return "WITH " + source_header + ",\n" + rest return "WITH " + source_header + "\n" + outer_transpiled - # ── Path A: Simple (no fan-out) ──────────────────────────────────── + # ── Path A: Simple (no fanout) ──────────────────────────────────── def _generate_simple( self, plan: ResolvedPlan, sources: dict[str, SourceDefinition] @@ -216,7 +216,7 @@ class SqlGenerator: shared_dim_aliases = shared_dim_aliases or set() shared_dims = [dk for dk in all_dim_keys if dk["alias"] in shared_dim_aliases] - # Validate grain consistency: asymmetric dims cause FULL JOIN fan-out + # Validate grain consistency: asymmetric dims cause FULL JOIN fanout if len(plan.measure_groups) > 1: for group in plan.measure_groups: cte_dim_aliases = { diff --git a/python/ktx-sl/semantic_layer/loader.py b/python/ktx-sl/semantic_layer/loader.py index 55a3a0ee..1f505fe5 100644 --- a/python/ktx-sl/semantic_layer/loader.py +++ b/python/ktx-sl/semantic_layer/loader.py @@ -201,7 +201,7 @@ class SourceLoader: name = col.get("name") if name in base_by_name: raise ValueError( - f"column '{name}' in columns patches a manifest column on '{base.name}' — move it to 'column_overrides:'" + f"column '{name}' in columns patches a manifest column on '{base.name}' - move it to 'column_overrides:'" ) source.columns.append(SourceColumn(**col)) diff --git a/python/ktx-sl/semantic_layer/planner.py b/python/ktx-sl/semantic_layer/planner.py index bfd1d74f..e5ebf02e 100644 --- a/python/ktx-sl/semantic_layer/planner.py +++ b/python/ktx-sl/semantic_layer/planner.py @@ -107,7 +107,7 @@ class QueryPlanner: for e in tree.edges ] - # 8. Detect fan-out / chasm trap + # 8. Detect fanout / chasm trap has_fan_out, measure_groups, fan_out_desc, locality_descs = ( self._detect_fan_out(measures, dimensions, tree, filters=query.filters) ) @@ -937,7 +937,7 @@ class QueryPlanner: filters: list[str] | None = None, ) -> tuple[bool, list[MeasureGroup], str, list[str]]: """ - Detect fan-out and chasm traps. Group measures by source. + Detect fanout and chasm traps. Group measures by source. If multiple measure sources exist, each needs its own pre-aggregation CTE. Also checks filter sources — a filter forcing a one_to_many join from the measure source is an error (cannot be safely pre-aggregated). @@ -991,7 +991,7 @@ class QueryPlanner: if len(groups) <= 1: # Single measure group: check the path FROM measure source TO dimension sources. - # Only flag fan-out if those specific paths have one_to_many edges. + # Only flag fanout if those specific paths have one_to_many edges. if groups: source_name = next(iter(groups)) source_actual = self.graph.alias_map.get(source_name, source_name) @@ -999,7 +999,7 @@ class QueryPlanner: for dim_src in dim_sources: if dim_src == source_name: continue - # Skip alias siblings (same underlying source — no fan-out) + # Skip alias siblings (same underlying source — no fanout) dim_actual = self.graph.alias_map.get(dim_src, dim_src) if dim_actual == source_actual: continue @@ -1008,7 +1008,7 @@ class QueryPlanner: has_o2m = True break - # Also check filter sources for one_to_many fan-out + # Also check filter sources for one_to_many fanout if not has_o2m: for filter_src in filter_sources - dim_sources - {source_name}: filter_actual = self.graph.alias_map.get(filter_src, filter_src) @@ -1019,7 +1019,7 @@ class QueryPlanner: raise ValueError( f"Filter on '{filter_src}' requires a one_to_many join " f"from measure source '{source_name}', which would cause " - f"incorrect aggregation (fan-out). Consider rewriting the " + f"incorrect aggregation (fanout). Consider rewriting the " f"filter as a subquery or adding the filter source as a " f"dimension source." ) @@ -1033,10 +1033,10 @@ class QueryPlanner: return ( True, measure_groups, - f"Fan-out detected: one_to_many edges from {source_name} to dimensions", + f"Fanout detected: one_to_many edges from {source_name} to dimensions", [f"Pre-aggregate {source_name} measures before joining"], ) - return False, [], "No fan-out", [] + return False, [], "No fanout", [] # Multiple measure sources. Only merge groups that are provably row-safe # (alias siblings or pure one_to_one chains). many_to_one chains are not @@ -1048,7 +1048,7 @@ class QueryPlanner: # All measure sources are on the same safe join chain if merged_groups: mg_name, mg_measures = next(iter(merged_groups.items())) - # Still check if there's fan-out to dimension sources + # Still check if there's fanout to dimension sources has_o2m = False for dim_src in dim_sources: if dim_src == mg_name: @@ -1061,10 +1061,10 @@ class QueryPlanner: return ( True, [MeasureGroup(source_name=mg_name, measures=mg_measures)], - f"Fan-out detected: one_to_many edges from {mg_name} to dimensions", + f"Fanout detected: one_to_many edges from {mg_name} to dimensions", [f"Pre-aggregate {mg_name} measures before joining"], ) - return False, [], "No fan-out", [] + return False, [], "No fanout", [] # True chasm trap — independent measure sources that can't be safely merged. # Before building groups, validate that all filter sources are reachable diff --git a/python/ktx-sl/tests/test_aggregate_locality.py b/python/ktx-sl/tests/test_aggregate_locality.py index 9080d608..7b120ef2 100644 --- a/python/ktx-sl/tests/test_aggregate_locality.py +++ b/python/ktx-sl/tests/test_aggregate_locality.py @@ -1,4 +1,4 @@ -"""Dedicated tests for aggregate locality (fan-out/chasm trap correctness).""" +"""Dedicated tests for aggregate locality (fanout/chasm trap correctness).""" import pytest import sqlglot @@ -213,7 +213,7 @@ class TestNoFanOut: sqlglot.parse(sql) def test_m2o_join_no_ctes(self, ecommerce_sources): - """orders → customers is m2o, no fan-out.""" + """orders → customers is m2o, no fanout.""" graph = JoinGraph(ecommerce_sources) graph.build() planner = QueryPlanner(ecommerce_sources, graph) @@ -540,7 +540,7 @@ class TestFactSideDimensionsInChasm: """LIMIT 1: Fact-side dimensions in chasm trap (local to one CTE only).""" def test_fact_side_dimension_in_chasm_raises_error(self): - """Asymmetric dim from fact_a only → raises error (would cause FULL JOIN fan-out).""" + """Asymmetric dim from fact_a only → raises error (would cause FULL JOIN fanout).""" hub = SourceDefinition( name="hub", table="public.hub", @@ -977,7 +977,7 @@ class TestBug13_FalseChasm_AliasAggregate: dimensions=["billing_customer.name", "shipping_customer.name"], ) plan = planner.plan(query) - assert not plan.has_fan_out, "Should not detect fan-out between alias siblings" + assert not plan.has_fan_out, "Should not detect fanout between alias siblings" sql = gen.generate(plan, sources) sqlglot.parse(sql) diff --git a/python/ktx-sl/tests/test_coverage_gaps.py b/python/ktx-sl/tests/test_coverage_gaps.py index eea91d2f..2c3a9b9b 100644 --- a/python/ktx-sl/tests/test_coverage_gaps.py +++ b/python/ktx-sl/tests/test_coverage_gaps.py @@ -305,12 +305,12 @@ class TestPredefinedMeasureDeps: assert "GROUP BY" in sql.upper() -# ── Planner: fan-out with one_to_many to dimension sources (lines 595-643) ── +# ── Planner: fanout with one_to_many to dimension sources (lines 595-643) ── class TestFanOutEdgeCases: def test_single_source_fan_out_to_dimension(self): - """Measure source with one_to_many to dimension should trigger fan-out.""" + """Measure source with one_to_many to dimension should trigger fanout.""" hub = SourceDefinition( name="hub", table="public.hub", diff --git a/python/ktx-sl/tests/test_generator.py b/python/ktx-sl/tests/test_generator.py index 9ef147ea..5b5b6894 100644 --- a/python/ktx-sl/tests/test_generator.py +++ b/python/ktx-sl/tests/test_generator.py @@ -89,10 +89,10 @@ class TestCrossSourceM2O: class TestFanOut: - """Test 3: Fan-out (aggregate locality).""" + """Test 3: Fanout (aggregate locality).""" def test_orders_by_region_no_fanout(self, planner, generator, ecommerce_sources): - """orders → customers → regions is all m2o. No fan-out needed.""" + """orders → customers → regions is all m2o. No fanout needed.""" sql = generate_sql( planner, generator, diff --git a/python/ktx-sl/tests/test_planner.py b/python/ktx-sl/tests/test_planner.py index dd6483b7..43c4c488 100644 --- a/python/ktx-sl/tests/test_planner.py +++ b/python/ktx-sl/tests/test_planner.py @@ -200,12 +200,12 @@ class TestFanOutDetection: class TestFanOutSingleSource: - """Fan-out when a single measure source has o2m path to dimension source.""" + """Fanout when a single measure source has o2m path to dimension source.""" def test_reverse_path_fan_out(self): - """Querying from customers (dimension) with measures from orders triggers fan-out + """Querying from customers (dimension) with measures from orders triggers fanout when the path from the measure source (orders) to the dimension source (customers) - is m2o — so no fan-out. But reversed: measure on customers, dim on orders.""" + is m2o — so no fanout. But reversed: measure on customers, dim on orders.""" customers = SourceDefinition( name="customers", table="t", @@ -248,7 +248,7 @@ class TestFanOutSingleSource: assert plan.has_fan_out def test_m2o_multi_hop_no_fan_out(self, planner): - """orders → customers → regions is all m2o. No fan-out.""" + """orders → customers → regions is all m2o. No fanout.""" query = SemanticQuery( measures=["sum(orders.amount)"], dimensions=["regions.name"], @@ -1116,7 +1116,7 @@ class TestDerivedMeasureEdgeCases: assert_valid_sql(result.sql) -# ── From test_edge_cases.py: filter fan-out detection ──────────────── +# ── From test_edge_cases.py: filter fanout detection ──────────────── class TestFilterFanOutDetection: diff --git a/release-policy.json b/release-policy.json index 0acaaeb5..33774673 100644 --- a/release-policy.json +++ b/release-policy.json @@ -19,7 +19,7 @@ }, "publishedPackageSmoke": { "packageName": "@kaelio/ktx", - "version": "0.5.0", + "version": "0.9.0", "registry": null }, "runtimeInstaller": { diff --git a/scripts/anti-fixture-conditional.test.mjs b/scripts/anti-fixture-conditional.test.mjs index b4a3241c..44728c48 100644 --- a/scripts/anti-fixture-conditional.test.mjs +++ b/scripts/anti-fixture-conditional.test.mjs @@ -19,7 +19,7 @@ const RELATIONSHIP_RUNTIME_SOURCES = Object.freeze([ ]); async function checkedInFixtureIds() { - const fixtureRoot = new URL('packages/cli/src/test/fixtures/relationship-benchmarks/', KTX_ROOT); + const fixtureRoot = new URL('packages/cli/test/fixtures/relationship-benchmarks/', KTX_ROOT); const entries = await readdir(fixtureRoot, { withFileTypes: true }); return entries .filter((entry) => entry.isDirectory()) diff --git a/scripts/build-benchmark-snapshot.test.mjs b/scripts/build-benchmark-snapshot.test.mjs index 6e3f5189..acea6ff9 100644 --- a/scripts/build-benchmark-snapshot.test.mjs +++ b/scripts/build-benchmark-snapshot.test.mjs @@ -257,7 +257,7 @@ describe('buildBenchmarkSnapshot', () => { assert.equal( packageJson.scripts['relationships:benchmarks:test'], - 'KTX_RUN_RELATIONSHIP_BENCHMARKS=1 vitest run src/context/scan/relationship-benchmarks.test.ts', + 'KTX_RUN_RELATIONSHIP_BENCHMARKS=1 vitest run test/context/scan/relationship-benchmarks.test.ts', ); }); }); diff --git a/scripts/check-boundaries.mjs b/scripts/check-boundaries.mjs index 5766ac78..b47d0db0 100644 --- a/scripts/check-boundaries.mjs +++ b/scripts/check-boundaries.mjs @@ -5,15 +5,6 @@ import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; const codeExtensions = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.py']); -const runtimeAssetPatterns = [/^packages\/cli\/src\/prompts\/.+\.md$/, /^packages\/cli\/src\/skills\/.+\.md$/]; -const identifierSkipPrefixes = ['docs/', 'docs-site/', 'examples/', 'python/ktx-sl/plans/', 'python/ktx-sl/openspec/']; -const identifierAllowPatterns = [ - /^packages\/cli\/src\/(?:index|managed-local-embeddings|managed-python-command|managed-python-daemon|managed-python-runtime|release-version|runtime)(?:\.test)?\.ts$/, - /^python\/ktx-daemon\/src\/ktx_daemon\/__init__\.py$/, - /^scripts\/(?:build-python-runtime-wheel|local-embeddings-runtime-smoke|package-artifacts|public-npm-release-metadata|published-package-smoke|release-readiness)(?:\.test)?\.mjs$/, - /^scripts\/semantic-release-config\.cjs$/, -]; -const forbiddenIdentifierTerms = ['kae' + 'lio', 'Kae' + 'lio', 'KAE' + 'LIO_']; const appImportPatterns = [ { @@ -68,6 +59,13 @@ const contextProductionLlmBoundaryPatterns = [ }, ]; +const concreteDialectImportPatterns = [ + /from\s+['"][^'"]*connectors\/[^'"]+\/dialect\.js['"]/, + /import\s*\(\s*['"][^'"]*connectors\/[^'"]+\/dialect\.js['"]\s*\)/, + /from\s+['"]\.\/dialect\.js['"]/, + /import\s*\(\s*['"]\.\/dialect\.js['"]\s*\)/, +]; + function normalizePath(filePath) { return filePath.split(path.sep).join('/'); } @@ -76,10 +74,6 @@ function isCodeSource(relativePath) { return codeExtensions.has(path.extname(relativePath)); } -function isRuntimeAsset(relativePath) { - return runtimeAssetPatterns.some((pattern) => pattern.test(relativePath)); -} - function scansForAppImports(relativePath) { return isCodeSource(relativePath); } @@ -99,16 +93,11 @@ function scansForContextProductionLlmBoundaries(relativePath) { return scansForLlmBoundaries(relativePath) && !isTestSource(relativePath); } -function scansForForbiddenIdentifiers(relativePath) { - return (isCodeSource(relativePath) && !isTestSource(relativePath)) || isRuntimeAsset(relativePath); -} - -function skipsIdentifierScan(relativePath) { - return identifierSkipPrefixes.some((prefix) => relativePath.startsWith(prefix)); -} - -function allowsForbiddenIdentifier(relativePath) { - return identifierAllowPatterns.some((pattern) => pattern.test(relativePath)); +function scansForConcreteDialectImportBoundaries(relativePath) { + return ( + relativePath.startsWith('packages/cli/src/context/scan/') || + /^packages\/cli\/src\/connectors\/[^/]+\/connector\.ts$/.test(relativePath) + ); } export function scanFileContent(relativePath, content) { @@ -151,17 +140,14 @@ export function scanFileContent(relativePath, content) { } } - if ( - scansForForbiddenIdentifiers(normalizedPath) && - !skipsIdentifierScan(normalizedPath) && - !allowsForbiddenIdentifier(normalizedPath) - ) { - for (const term of forbiddenIdentifierTerms) { - if (content.includes(term)) { + if (scansForConcreteDialectImportBoundaries(normalizedPath)) { + for (const pattern of concreteDialectImportPatterns) { + if (pattern.test(content)) { violations.push({ file: normalizedPath, - kind: 'identifier', - message: `Forbidden product identifier "${term}"`, + kind: 'dialect-boundary', + message: + 'Forbidden concrete connector dialect import; use getDialectForDriver() from context/connections/dialects.ts', }); } } diff --git a/scripts/check-boundaries.test.mjs b/scripts/check-boundaries.test.mjs index 25cd0f85..e6dd4bb8 100644 --- a/scripts/check-boundaries.test.mjs +++ b/scripts/check-boundaries.test.mjs @@ -3,14 +3,6 @@ import { describe, it } from 'node:test'; import { scanFileContent } from './check-boundaries.mjs'; -function productName() { - return ['Kae', 'lio'].join(''); -} - -function lowerProductName() { - return ['kae', 'lio'].join(''); -} - describe('scanFileContent', () => { it('rejects source imports from application directories', () => { const serverAlias = '@' + 'server/contracts'; @@ -27,64 +19,6 @@ describe('scanFileContent', () => { ); }); - it('rejects forbidden product identifiers in code source files', () => { - const violations = scanFileContent('packages/cli/src/context/index.ts', `export const owner = '${lowerProductName()}';`); - - assert.equal(violations.length, 1); - assert.equal(violations[0]?.kind, 'identifier'); - }); - - it('rejects forbidden product identifiers in shipped runtime prompt assets', () => { - const violations = scanFileContent( - 'packages/cli/src/prompts/memory_agent_bundle_ingest_work_unit.md', - `Write output for ${productName()}.`, - ); - - assert.equal(violations.length, 1); - assert.equal(violations[0]?.kind, 'identifier'); - assert.equal(violations[0]?.file, 'packages/cli/src/prompts/memory_agent_bundle_ingest_work_unit.md'); - }); - - it('rejects forbidden product identifiers in shipped runtime skill assets', () => { - const violations = scanFileContent( - 'packages/cli/src/skills/metabase_ingest/SKILL.md', - `Use ${productName()} project conventions.`, - ); - - assert.equal(violations.length, 1); - assert.equal(violations[0]?.kind, 'identifier'); - assert.equal(violations[0]?.file, 'packages/cli/src/skills/metabase_ingest/SKILL.md'); - }); - - it('allows product identifiers in docs, examples, and transition metadata', () => { - const name = productName(); - - assert.equal(scanFileContent('docs/transition.md', name).length, 0); - assert.equal(scanFileContent('examples/transition.md', name).length, 0); - assert.equal(scanFileContent('python/ktx-sl/plans/brainstorm.md', name).length, 0); - assert.equal(scanFileContent('python/ktx-sl/openspec/specs/semantic-layer/spec.md', name).length, 0); - }); - - it('allows product identifiers in test fixtures', () => { - const name = lowerProductName(); - - assert.equal(scanFileContent('packages/cli/src/setup.test.ts', `project: ${name}-dev`).length, 0); - assert.equal(scanFileContent('packages/cli/src/context/ingest/importer.test.ts', `email: system@${name}.dev`).length, 0); - assert.equal(scanFileContent('python/ktx-daemon/tests/test_package.py', `${name}-ktx`).length, 0); - }); - - it('allows public package identifiers in release packaging and managed runtime source', () => { - const name = lowerProductName(); - - assert.equal(scanFileContent('scripts/local-embeddings-runtime-smoke.mjs', `@${name}/ktx`).length, 0); - assert.equal(scanFileContent('scripts/package-artifacts.test.mjs', `${name}-ktx`).length, 0); - assert.equal(scanFileContent('scripts/public-npm-release-metadata.mjs', `@${name}/ktx`).length, 0); - assert.equal(scanFileContent('scripts/semantic-release-config.cjs', `${name}-ktx-`).length, 0); - assert.equal(scanFileContent('packages/cli/src/release-version.ts', `@${name}/ktx`).length, 0); - assert.equal(scanFileContent('packages/cli/src/managed-python-runtime.ts', `${name}_ktx`).length, 0); - assert.equal(scanFileContent('python/ktx-daemon/src/ktx_daemon/__init__.py', `${name}-ktx`).length, 0); - }); - it('allows clean source files and clean runtime prompt assets', () => { assert.deepEqual( scanFileContent('packages/cli/src/context/index.ts', "export const packageName = 'ktx';"), @@ -112,6 +46,43 @@ describe('scanFileContent', () => { ); }); + it('rejects concrete connector dialect imports from scan workflow and connector classes', () => { + const violations = [ + ...scanFileContent( + 'packages/cli/src/context/scan/relationship-profiling.ts', + "import { KtxPostgresDialect } from '../../connectors/postgres/dialect.js';", + ), + ...scanFileContent( + 'packages/cli/src/connectors/postgres/connector.ts', + "import { KtxPostgresDialect } from './dialect.js';", + ), + ]; + + assert.deepEqual( + violations.map((violation) => violation.kind), + ['dialect-boundary', 'dialect-boundary'], + ); + assert.equal( + violations[0]?.message, + 'Forbidden concrete connector dialect import; use getDialectForDriver() from context/connections/dialects.ts', + ); + + assert.deepEqual( + scanFileContent( + 'packages/cli/src/context/connections/dialects.ts', + "import { KtxPostgresDialect } from '../../connectors/postgres/dialect.js';", + ), + [], + ); + assert.deepEqual( + scanFileContent( + 'packages/cli/test/connectors/postgres/dialect.test.ts', + "import { KtxPostgresDialect } from './dialect.js';", + ), + [], + ); + }); + it('rejects old KTX LLM port declarations in context', () => { const violations = [ ...scanFileContent('packages/cli/src/context/agent/agent-runner.service.ts', 'export interface LlmProviderPort {}'), @@ -150,7 +121,7 @@ describe('scanFileContent', () => { assert.deepEqual( scanFileContent( - 'packages/cli/src/context/ingest/page-triage/page-triage.service.test.ts', + 'packages/cli/test/context/ingest/page-triage/page-triage.service.test.ts', "const model = this.deps.llmProvider.getModelByName('test-model');", ), [], diff --git a/scripts/codex-backend-live-smoke.mjs b/scripts/codex-backend-live-smoke.mjs new file mode 100644 index 00000000..7793fefc --- /dev/null +++ b/scripts/codex-backend-live-smoke.mjs @@ -0,0 +1,160 @@ +import { execFile } from 'node:child_process'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); +const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)); +const ROOT_DIR = resolve(SCRIPT_DIR, '..'); +const OPT_IN_MESSAGE = + 'Set KTX_RUN_CODEX_BACKEND_SMOKE=1 or pass --force to run the Codex backend live smoke.'; + +export function codexBackendSmokeOptIn(env = process.env, args = process.argv.slice(2)) { + if (env.KTX_RUN_CODEX_BACKEND_SMOKE === '1' || args.includes('--force')) { + return { run: true }; + } + return { run: false, message: OPT_IN_MESSAGE }; +} + +async function run(command, args, options = {}) { + process.stdout.write(`$ ${command} ${args.join(' ')}\n`); + try { + const result = await execFileAsync(command, args, { + cwd: options.cwd ?? ROOT_DIR, + env: { ...process.env, ...(options.env ?? {}) }, + encoding: 'utf8', + maxBuffer: 1024 * 1024 * 20, + timeout: options.timeoutMs ?? 300_000, + }); + if (result.stdout) { + process.stdout.write(result.stdout); + } + if (result.stderr) { + process.stderr.write(result.stderr); + } + return { code: 0, stdout: result.stdout, stderr: result.stderr }; + } catch (error) { + const stdout = typeof error.stdout === 'string' ? error.stdout : ''; + const stderr = typeof error.stderr === 'string' ? error.stderr : error.message; + if (stdout) { + process.stdout.write(stdout); + } + if (stderr) { + process.stderr.write(stderr); + } + return { + code: typeof error.code === 'number' ? error.code : 1, + stdout, + stderr, + }; + } +} + +function requireSuccess(label, result) { + if (result.code !== 0) { + throw new Error(`${label} failed with code ${result.code}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`); + } +} + +async function runSetupSmoke(projectDir) { + const result = await run( + 'node', + [ + join(ROOT_DIR, 'packages/cli/dist/bin.js'), + 'setup', + '--project-dir', + projectDir, + '--llm-backend', + 'codex', + '--llm-model', + 'gpt-5.3-codex', + '--no-input', + '--yes', + '--skip-databases', + '--skip-sources', + '--skip-agents', + ], + { timeoutMs: 600_000 }, + ); + requireSuccess('ktx setup codex backend', result); + if (!result.stdout.includes('LLM ready: yes (codex, gpt-5.3-codex)')) { + throw new Error(`setup did not report Codex LLM readiness\nstdout:\n${result.stdout}`); + } +} + +async function runRuntimeSmoke(projectDir) { + const runtimeUrl = pathToFileURL(join(ROOT_DIR, 'packages/cli/dist/context/llm/codex-runtime.js')).href; + const zodUrl = pathToFileURL(join(ROOT_DIR, 'packages/cli/node_modules/zod/index.js')).href; + const { CodexKtxLlmRuntime } = await import(runtimeUrl); + const { z } = await import(zodUrl); + const runtime = new CodexKtxLlmRuntime({ + projectDir, + modelSlots: { default: 'gpt-5.3-codex' }, + }); + + const text = await runtime.generateText({ + role: 'default', + prompt: 'Reply with exactly: ktx_codex_text_ok', + }); + if (text.trim() !== 'ktx_codex_text_ok') { + throw new Error(`Codex text smoke returned unexpected text: ${text}`); + } + + let toolCalls = 0; + const loop = await runtime.runAgentLoop({ + modelRole: 'default', + systemPrompt: 'You must use available tools when the user asks for a tool result.', + userPrompt: + 'Call the echo_value tool with {"value":"ktx_codex_tool_ok"}, then finish after the tool returns.', + toolSet: { + echo_value: { + name: 'echo_value', + description: 'Return the provided value as markdown.', + inputSchema: z.object({ value: z.string() }), + execute: async (input) => { + toolCalls += 1; + return { markdown: `echo:${input.value}` }; + }, + }, + }, + stepBudget: 4, + telemetryTags: {}, + }); + + if (loop.stopReason !== 'natural') { + throw new Error(`Codex tool smoke stopped with ${loop.stopReason}: ${loop.error?.message ?? 'no error'}`); + } + if (toolCalls !== 1) { + throw new Error(`Expected Codex to call echo_value exactly once, got ${toolCalls}`); + } +} + +export async function runCodexBackendLiveSmoke() { + const projectDir = await mkdtemp(join(tmpdir(), 'ktx-codex-backend-smoke-')); + try { + requireSuccess( + 'ktx build', + await run('pnpm', ['--filter', '@kaelio/ktx', 'run', 'build'], { timeoutMs: 600_000 }), + ); + await runSetupSmoke(projectDir); + await runRuntimeSmoke(projectDir); + process.stdout.write(`Codex backend live smoke passed in ${projectDir}\n`); + } finally { + await rm(projectDir, { recursive: true, force: true }); + } +} + +async function main() { + const optIn = codexBackendSmokeOptIn(); + if (!optIn.run) { + process.stdout.write(`${optIn.message}\n`); + return; + } + await runCodexBackendLiveSmoke(); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + await main(); +} diff --git a/scripts/codex-backend-live-smoke.test.mjs b/scripts/codex-backend-live-smoke.test.mjs new file mode 100644 index 00000000..8d8c051f --- /dev/null +++ b/scripts/codex-backend-live-smoke.test.mjs @@ -0,0 +1,18 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { codexBackendSmokeOptIn } from './codex-backend-live-smoke.mjs'; + +test('codex backend smoke stays disabled by default', () => { + assert.deepEqual(codexBackendSmokeOptIn({}, []), { + run: false, + message: 'Set KTX_RUN_CODEX_BACKEND_SMOKE=1 or pass --force to run the Codex backend live smoke.', + }); +}); + +test('codex backend smoke runs with env opt-in', () => { + assert.deepEqual(codexBackendSmokeOptIn({ KTX_RUN_CODEX_BACKEND_SMOKE: '1' }, []), { run: true }); +}); + +test('codex backend smoke runs with force flag', () => { + assert.deepEqual(codexBackendSmokeOptIn({}, ['--force']), { run: true }); +}); diff --git a/scripts/examples-docs.test.mjs b/scripts/examples-docs.test.mjs index 41b6d346..2ea9ce27 100644 --- a/scripts/examples-docs.test.mjs +++ b/scripts/examples-docs.test.mjs @@ -255,8 +255,9 @@ describe('standalone example docs', () => { assert.match(reviewingContext, /ktx ingest --all --no-input/); assert.match(quickstart, /schema context/); assert.match(primarySources, /context:\n queryHistory:/); - assert.match(rootReadme, /`ktx ingest ` \| Build context for one connection/); - assert.match(quickstart, /Databases:\n warehouse: deep context complete/); + assert.match(rootReadme, /`ktx ingest` \| Build context for every configured connection/); + assert.doesNotMatch(rootReadme, /`ktx ingest `/); + assert.match(quickstart, /Databases:\n warehouse: database context complete/); assert.match(quickstart, /Databases configured: yes \(warehouse\)/); assert.match(setupReference, /Databases configured: yes \(postgres-warehouse\)/); assert.doesNotMatch(rootReadme, new RegExp(['Primary sources', 'configured'].join(' '))); @@ -309,10 +310,10 @@ describe('standalone example docs', () => { it('runs the example smoke in the cli smoke script', async () => { const packageJson = JSON.parse(await readText('packages/cli/package.json')); - assert.match(packageJson.scripts.smoke, /src\/standalone-smoke\.test\.ts/); - assert.match(packageJson.scripts.smoke, /src\/example-smoke\.test\.ts/); - assert.match(packageJson.scripts.test, /--exclude src\/standalone-smoke\.test\.ts/); - assert.match(packageJson.scripts.test, /--exclude src\/example-smoke\.test\.ts/); + assert.match(packageJson.scripts.smoke, /test\/standalone-smoke\.test\.ts/); + assert.match(packageJson.scripts.smoke, /test\/example-smoke\.test\.ts/); + assert.match(packageJson.scripts.test, /--exclude test\/standalone-smoke\.test\.ts/); + assert.match(packageJson.scripts.test, /--exclude test\/example-smoke\.test\.ts/); }); it('documents daemon HTTP database, source generation, LookML, embedding, and code execution support', async () => { diff --git a/scripts/installed-live-database-smoke.mjs b/scripts/installed-live-database-smoke.mjs index a11e38d2..20bad6b5 100644 --- a/scripts/installed-live-database-smoke.mjs +++ b/scripts/installed-live-database-smoke.mjs @@ -106,7 +106,6 @@ export function buildLiveDatabaseIngestArgs(projectDir, _databaseIntrospectionUr connectionId, '--project-dir', projectDir, - '--fast', '--no-input', ]; } @@ -152,20 +151,20 @@ function requireSuccess(label, result) { } } +function requireFailure(label, result) { + if (result.code === 0) { + throw new Error( + `${label} unexpectedly succeeded\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + function requireOutput(label, result, pattern) { if (!pattern.test(result.stdout)) { throw new Error(`${label} output did not match ${pattern}\nstdout:\n${result.stdout}`); } } -function getRunId(stdout) { - const match = stdout.match(/^Run: (.+)$/m); - if (!match) { - throw new Error(`ingest output did not include a run id\nstdout:\n${stdout}`); - } - return match[1]; -} - async function requireDocker() { const result = await run('docker', ['info'], { timeout: 20_000 }); if (result.code !== 0) { @@ -310,13 +309,17 @@ async function main() { env: managedRuntimeEnv(cleanInstallDir), timeout: 120_000, }); - requireSuccess('ktx ingest warehouse --fast', ingestRun); - requireOutput('ktx ingest warehouse --fast', ingestRun, /Ingest finished/); - requireOutput('ktx ingest warehouse --fast', ingestRun, /Database schema/); + // ktx ingest now always builds enriched context and requires a configured + // model and embeddings. This smoke project has neither, so the database + // target fails the enrichment-readiness preflight before any work runs. + // This still exercises the packaged binary, daemon startup, and the live + // database connection end to end. + requireFailure('ktx ingest warehouse', ingestRun); + requireOutput('ktx ingest warehouse', ingestRun, /Ingest finished with partial failures/); + requireOutput('ktx ingest warehouse', ingestRun, /enrichment is not configured/); - const runId = getRunId(ingestRun.stdout); await assertPathExists(join(projectDir, '.ktx', 'db.sqlite'), 'SQLite local ingest state'); - process.stdout.write(`Installed live-database artifact smoke passed: ${runId}\n`); + process.stdout.write('Installed live-database artifact smoke passed: enrichment-readiness guard verified\n'); } finally { if (daemonStarted && cleanInstallDir) { await stopDaemon(cleanInstallDir); diff --git a/scripts/installed-live-database-smoke.test.mjs b/scripts/installed-live-database-smoke.test.mjs index ef618725..2ddeed5d 100644 --- a/scripts/installed-live-database-smoke.test.mjs +++ b/scripts/installed-live-database-smoke.test.mjs @@ -100,7 +100,6 @@ describe('installed live-database artifact smoke helpers', () => { 'warehouse', '--project-dir', '/tmp/project', - '--fast', '--no-input', ]); diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index 627850b4..e9ab5e9a 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -28,6 +28,20 @@ export const NPM_ARTIFACT_PACKAGES = [{ name: PUBLIC_NPM_PACKAGE_NAME, packageRo export const CLI_PYTHON_ASSET_MANIFEST = 'manifest.json'; +function pnpmCommand(args) { + if (process.platform === 'win32') { + return { + command: 'cmd.exe', + args: ['/d', '/s', '/c', 'pnpm', ...args], + }; + } + + return { + command: 'pnpm', + args, + }; +} + function scriptRootDir() { return resolve(dirname(fileURLToPath(import.meta.url)), '..'); } @@ -70,8 +84,7 @@ export function packageArtifactLayout(rootDir = scriptRootDir(), version = publi export function buildArtifactCommands(layout) { return [ { - command: 'pnpm', - args: ['--filter', PUBLIC_NPM_PACKAGE_NAME, 'run', 'build'], + ...pnpmCommand(['--filter', PUBLIC_NPM_PACKAGE_NAME, 'run', 'build']), cwd: layout.rootDir, }, { @@ -80,8 +93,7 @@ export function buildArtifactCommands(layout) { cwd: layout.rootDir, }, { - command: 'pnpm', - args: ['pack', '--out', layout.cliTarball], + ...pnpmCommand(['pack', '--out', layout.cliTarball]), cwd: join(layout.rootDir, 'packages', 'cli'), }, ]; @@ -460,11 +472,19 @@ import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { DatabaseSync } from 'node:sqlite'; +import { setTimeout as delay } from 'node:timers/promises'; import { promisify } from 'node:util'; const execFileAsync = promisify(execFile); const require = createRequire(import.meta.url); +function pnpmCommand(args) { + if (process.platform === 'win32') { + return { command: 'cmd.exe', args: ['/d', '/s', '/c', 'pnpm', ...args] }; + } + return { command: 'pnpm', args }; +} + async function run(command, args, options = {}) { process.stdout.write('$ ' + command + ' ' + args.join(' ') + '\\n'); try { @@ -492,22 +512,16 @@ function requireSuccess(label, result) { assert.equal(result.stderr, '', label + ' wrote unexpected stderr'); } -function requireSuccessWithProjectStderr(label, result, projectDir) { - assert.equal( - result.code, - 0, - label + ' failed with code ' + result.code + '\\nstdout:\\n' + result.stdout + '\\nstderr:\\n' + result.stderr, - ); - assert.equal(result.stderr, 'Project: ' + projectDir + '\\n', label + ' wrote unexpected stderr'); -} - function requireExitCodeWithProjectStderr(label, result, projectDir, expectedCode) { assert.equal( result.code, expectedCode, label + ' failed with code ' + result.code + '\\nstdout:\\n' + result.stdout + '\\nstderr:\\n' + result.stderr, ); - assert.equal(result.stderr, 'Project: ' + projectDir + '\\n', label + ' wrote unexpected stderr'); + assert.ok( + result.stderr.startsWith('Project: ' + projectDir + '\\n'), + label + ' did not lead stderr with the project notice\\nstderr:\\n' + result.stderr, + ); } function requireSuccessWithStderr(label, result, stderrPattern) { @@ -523,6 +537,10 @@ function requireOutput(label, result, text) { assert.match(result.stdout, text, label + ' output did not match ' + text); } +function requireStderr(label, result, stderrPattern) { + assert.match(result.stderr, stderrPattern, label + ' stderr did not match ' + stderrPattern); +} + function escapeRegExp(value) { return value.replace(/[|\\\\{}()[\\]^$+*?.]/g, '\\\\$&'); } @@ -551,6 +569,21 @@ function requireIncludes(values, expected, label) { assert.ok(values.includes(expected), label + ' did not include ' + expected + ': ' + values.join(', ')); } +async function rmWithRetry(path) { + for (let attempt = 0; ; attempt += 1) { + try { + await rm(path, { recursive: true, force: true }); + return; + } catch (error) { + const code = typeof error?.code === 'string' ? error.code : ''; + if (attempt >= 4 || !['EBUSY', 'ENOTEMPTY', 'EPERM'].includes(code)) { + throw error; + } + await delay(500); + } + } +} + async function writeSqliteWarehouse(projectDir) { const database = new DatabaseSync(join(projectDir, 'warehouse.db')); try { @@ -575,50 +608,58 @@ let daemonStarted = false; try { const projectDir = join(root, 'project'); - const version = await run('pnpm', ['exec', 'ktx', '--version']); + const version = await run(...Object.values(pnpmCommand(['exec', 'ktx', '--version']))); requireSuccess('ktx public package version', version); requireOutput('ktx public package version', version, await installedPackageVersionPattern()); const runtimeStatusBefore = parseJsonResultWithExitCode( 'ktx admin runtime status missing', - await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'status', '--json']), + await run(...Object.values(pnpmCommand(['exec', 'ktx', 'admin', 'runtime', 'status', '--json']))), 1, ); assert.equal(runtimeStatusBefore.kind, 'missing'); assert.equal(runtimeStatusBefore.layout.runtimeRoot, process.env.KTX_RUNTIME_ROOT); process.stdout.write('ktx managed runtime starts missing in isolated root\\n'); - const init = await run('pnpm', [ - 'exec', - 'ktx', - 'setup', - '--project-dir', - projectDir, - '--no-input', - '--yes', - '--skip-llm', - '--skip-embeddings', - '--skip-databases', - '--skip-sources', - '--skip-agents', - ]); + const init = await run( + ...Object.values( + pnpmCommand([ + 'exec', + 'ktx', + 'setup', + '--project-dir', + projectDir, + '--no-input', + '--yes', + '--skip-llm', + '--skip-embeddings', + '--skip-databases', + '--skip-sources', + '--skip-agents', + ]), + ), + ); requireSuccess('ktx setup', init); const emptyProjectDir = join(root, 'empty-project'); - const emptyInit = await run('pnpm', [ - 'exec', - 'ktx', - 'setup', - '--project-dir', - emptyProjectDir, - '--no-input', - '--yes', - '--skip-llm', - '--skip-embeddings', - '--skip-databases', - '--skip-sources', - '--skip-agents', - ]); + const emptyInit = await run( + ...Object.values( + pnpmCommand([ + 'exec', + 'ktx', + 'setup', + '--project-dir', + emptyProjectDir, + '--no-input', + '--yes', + '--skip-llm', + '--skip-embeddings', + '--skip-databases', + '--skip-sources', + '--skip-agents', + ]), + ), + ); requireSuccess('ktx setup empty project', emptyInit); await writeFile( join(projectDir, 'ktx.yaml'), @@ -658,17 +699,21 @@ try { 'utf-8', ); - const wikiSearch = await run('pnpm', [ - 'exec', - 'ktx', - 'wiki', - 'revenue', - '--json', - '--limit', - '5', - '--project-dir', - projectDir, - ]); + const wikiSearch = await run( + ...Object.values( + pnpmCommand([ + 'exec', + 'ktx', + 'wiki', + 'revenue', + '--json', + '--limit', + '5', + '--project-dir', + projectDir, + ]), + ), + ); const wikiSearchJson = parseJsonResult('ktx wiki search', wikiSearch); assert.equal(wikiSearchJson.kind, 'list'); assert.equal(wikiSearchJson.data.items.length, 1); @@ -700,17 +745,21 @@ try { await mkdir(join(projectDir, 'semantic-layer', 'warehouse'), { recursive: true }); await writeFile(join(projectDir, 'semantic-layer', 'warehouse', 'orders.yaml'), slYaml, 'utf-8'); - const slSearch = await run('pnpm', [ - 'exec', - 'ktx', - 'sl', - 'orders', - '--json', - '--connection-id', - 'warehouse', - '--project-dir', - projectDir, - ]); + const slSearch = await run( + ...Object.values( + pnpmCommand([ + 'exec', + 'ktx', + 'sl', + 'orders', + '--json', + '--connection-id', + 'warehouse', + '--project-dir', + projectDir, + ]), + ), + ); const slSearchJson = parseJsonResult('ktx sl search', slSearch); assert.equal(slSearchJson.kind, 'list'); assert.equal(slSearchJson.data.items.length, 1); @@ -720,17 +769,25 @@ try { requireIncludes(slSearchJson.data.items[0].matchReasons, 'lexical', 'sl search match reasons'); process.stdout.write('ktx sl search hybrid metadata verified\\n'); - const slQuery = await run('pnpm', ['exec', 'ktx', 'sl', 'query', - '--connection-id', - 'warehouse', - '--measure', - 'orders.order_count', - '--format', - 'json', - '--yes', - '--project-dir', - projectDir, - ]); + const slQuery = await run( + ...Object.values( + pnpmCommand([ + 'exec', + 'ktx', + 'sl', + 'query', + '--connection-id', + 'warehouse', + '--measure', + 'orders.order_count', + '--format', + 'json', + '--yes', + '--project-dir', + projectDir, + ]), + ), + ); requireSuccessWithStderr( 'ktx sl query first managed runtime install', slQuery, @@ -741,27 +798,35 @@ try { const runtimeStatusAfter = parseJsonResult( 'ktx admin runtime status ready', - await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'status', '--json']), + await run(...Object.values(pnpmCommand(['exec', 'ktx', 'admin', 'runtime', 'status', '--json']))), ); assert.equal(runtimeStatusAfter.kind, 'ready'); assert.deepEqual(runtimeStatusAfter.manifest.features, ['core']); assert.equal(runtimeStatusAfter.layout.runtimeRoot, process.env.KTX_RUNTIME_ROOT); process.stdout.write('ktx managed runtime lazy install verified\\n'); - const sqliteSlQuery = await run('pnpm', ['exec', 'ktx', 'sl', 'query', - '--connection-id', - 'warehouse', - '--measure', - 'orders.order_count', - '--format', - 'json', - '--execute', - '--max-rows', - '100', - '--yes', - '--project-dir', - projectDir, - ]); + const sqliteSlQuery = await run( + ...Object.values( + pnpmCommand([ + 'exec', + 'ktx', + 'sl', + 'query', + '--connection-id', + 'warehouse', + '--measure', + 'orders.order_count', + '--format', + 'json', + '--execute', + '--max-rows', + '100', + '--yes', + '--project-dir', + projectDir, + ]), + ), + ); requireSuccess('ktx sl query sqlite execute', sqliteSlQuery); requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"dialect": "sqlite"/); requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"mode": "executed"/); @@ -769,66 +834,54 @@ try { requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"rows": \\[\\s*\\[\\s*3\\s*\\]\\s*\\]/); process.stdout.write('ktx sl query sqlite execute verified\\n'); - const runtimeDoctor = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'status']); + const runtimeDoctor = await run(...Object.values(pnpmCommand(['exec', 'ktx', 'admin', 'runtime', 'status']))); requireSuccess('ktx admin runtime status', runtimeDoctor); requireOutput('ktx admin runtime status', runtimeDoctor, /KTX Python runtime/); requireOutput('ktx admin runtime status', runtimeDoctor, /status: ready/); process.stdout.write('ktx admin runtime status verified\\n'); - const runtimeStart = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'start']); + const runtimeStart = await run(...Object.values(pnpmCommand(['exec', 'ktx', 'admin', 'runtime', 'start']))); requireSuccess('ktx admin runtime start', runtimeStart); daemonStarted = true; requireOutput('ktx admin runtime start', runtimeStart, /Started KTX daemon/); requireOutput('ktx admin runtime start', runtimeStart, /url: http:\\/\\/127\\.0\\.0\\.1:\\d+/); requireOutput('ktx admin runtime start', runtimeStart, /features: core/); - const runtimeStartReuse = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'start']); + const runtimeStartReuse = await run(...Object.values(pnpmCommand(['exec', 'ktx', 'admin', 'runtime', 'start']))); requireSuccess('ktx admin runtime start reuse', runtimeStartReuse); requireOutput('ktx admin runtime start reuse', runtimeStartReuse, /Using existing KTX daemon/); requireOutput('ktx admin runtime start reuse', runtimeStartReuse, /features: core/); - const runtimeStop = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'stop']); + const runtimeStop = await run(...Object.values(pnpmCommand(['exec', 'ktx', 'admin', 'runtime', 'stop']))); requireSuccess('ktx admin runtime stop', runtimeStop); daemonStarted = false; requireOutput('ktx admin runtime stop', runtimeStop, /Stopped KTX daemon/); process.stdout.write('ktx admin runtime daemon lifecycle verified\\n'); - const structuralScan = await run('pnpm', ['exec', 'ktx', 'ingest', 'warehouse', - '--project-dir', - projectDir, - '--fast', - '--no-input', - ]); - requireSuccessWithProjectStderr('ktx ingest fast', structuralScan, projectDir); - requireOutput('ktx ingest fast', structuralScan, /Ingest finished/); - requireOutput('ktx ingest fast', structuralScan, /Database schema/); - requireOutput('ktx ingest fast', structuralScan, /warehouse\\s+done/); - await access(join(projectDir, 'semantic-layer', 'warehouse', '_schema', 'public.yaml')); - process.stdout.write('ktx ingest fast verified\\n'); - - const enrichedScan = await run('pnpm', ['exec', 'ktx', 'ingest', 'warehouse', - '--project-dir', - projectDir, - '--deep', - '--no-input', - ]); - requireExitCodeWithProjectStderr('ktx ingest deep readiness guard', enrichedScan, projectDir, 1); - requireOutput('ktx ingest deep readiness guard', enrichedScan, /Ingest finished with partial failures/); - requireOutput('ktx ingest deep readiness guard', enrichedScan, /requires deep ingest readiness/); - process.stdout.write('ktx ingest deep readiness guard verified\\n'); + const databaseIngest = await run( + ...Object.values( + pnpmCommand(['exec', 'ktx', 'ingest', 'warehouse', '--project-dir', projectDir, '--no-input']), + ), + ); + requireExitCodeWithProjectStderr('ktx ingest enrichment guard', databaseIngest, projectDir, 1); + requireStderr('ktx ingest enrichment guard', databaseIngest, /^ {2}failed /m); + requireOutput('ktx ingest enrichment guard', databaseIngest, /Ingest finished with partial failures/); + requireOutput('ktx ingest enrichment guard', databaseIngest, /enrichment is not configured/); + process.stdout.write('ktx ingest enrichment guard verified\\n'); await access(join(projectDir, '.ktx', 'db.sqlite')); process.stdout.write('ktx ingest state verified\\n'); } finally { if (daemonStarted) { - await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'stop']); + await run(...Object.values(pnpmCommand(['exec', 'ktx', 'admin', 'runtime', 'stop']))); + await delay(500); } if (previousRuntimeRoot === undefined) { delete process.env.KTX_RUNTIME_ROOT; } else { process.env.KTX_RUNTIME_ROOT = previousRuntimeRoot; } - await rm(root, { recursive: true, force: true }); + await rmWithRetry(root); } `; } @@ -844,6 +897,13 @@ import { promisify } from 'node:util'; const execFileAsync = promisify(execFile); +function pnpmCommand(args) { + if (process.platform === 'win32') { + return { command: 'cmd.exe', args: ['/d', '/s', '/c', 'pnpm', ...args] }; + } + return { command: 'pnpm', args }; +} + async function run(command, args, options = {}) { process.stdout.write('$ ' + command + ' ' + args.join(' ') + '\\n'); try { @@ -880,17 +940,17 @@ try { const packageJson = JSON.parse(await readFile(join(process.cwd(), 'package.json'), 'utf8')); assert.deepEqual(Object.keys(packageJson.dependencies), ['@kaelio/ktx']); - const help = await run('pnpm', ['exec', 'ktx', '--help']); + const help = await run(...Object.values(pnpmCommand(['exec', 'ktx', '--help']))); requireSuccess('ktx --help', help); requireStdout('ktx --help', help, /Usage: ktx/); requireStdout('ktx --help', help, /setup/); - const setupHelp = await run('pnpm', ['exec', 'ktx', 'setup', '--help']); + const setupHelp = await run(...Object.values(pnpmCommand(['exec', 'ktx', 'setup', '--help']))); requireSuccess('ktx setup --help', setupHelp); requireStdout('ktx setup --help', setupHelp, /Usage: ktx setup/); requireStdout('ktx setup --help', setupHelp, /--no-input/); - const doctor = await run('pnpm', ['exec', 'ktx', 'status', '--verbose', '--no-input']); + const doctor = await run(...Object.values(pnpmCommand(['exec', 'ktx', 'status', '--verbose', '--no-input']))); assert.ok([0, 1].includes(doctor.code), 'ktx status setup exit code must be 0 or 1'); requireStdout('ktx status setup', doctor, /KTX status/); requireStdout('ktx status setup', doctor, /No project here yet\\./); @@ -949,10 +1009,19 @@ async function verifyNpmArtifacts(layout, tmpRoot) { await writeFile(join(projectDir, 'verify-installed-cli.mjs'), npmRuntimeSmokeSource()); await writeFile(join(projectDir, 'verify-installed-cli-commands.mjs'), npmCliSmokeSource()); - await runCommand('pnpm', ['install'], { cwd: projectDir }); - await runCommand('pnpm', ['rebuild', 'better-sqlite3'], { cwd: projectDir }); + { + const pnpmInstall = pnpmCommand(['install']); + await runCommand(pnpmInstall.command, pnpmInstall.args, { cwd: projectDir }); + } + { + const pnpmRebuild = pnpmCommand(['rebuild', 'better-sqlite3']); + await runCommand(pnpmRebuild.command, pnpmRebuild.args, { cwd: projectDir }); + } await runCommand('node', ['verify-npm.mjs'], { cwd: projectDir }); - await runCommand('pnpm', ['exec', 'ktx', '--version'], { cwd: projectDir }); + { + const pnpmExecVersion = pnpmCommand(['exec', 'ktx', '--version']); + await runCommand(pnpmExecVersion.command, pnpmExecVersion.args, { cwd: projectDir }); + } await runCommand('node', ['verify-installed-cli.mjs'], { cwd: projectDir }); await runCommand('node', ['verify-installed-cli-commands.mjs'], { cwd: projectDir }); } @@ -968,7 +1037,10 @@ async function verifyNpmCliArtifacts(layout, tmpRoot) { await writeFile(join(projectDir, 'pnpm-workspace.yaml'), npmSmokePnpmWorkspaceYaml()); await writeFile(join(projectDir, 'verify-installed-cli-commands.mjs'), npmCliSmokeSource()); - await runCommand('pnpm', ['install'], { cwd: projectDir }); + { + const pnpmInstall = pnpmCommand(['install']); + await runCommand(pnpmInstall.command, pnpmInstall.args, { cwd: projectDir }); + } await runCommand('node', ['verify-installed-cli-commands.mjs'], { cwd: projectDir }); } diff --git a/scripts/package-artifacts.test.mjs b/scripts/package-artifacts.test.mjs index 7ea9339b..29e7fb1e 100644 --- a/scripts/package-artifacts.test.mjs +++ b/scripts/package-artifacts.test.mjs @@ -97,12 +97,12 @@ describe('packageArtifactLayout', () => { it('uses stable artifact paths under ktx/dist/artifacts', () => { const layout = packageArtifactLayout('/repo/ktx', PUBLIC_NPM_PACKAGE_VERSION); - assert.equal(layout.artifactDir, '/repo/ktx/dist/artifacts'); - assert.equal(layout.npmDir, '/repo/ktx/dist/artifacts/npm'); - assert.equal(layout.pythonDir, '/repo/ktx/dist/artifacts/python'); + assert.equal(layout.artifactDir, join('/repo/ktx', 'dist', 'artifacts')); + assert.equal(layout.npmDir, join('/repo/ktx', 'dist', 'artifacts', 'npm')); + assert.equal(layout.pythonDir, join('/repo/ktx', 'dist', 'artifacts', 'python')); assert.equal( layout.cliTarball, - `/repo/ktx/dist/artifacts/npm/kaelio-ktx-${PUBLIC_NPM_PACKAGE_VERSION}.tgz`, + join('/repo/ktx', 'dist', 'artifacts', 'npm', `kaelio-ktx-${PUBLIC_NPM_PACKAGE_VERSION}.tgz`), ); assert.deepEqual(Object.keys(layout.npmTarballs), ['@kaelio/ktx']); }); @@ -112,17 +112,21 @@ describe('buildArtifactCommands', () => { it('builds the CLI package, then the runtime wheel, then packs the npm tarball directly', () => { const layout = packageArtifactLayout('/repo/ktx', PUBLIC_NPM_PACKAGE_VERSION); const commands = buildArtifactCommands(layout); + const expectedBuildCommand = + process.platform === 'win32' + ? ['cmd.exe', ['/d', '/s', '/c', 'pnpm', '--filter', '@kaelio/ktx', 'run', 'build'], layout.rootDir] + : ['pnpm', ['--filter', '@kaelio/ktx', 'run', 'build'], layout.rootDir]; + const expectedPackCommand = + process.platform === 'win32' + ? ['cmd.exe', ['/d', '/s', '/c', 'pnpm', 'pack', '--out', layout.cliTarball], join('/repo/ktx', 'packages', 'cli')] + : ['pnpm', ['pack', '--out', layout.cliTarball], join('/repo/ktx', 'packages', 'cli')]; assert.deepEqual( commands.map((command) => [command.command, command.args, command.cwd]), [ - ['pnpm', ['--filter', '@kaelio/ktx', 'run', 'build'], '/repo/ktx'], - [process.execPath, ['scripts/build-python-runtime-wheel.mjs'], '/repo/ktx'], - [ - 'pnpm', - ['pack', '--out', `/repo/ktx/dist/artifacts/npm/kaelio-ktx-${PUBLIC_NPM_PACKAGE_VERSION}.tgz`], - '/repo/ktx/packages/cli', - ], + expectedBuildCommand, + [process.execPath, ['scripts/build-python-runtime-wheel.mjs'], layout.rootDir], + expectedPackCommand, ], ); }); @@ -476,18 +480,27 @@ describe('verification snippets', () => { it('runs installed CLI commands through the public package runtime', () => { const source = npmRuntimeSmokeSource(); + assert.match(source, /function pnpmCommand\(args\)/); + assert.match(source, /process\.platform === 'win32'/); + assert.match(source, /command: 'cmd\.exe'/); + assert.match(source, /args: \['\/d', '\/s', '\/c', 'pnpm', \.\.\.args\]/); + assert.match(source, /import \{ setTimeout as delay \} from 'node:timers\/promises';/); + assert.match(source, /async function rmWithRetry\(path\)/); + assert.match(source, /await delay\(500\)/); + assert.match(source, /await rmWithRetry\(root\)/); assert.match(source, /ktx public package version/); assert.match(source, /installedPackageVersionPattern/); assert.doesNotMatch(source, /@kaelio\\\/ktx 0\\\.1\\\.0/); - assert.match(source, /'ktx', 'sl', 'query'/); + assert.match(source, /pnpmCommand\(\[\s*'exec',\s*'ktx',\s*'sl',\s*'query'/); assert.doesNotMatch(source, /@ktx\/context/); assert.doesNotMatch(source, /@modelcontextprotocol/); assert.doesNotMatch(source, /startSemanticDaemon/); - assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'setup'/); + assert.doesNotMatch(source, /run\('pnpm',/); + assert.match(source, /pnpmCommand\(\[\s*'exec',\s*'ktx',\s*'setup'/); assert.match(source, /wiki', 'global', 'revenue\.md'/); - assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'wiki',\s*'revenue'/); + assert.match(source, /pnpmCommand\(\[\s*'exec',\s*'ktx',\s*'wiki',\s*'revenue'/); assert.match(source, /semantic-layer', 'warehouse', 'orders\.yaml'/); - assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'sl',\s*'orders'/); + assert.match(source, /pnpmCommand\(\[\s*'exec',\s*'ktx',\s*'sl',\s*'orders'/); assert.match(source, /orders\.order_count/); assert.match(source, /node:sqlite/); assert.match(source, /driver: sqlite/); @@ -516,11 +529,13 @@ describe('verification snippets', () => { assert.match(source, /ktx admin runtime stop/); assert.doesNotMatch(source, /ktx admin runtime prune/); assert.doesNotMatch(source, /staleRuntimeDir/); - assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'ingest',\s*'warehouse'/); - assert.match(source, /'--deep'/); + assert.match(source, /pnpmCommand\(\['exec', 'ktx', 'ingest', 'warehouse'/); + assert.doesNotMatch(source, /'--fast'/); + assert.doesNotMatch(source, /'--deep'/); assert.doesNotMatch(source, /'--enrich'/); - assert.match(source, /ktx ingest fast verified/); - assert.match(source, /ktx ingest deep readiness guard verified/); + assert.match(source, /ktx ingest enrichment guard verified/); + assert.match(source, /enrichment is not configured/); + assert.match(source, /requireStderr\('ktx ingest enrichment guard'/); assert.match(source, /enrichment:/); assert.match(source, /mode: deterministic/); assert.doesNotMatch(source, /run\('pnpm', \['exec', 'ktx', 'ingest', 'run'/); @@ -534,8 +549,11 @@ describe('verification snippets', () => { it('exercises supported public package CLI commands', () => { const source = npmCliSmokeSource(); - assert.match(source, /pnpm', \['exec', 'ktx', '--help'\]/); - assert.match(source, /pnpm', \['exec', 'ktx', 'setup', '--help'\]/); + assert.match(source, /function pnpmCommand\(args\)/); + assert.match(source, /process\.platform === 'win32'/); + assert.doesNotMatch(source, /run\('pnpm',/); + assert.match(source, /pnpmCommand\(\['exec', 'ktx', '--help'\]\)/); + assert.match(source, /pnpmCommand\(\['exec', 'ktx', 'setup', '--help'\]\)/); assert.match(source, /Usage: ktx setup/); assert.doesNotMatch(source, new RegExp(["'demo'", "'--mode'", "'deterministic'"].join(', '))); assert.match(source, /'status', '--verbose', '--no-input'/); diff --git a/scripts/relationship-benchmark-report.mjs b/scripts/relationship-benchmark-report.mjs index 7af82d57..9901d77c 100644 --- a/scripts/relationship-benchmark-report.mjs +++ b/scripts/relationship-benchmark-report.mjs @@ -14,7 +14,7 @@ import { const scriptDir = dirname(fileURLToPath(import.meta.url)); const ktxRoot = resolve(scriptDir, '..'); -const fixtureRoot = join(ktxRoot, 'packages/cli/src/test/fixtures/relationship-benchmarks'); +const fixtureRoot = join(ktxRoot, 'packages/cli/test/fixtures/relationship-benchmarks'); async function buildDetector() { const backend = process.env.KTX_BENCHMARK_LLM_BACKEND; diff --git a/scripts/test-tiering.test.mjs b/scripts/test-tiering.test.mjs index bd43ce71..a54c6a16 100644 --- a/scripts/test-tiering.test.mjs +++ b/scripts/test-tiering.test.mjs @@ -14,41 +14,41 @@ function assertScriptContainsAll(script, expected) { describe('test tiering', () => { const cliSlowTests = [ - 'src/setup-databases.test.ts', - 'src/scan.test.ts', - 'src/commands/connection-metabase-setup.test.ts', - 'src/setup-models.test.ts', - 'src/setup-sources.test.ts', - 'src/setup.test.ts', - 'src/connection.test.ts', - 'src/setup-embeddings.test.ts', - 'src/ingest.test.ts', - 'src/commands/connection-mapping.test.ts', - 'src/ingest-viz.test.ts', - 'src/demo.test.ts', - 'src/setup-project.test.ts', - 'src/sl.test.ts', - 'src/local-scan-connectors.test.ts', - 'src/commands/connection-notion.test.ts', + 'test/setup-databases.test.ts', + 'test/scan.test.ts', + 'test/commands/connection-metabase-setup.test.ts', + 'test/setup-models.test.ts', + 'test/setup-sources.test.ts', + 'test/setup.test.ts', + 'test/connection.test.ts', + 'test/setup-embeddings.test.ts', + 'test/ingest.test.ts', + 'test/commands/connection-mapping.test.ts', + 'test/ingest-viz.test.ts', + 'test/demo.test.ts', + 'test/setup-project.test.ts', + 'test/sl.test.ts', + 'test/local-scan-connectors.test.ts', + 'test/commands/connection-notion.test.ts', ]; const contextSlowTests = [ - 'src/context/scan/local-scan.test.ts', - 'src/context/mcp/local-project-ports.test.ts', - 'src/context/ingest/local-stage-ingest.test.ts', - 'src/context/sl/pglite-sl-search-prototype.test.ts', - 'src/context/core/git.service.test.ts', - 'src/context/ingest/local-adapters.test.ts', - 'src/context/ingest/local-bundle-ingest.test.ts', - 'src/context/ingest/local-metabase-ingest.test.ts', - 'src/context/sl/local-sl.test.ts', - 'src/context/search/pglite-owner-process.test.ts', - 'src/context/scan/local-enrichment-artifacts.test.ts', - 'src/context/search/pglite-spike.test.ts', - 'src/context/wiki/local-knowledge.test.ts', - 'src/context/sl/local-query.test.ts', - 'src/context/scan/relationship-review-decisions.test.ts', - 'src/context/scan/relationship-profiling.test.ts', + 'test/context/scan/local-scan.test.ts', + 'test/context/mcp/local-project-ports.test.ts', + 'test/context/ingest/local-stage-ingest.test.ts', + 'test/context/sl/pglite-sl-search-prototype.test.ts', + 'test/context/core/git.service.test.ts', + 'test/context/ingest/local-adapters.test.ts', + 'test/context/ingest/local-bundle-ingest.test.ts', + 'test/context/ingest/local-metabase-ingest.test.ts', + 'test/context/sl/local-sl.test.ts', + 'test/context/search/pglite-owner-process.test.ts', + 'test/context/scan/local-enrichment-artifacts.test.ts', + 'test/context/search/pglite-spike.test.ts', + 'test/context/wiki/local-knowledge.test.ts', + 'test/context/sl/local-query.test.ts', + 'test/context/scan/relationship-review-decisions.test.ts', + 'test/context/scan/relationship-profiling.test.ts', ]; it('keeps slow package tests out of default local package test scripts', async () => { diff --git a/scripts/upgrade-dependencies.mjs b/scripts/upgrade-dependencies.mjs new file mode 100644 index 00000000..5ec4996e --- /dev/null +++ b/scripts/upgrade-dependencies.mjs @@ -0,0 +1,111 @@ +#!/usr/bin/env node + +import { execFile as execFileCallback } from 'node:child_process'; +import { readFile as fsReadFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFileCallback); +const npmCheckUpdatesRejectArgs = ['--reject', 'fumadocs-core,fumadocs-ui']; + +function ktxRootDir() { + return resolve(dirname(fileURLToPath(import.meta.url)), '..'); +} + +function failureText(error) { + const stdout = typeof error?.stdout === 'string' ? error.stdout.trim() : ''; + const stderr = typeof error?.stderr === 'string' ? error.stderr.trim() : ''; + const message = error instanceof Error ? error.message.trim() : String(error); + return [stderr, stdout, message].filter((line) => line.length > 0).join('\n') || 'Command failed'; +} + +function commandText(command, args) { + return [command, ...args].join(' '); +} + +function pythonDependencyUpdatePhases() { + const manifests = ['pyproject.toml', 'python/ktx-sl/pyproject.toml', 'python/ktx-daemon/pyproject.toml']; + return manifests.map((manifest) => ({ + name: `Python dependency constraints: ${manifest}`, + command: 'uvx', + args: ['dependency-check-updates', '--manifest', manifest, '-u'], + retry: commandText('uvx', ['dependency-check-updates', '--manifest', manifest, '-u']), + })); +} + +async function pnpmMinimumReleaseAgeCooldown(rootDir, readFile) { + let workspaceConfig; + try { + workspaceConfig = await readFile(resolve(rootDir, 'pnpm-workspace.yaml'), 'utf8'); + } catch (error) { + if (error?.code === 'ENOENT') { + return []; + } + throw error; + } + + const match = workspaceConfig.match(/^\s*minimumReleaseAge:\s*(\d+)\s*$/m); + if (!match) { + return []; + } + return ['--cooldown', `${match[1]}m`]; +} + +export async function runDependencyUpgrade(options = {}) { + const rootDir = options.rootDir ?? ktxRootDir(); + const execFile = options.execFile ?? execFileAsync; + const readFile = options.readFile ?? fsReadFile; + const log = options.log ?? ((line) => process.stdout.write(`${line}\n`)); + const npmCheckUpdatesCooldownArgs = await pnpmMinimumReleaseAgeCooldown(rootDir, readFile); + const phases = [ + { + name: 'TypeScript dependency constraints', + command: 'pnpm', + args: ['dlx', 'npm-check-updates', '-u', '--deep', ...npmCheckUpdatesRejectArgs, ...npmCheckUpdatesCooldownArgs], + retry: commandText('pnpm', [ + 'dlx', + 'npm-check-updates', + '-u', + '--deep', + ...npmCheckUpdatesRejectArgs, + ...npmCheckUpdatesCooldownArgs, + ]), + }, + ...pythonDependencyUpdatePhases(), + { + name: 'TypeScript lockfile', + command: 'pnpm', + args: ['install'], + retry: 'pnpm install', + }, + { + name: 'Python lockfile', + command: 'uv', + args: ['lock', '--upgrade'], + retry: 'uv lock --upgrade', + }, + ]; + + for (const phase of phases) { + log(`RUN ${phase.name}: ${commandText(phase.command, phase.args)}`); + try { + await execFile(phase.command, phase.args, { cwd: rootDir, maxBuffer: 1024 * 1024 * 64 }); + log(`PASS ${phase.name}`); + } catch (error) { + log(`FAIL ${phase.name}: ${failureText(error)}`); + log(`Retry: ${phase.retry}`); + return { ok: false, failedPhase: phase }; + } + } + + log('Dependency manifests and lockfiles were updated. Run `pnpm run check` before committing.'); + return { ok: true }; +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + const result = await runDependencyUpgrade(); + if (!result.ok) { + process.exitCode = 1; + } +} diff --git a/scripts/upgrade-dependencies.test.mjs b/scripts/upgrade-dependencies.test.mjs new file mode 100644 index 00000000..6f42bf7d --- /dev/null +++ b/scripts/upgrade-dependencies.test.mjs @@ -0,0 +1,123 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { test } from 'node:test'; +import { runDependencyUpgrade } from './upgrade-dependencies.mjs'; + +test('runDependencyUpgrade updates TypeScript and Python manifests before regenerating lockfiles', async () => { + const calls = []; + const logs = []; + + const result = await runDependencyUpgrade({ + rootDir: '/workspace/ktx', + readFile: async (path) => { + assert.equal(path, '/workspace/ktx/pnpm-workspace.yaml'); + return 'packages: []\nminimumReleaseAge: 10080\n'; + }, + execFile: async (command, args, options) => { + calls.push({ command, args, cwd: options.cwd }); + return { stdout: '', stderr: '' }; + }, + log: (line) => logs.push(line), + }); + + assert.equal(result.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.args]), + [ + [ + 'pnpm', + [ + 'dlx', + 'npm-check-updates', + '-u', + '--deep', + '--reject', + 'fumadocs-core,fumadocs-ui', + '--cooldown', + '10080m', + ], + ], + ['uvx', ['dependency-check-updates', '--manifest', 'pyproject.toml', '-u']], + ['uvx', ['dependency-check-updates', '--manifest', 'python/ktx-sl/pyproject.toml', '-u']], + ['uvx', ['dependency-check-updates', '--manifest', 'python/ktx-daemon/pyproject.toml', '-u']], + ['pnpm', ['install']], + ['uv', ['lock', '--upgrade']], + ], + ); + assert.equal(calls.every((call) => call.cwd === '/workspace/ktx'), true); + assert.equal(logs.some((line) => line.includes('PASS Python dependency constraints')), true); +}); + +test('runDependencyUpgrade stops at the failed phase and prints a retry command', async () => { + const calls = []; + const logs = []; + + const result = await runDependencyUpgrade({ + rootDir: '/workspace/ktx', + readFile: async () => 'packages: []\n', + execFile: async (command, args) => { + calls.push({ command, args }); + if (command === 'uvx' && args.includes('python/ktx-sl/pyproject.toml')) { + const error = new Error('dependency-check-updates failed'); + error.stdout = 'checking Python dependencies'; + error.stderr = 'could not read pyproject.toml'; + throw error; + } + return { stdout: '', stderr: '' }; + }, + log: (line) => logs.push(line), + }); + + assert.equal(result.ok, false); + assert.equal(result.failedPhase.name, 'Python dependency constraints: python/ktx-sl/pyproject.toml'); + assert.equal(result.failedPhase.retry, 'uvx dependency-check-updates --manifest python/ktx-sl/pyproject.toml -u'); + assert.deepEqual( + calls.map((call) => [call.command, call.args]), + [ + ['pnpm', ['dlx', 'npm-check-updates', '-u', '--deep', '--reject', 'fumadocs-core,fumadocs-ui']], + ['uvx', ['dependency-check-updates', '--manifest', 'pyproject.toml', '-u']], + ['uvx', ['dependency-check-updates', '--manifest', 'python/ktx-sl/pyproject.toml', '-u']], + ], + ); + assert.equal(logs.some((line) => line.includes('FAIL Python dependency constraints')), true); + assert.equal(logs.some((line) => line.includes('could not read pyproject.toml')), true); + assert.equal(logs.some((line) => line.includes('checking Python dependencies')), true); + assert.equal( + logs.some((line) => line.includes('Retry: uvx dependency-check-updates --manifest python/ktx-sl/pyproject.toml -u')), + true, + ); +}); + +test('runDependencyUpgrade ignores missing pnpm minimum release age config', async () => { + const calls = []; + + const result = await runDependencyUpgrade({ + rootDir: '/workspace/ktx', + readFile: async () => { + throw Object.assign(new Error('missing'), { code: 'ENOENT' }); + }, + execFile: async (command, args) => { + calls.push({ command, args }); + return { stdout: '', stderr: '' }; + }, + log: () => undefined, + }); + + assert.equal(result.ok, true); + assert.deepEqual(calls[0], { + command: 'pnpm', + args: ['dlx', 'npm-check-updates', '-u', '--deep', '--reject', 'fumadocs-core,fumadocs-ui'], + }); + assert.equal( + calls + .filter((call) => call.command === 'uvx') + .every((call) => call.args.includes('--manifest') && !call.args.includes('-d')), + true, + ); +}); + +test('package scripts expose the full dependency upgrade command', async () => { + const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')); + + assert.equal(packageJson.scripts['deps:upgrade'], 'node scripts/upgrade-dependencies.mjs'); +}); diff --git a/skills.sh.json b/skills.sh.json new file mode 100644 index 00000000..6bc144ae --- /dev/null +++ b/skills.sh.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://skills.sh/schemas/skills.sh.schema.json", + "notGrouped": "bottom", + "groupings": [ + { + "title": "ktx", + "description": "Skills for installing, configuring, and operating ktx.", + "skills": ["ktx"] + } + ] +} diff --git a/skills/ktx/SKILL.md b/skills/ktx/SKILL.md new file mode 100644 index 00000000..ef316b63 --- /dev/null +++ b/skills/ktx/SKILL.md @@ -0,0 +1,267 @@ +--- +name: ktx +description: Installs and configures ktx, the open-source context layer for data agents — runs ktx setup non-interactively with hidden CLI flags, configures database connections and embeddings, installs agent integration, and verifies readiness. Use when the user asks an agent to add ktx to a project, connect data sources, install agent rules, ingest schema, or troubleshoot a local ktx install. +--- + +# ktx + +Install and configure **ktx**, the open-source context layer for data agents. +Use this skill when a user wants an agent to add **ktx** to a project, connect +data sources, build initial context, install agent integration, or troubleshoot +a local **ktx** setup. + +## Operating rules + +- Act autonomously when the user asks you to install or configure **ktx**. + The non-interactive scripted flow below is the canonical path — bare + `ktx setup` is interactive (clack prompts) and an agent cannot drive it. +- Setup's non-interactive flags are intentionally hidden from `--help`. Use the + flags listed below; verify uncommon flags against the docs at + `https://docs.kaelio.com/ktx/` or this skill — not against `--help` output. +- Ask only for values you cannot infer: project directory, connection targets, + credentials, account identifiers, and source selections. +- Prefer `file:/abs/path` secret refs over `env:VAR_NAME`. `env:` refs are + re-resolved against the process environment on **every** `ktx` run, so a var + exported only in the setup shell is gone when `ktx ingest` or `ktx mcp start` + runs later — the secret silently resolves to empty and the connection fails. + `file:` refs read from disk and survive across shells. The same caveat + applies to `--*-api-key-env` flags: the named var must be present in every + shell that runs `ktx`, including the `ktx mcp` daemon's environment. +- A literal database URL is safe to pass — `ktx setup` auto-externalizes it + into `.ktx/secrets/-url` and rewrites `ktx.yaml` to a `file:` ref (see + workflow step 2). Source credential refs are **not** auto-externalized: write + the secret to a file under `.ktx/secrets/` (`chmod 600`) and pass a `file:` + ref. Never ask the user to paste a secret when a `file:` or `env:` ref works. +- Do not commit `.ktx/secrets/*`. +- Print each command you run and its result. +- Setup and ingest can run for many minutes (LLM-heavy source ingests take the + longest), and from the outside a slow step looks identical to a stuck one. + Don't go silent: say what's about to run and that it may take a while, then + post brief progress/liveness updates while it runs (see step 4) so the user + never has to wonder whether it stalled — otherwise they may kill it mid-run. +- If a command fails, identify the cause and change something before retrying. + +## Gather inputs once + +Before invoking `ktx setup`, collect in one round: + +1. Project directory (default: current working directory). +2. LLM backend and key strategy. In `--no-input` mode the CLI defaults to + `anthropic` and **requires an API key**. When the user is inside Claude + Code, pass `--llm-backend claude-code` explicitly; otherwise pass + `--llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY`. +3. Embedding backend (`sentence-transformers` is the local default and needs + no key; use `openai` only if the user already has a key, then pass + `--embedding-api-key-env OPENAI_API_KEY`). +4. Database: driver, connection id, URL (or `env:` / `file:` ref), and one or + more schemas. +5. Optional context sources (dbt, Metabase, Looker, LookML, MetricFlow, + Notion). Add each one with a follow-up `ktx setup --source …` run (see + [Add context sources](#add-context-sources)); use `--skip-sources` only + when the user has none. + +Do not discover these inputs across multiple setup runs. + +## Install workflow + +1. **Detect the install path.** If the working directory contains + `packages/cli/dist/bin.js` or `pnpm-workspace.yaml` referencing + `@kaelio/ktx` you are inside the **ktx** monorepo — build and link the + local CLI with `pnpm` and do **not** run `npm install -g`. Otherwise: + + ```bash + node --version # require >= 22; stop and ask the user if older + ktx --version || npm install -g @kaelio/ktx + ``` + +2. **Run scripted setup** (canonical path): + + ```bash + ktx setup --no-input --yes \ + --project-dir \ + --llm-backend claude-code \ + --embedding-backend sentence-transformers \ + --database --database-connection-id \ + --database-url '' \ + --database-schema \ + --skip-sources \ + --skip-agents + ``` + + - Configure one new database connection per setup invocation. For multiple + connections, rerun setup once per connection. + - Pasting a literal `--database-url` is safe: the CLI relocates the URL + into `.ktx/secrets/-url` and rewrites `ktx.yaml` to a + `file:` ref automatically. + - `ktx setup` runs agent integration as its **last** step. In `--no-input` + mode with neither `--target` nor `--skip-agents`, that step has no input, + prints `Run in a TTY, or pass --target .`, and the command exits + non-zero **even though every database/LLM/embedding step succeeded**. Pass + `--skip-agents` to defer agents to step 5 (as above), or `--target ` + to install them inline and exit 0. Judge data-layer success from + `ktx status`, not from this exit code. + +3. **Resumability and `--skip-*`.** Re-running `ktx setup` against an existing + project resumes its config. Use `--skip-llm`, `--skip-databases`, + `--skip-sources`, or `--skip-embeddings` to leave a slice unconfigured but + let the rest complete instead of aborting on the first failure. **When + resuming an existing project to change one slice (e.g. only LLM), still + pass the database flags from the previous run** — setup validates current + flags, not persisted `ktx.yaml` state. + +4. **Build context** if setup did not already complete one: + + ```bash + ktx ingest --no-input + ``` + + `ktx ingest` always builds enriched context and requires a configured model + and embeddings (set during setup); a database connection without them fails + with an enrichment-readiness error. Note: `ktx ingest` rejects `--yes` + together with `--no-input` (*Choose only one runtime install mode*); + `ktx setup` accepts both. Use `--no-input` only for ingest. + + Ingest one connection at a time. It can run for many minutes with **no + stdout** until it exits (LLM-heavy sources like Metabase are the slowest), so + don't assume it hung, and don't pipe it through `tail`/`head` — that buffers + all output to the end, so run it raw. Tell the user up front that the step is + slow, then keep them posted instead of blocking silently: run the ingest in + the background and poll for liveness every minute or so, reporting a one-line + update each time (which connection, roughly how long it's been running, and + that `.ktx` files are still changing) so a long run never looks stuck: + + ```bash + find /.ktx/worktrees /.ktx/ingest-transcripts -type f -mmin -3 + ``` + + On success, the `Ingest finished` summary table shows `done` in the + `Source ingest` and `Memory update` columns with no `Failed sources:` + section. + +5. **Install agent integration:** + + ```bash + ktx setup --agents --target + ktx mcp start --project-dir + ``` + + Agent integration is **not usable until `ktx mcp start` is running**. The + `--agents` step prints this requirement as `Required before using agents`. + +6. **Fall back to bare `ktx setup` only when a human is at the keyboard** — + it uses interactive prompts an agent cannot answer. + +## Add context sources + +Context sources (dbt, Metabase, Looker, LookML, MetricFlow, Notion) are added +**one at a time** — `--source` is not repeatable, so run `ktx setup` once per +source. Source setup is resumable against an existing project: pass +`--skip-databases --skip-llm --skip-embeddings --skip-agents` so only the source +is configured (the trailing agent step otherwise fails the run — see install +step 2). Map Metabase, Looker, and LookML to an existing database connection +with `--source-warehouse-connection-id ` (required for those). +**dbt ignores `--source-warehouse-connection-id`** — it maps to the warehouse by +table name — so omit it for dbt. Use `file:/abs/path` refs for keys and tokens +(see the secrets rule above); `env:` refs must be exported in every later `ktx` +shell. + +```bash +# dbt — pick exactly one of --source-path (local) or --source-git-url (remote). +# No --source-warehouse-connection-id: dbt maps to the warehouse by table name. +ktx setup --no-input --yes --skip-databases --skip-llm --skip-embeddings --skip-agents \ + --source dbt --source-connection-id \ + --source-git-url --source-branch + +# Metabase +ktx setup --no-input --yes --skip-databases --skip-llm --skip-embeddings --skip-agents \ + --source metabase --source-connection-id \ + --source-url --source-api-key-ref file:/abs/path/metabase-api-key \ + --source-warehouse-connection-id \ + --metabase-database-id + +# Notion +ktx setup --no-input --yes --skip-databases --skip-llm --skip-embeddings --skip-agents \ + --source notion --source-connection-id \ + --source-auth-token-ref file:/abs/path/notion-token \ + --notion-crawl-mode selected_roots --notion-root-page-id +``` + +Notes: + +- `--metabase-database-id` is the **numeric id of the warehouse inside + Metabase** (not the ktx connection id). Discover it from the Metabase API + (`GET /api/database`) or UI if the user doesn't know it. +- `--notion-crawl-mode selected_roots` requires at least one + `--notion-root-page-id` (repeatable); use `all_accessible` to crawl + everything the token can see. +- After adding sources, ingest each new connection so its context is queryable: + `ktx ingest --no-input`. + +## Files to inspect + +- `ktx.yaml`: project configuration. +- `.ktx/secrets/*`: local secret files. Never commit them. +- `semantic-layer//*.yaml`: semantic sources for SQL + compilation. +- `wiki/**/*.md`: project context pages for agents. +- `.claude/skills/ktx/`, `.agents/skills/ktx/`, `.cursor/rules/ktx.mdc`, and + `.opencode/commands/ktx.md`: generated agent integration files. + +## Verification + +After setup, run: + +```bash +ktx connection test +ktx status --json --no-input +ktx sl --output plain # lists compiled semantic sources; `ktx sl` has no --no-input +``` + +**Judge readiness from `ktx status --json` fields, not the exit code.** +`ktx status` exits 1 whenever the LLM is `none` (`verdict: "blocked"`), even +when embeddings and every database connection are healthy. Treat success as: + +- `verdict: "ready"` at the top of the JSON, and +- every `connections[].status === "ok"` (other levels: `warn`, `fail`, + `skipped`), and +- every `ktx connection test ` exited 0, and +- for each ingested source, `localStats.semanticLayer[].sourceCount > 0` and + `localStats.wikiPages[].count > 0` — these confirm the source actually + produced context. Do **not** rely on `localStats.ingest.perConnection` to + confirm source ingests: it reflects only completed warehouse ingest reports + and under-reports (often lists just the warehouse connection). + +If the LLM is intentionally left unconfigured, `verdict` is `blocked` and the +exit is non-zero by design — that is still a usable context layer, so report it +as "ready, LLM optional" and judge the data layer by the connection and +`localStats` fields above rather than retrying setup. + +## Troubleshooting + +For known failure signatures (`invalid ELF header`, +`Native CLI binary for not found`, `Missing Anthropic API key`, +`claude-code` probe failure, `KTX cannot work without a database` on resume, +`Run in a TTY, or pass --target .` with a misleading exit 1, and a +secret that resolves empty only during `ktx ingest`/`ktx mcp`), see +[troubleshooting.md](troubleshooting.md). + +## Final report + +End setup work with a concise report: + +```text +ktx SETUP COMPLETE + +Project: +LLM: / +Embeddings: / +Connections: () status= +Sources: +Verdict: + +Next: +1. +2. + +RESULT: PASS +``` diff --git a/skills/ktx/agents/openai.yaml b/skills/ktx/agents/openai.yaml new file mode 100644 index 00000000..41eb75d2 --- /dev/null +++ b/skills/ktx/agents/openai.yaml @@ -0,0 +1,7 @@ +interface: + display_name: "ktx" + short_description: "Install and configure ktx for data agents" + default_prompt: "Use $ktx to install and configure ktx in this project." + +policy: + allow_implicit_invocation: true diff --git a/skills/ktx/troubleshooting.md b/skills/ktx/troubleshooting.md new file mode 100644 index 00000000..20f72d23 --- /dev/null +++ b/skills/ktx/troubleshooting.md @@ -0,0 +1,120 @@ +# ktx setup troubleshooting + +Known failure signatures hit by agent-driven `ktx setup` runs. Match the +error string in the left column, apply the fix in the right column. + +## `Error: invalid ELF header` from `better-sqlite3` + +Native module compiled for a different platform or architecture (e.g. +installed under Rosetta then run under native arm64). + +Fix: + +```bash +# Inside the ktx monorepo: +pnpm rebuild better-sqlite3 + +# Or for a global install: +npm rebuild --global better-sqlite3 +``` + +## `Native CLI binary for not found` + +The platform-specific optional dependency that ships the native CLI binary +was skipped during install (npm/pnpm "optional dep not for this platform"). + +Fix: + +```bash +npm install -g @kaelio/ktx --force +``` + +## `Missing Anthropic API key: pass --anthropic-api-key-env or --anthropic-api-key-file` + +`--no-input` mode defaulted the LLM backend to `anthropic` because no +`--llm-backend` flag was supplied. The CLI then required a key. + +Fix — pick one: + +```bash +# Inside Claude Code, prefer the local backend: +ktx setup --no-input --llm-backend claude-code ...other flags... + +# Otherwise point at an existing env var: +ktx setup --no-input --llm-backend anthropic \ + --anthropic-api-key-env ANTHROPIC_API_KEY ...other flags... +``` + +## `claude-code` LLM probe fails (auth or binary not found) + +The `claude` CLI is not on the agent's `PATH`, or the user has not run +`claude` interactively at least once to log in. + +Fix: + +```bash +which claude # confirm the binary resolves +claude --version # confirm it runs +# If auth probe still fails, the user must run `claude` once interactively +# to complete login; agents cannot do this step. +``` + +If `claude-code` cannot be made to work, fall back to `--skip-llm` and let +the rest of setup complete; the project is still a usable context layer +without an LLM. + +## `KTX cannot work without a database` when resuming setup + +`ktx setup` validates the **current invocation's flags**, not the persisted +`ktx.yaml`. Resuming setup with only `--llm-backend …` fails even when the +project already has a healthy database connection. + +Fix — re-pass the database flags from the original setup run, even when +only changing one slice: + +```bash +ktx setup --no-input \ + --database --database-connection-id \ + --llm-backend claude-code +``` + +## `Run in a TTY, or pass --target .` and `ktx setup` exits 1 + +`ktx setup` runs agent integration as its last step. In `--no-input` mode with +neither `--target` nor `--skip-agents`, that step has no input and the whole +command exits non-zero — even when every database, LLM, and embedding step +already succeeded. The exit code is misleading here. + +Fix — pass one of these to the data-only setup runs: + +```bash +# Defer agents; install them later with `ktx setup --agents --target `: +ktx setup --no-input --yes ...other flags... --skip-agents + +# Or install agents inline and exit 0: +ktx setup --no-input --yes ...other flags... --target claude-code +``` + +Either way, confirm the data work landed with `ktx status --json` rather than +trusting the exit code. + +## A secret resolves empty only during `ktx ingest` or `ktx mcp` + +Setup succeeded, but a later `ktx ingest`/`ktx mcp start` fails to connect or +authenticate. The connection used an `env:VAR_NAME` ref (or a `--*-api-key-env` +flag) and the variable was exported only in the setup shell. `env:` refs are +re-resolved against the process environment on every `ktx` run, so they resolve +to empty wherever the var is absent — including the `ktx mcp` daemon. + +Fix — write the secret to a file and use a `file:` ref, which reads from disk +and survives across shells: + +```bash +mkdir -p "$PROJECT/.ktx/secrets" +printf '%s\n' '' > "$PROJECT/.ktx/secrets/-" +chmod 600 "$PROJECT/.ktx/secrets/"* +# then pass: --source-api-key-ref file:$PROJECT/.ktx/secrets/- +``` + +Alternatively, ensure the var is exported in every shell that runs `ktx`, +including the environment of the `ktx mcp` daemon. diff --git a/tombi.toml b/tombi.toml new file mode 100644 index 00000000..bf3b3519 --- /dev/null +++ b/tombi.toml @@ -0,0 +1,5 @@ +[[schemas]] +path = "tombi://www.schemastore.org/pyproject.json" +include = ["**/pyproject.toml"] +format.rules.array-values-order.enabled = false +format.rules.table-keys-order.enabled = false diff --git a/uv.lock b/uv.lock index 9c580fbf..40553e46 100644 --- a/uv.lock +++ b/uv.lock @@ -5,10 +5,10 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'darwin'", "python_full_version < '3.14' and sys_platform == 'win32'", "python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version >= '3.14' and sys_platform == 'darwin'", "python_full_version < '3.14' and sys_platform == 'darwin'", ] @@ -60,11 +60,11 @@ wheels = [ [[package]] name = "certifi" -version = "2026.4.22" +version = "2026.5.20" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/25/ee/6caf7a40c36a1220410afe15a1cc64993a1f864871f698c0f93acb72842a/certifi-2026.4.22.tar.gz", hash = "sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580", size = 137077, upload-time = "2026-04-22T11:26:11.191Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl", hash = "sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a", size = 135707, upload-time = "2026-04-22T11:26:09.372Z" }, + { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, ] [[package]] @@ -135,14 +135,14 @@ wheels = [ [[package]] name = "click" -version = "8.3.3" +version = "8.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bb/63/f9e1ea081ce35720d8b92acde70daaedace594dc93b693c869e0d5910718/click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2", size = 328061, upload-time = "2026-04-22T15:11:27.506Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9b/98/518d8e5081007684232226f475082b30087d0f585e8457db087298259f49/click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96", size = 353007, upload-time = "2026-05-22T04:08:37.769Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502, upload-time = "2026-04-22T15:11:25.044Z" }, + { url = "https://files.pythonhosted.org/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", size = 116639, upload-time = "2026-05-22T04:08:35.26Z" }, ] [[package]] @@ -156,71 +156,71 @@ wheels = [ [[package]] name = "coverage" -version = "7.13.5" +version = "7.14.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/e0/70553e3000e345daff267cec284ce4cbf3fc141b6da229ac52775b5428f1/coverage-7.13.5.tar.gz", hash = "sha256:c81f6515c4c40141f83f502b07bbfa5c240ba25bbe73da7b33f1e5b6120ff179", size = 915967, upload-time = "2026-03-17T10:33:18.341Z" } +sdist = { url = "https://files.pythonhosted.org/packages/54/fd/0ab2772530e946e1be1abd0bc09e647ec9b02e88f0867857601fefca8953/coverage-7.14.1.tar.gz", hash = "sha256:30c08f7d90415aa98b3c990385dea2939b0da55f38515e5b369b83655f8523be", size = 920132, upload-time = "2026-05-26T20:41:36.783Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/74/8c/74fedc9663dcf168b0a059d4ea756ecae4da77a489048f94b5f512a8d0b3/coverage-7.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ec4af212df513e399cf11610cc27063f1586419e814755ab362e50a85ea69c1", size = 219576, upload-time = "2026-03-17T10:31:09.045Z" }, - { url = "https://files.pythonhosted.org/packages/0c/c9/44fb661c55062f0818a6ffd2685c67aa30816200d5f2817543717d4b92eb/coverage-7.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:941617e518602e2d64942c88ec8499f7fbd49d3f6c4327d3a71d43a1973032f3", size = 219942, upload-time = "2026-03-17T10:31:10.708Z" }, - { url = "https://files.pythonhosted.org/packages/5f/13/93419671cee82b780bab7ea96b67c8ef448f5f295f36bf5031154ec9a790/coverage-7.13.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:da305e9937617ee95c2e39d8ff9f040e0487cbf1ac174f777ed5eddd7a7c1f26", size = 250935, upload-time = "2026-03-17T10:31:12.392Z" }, - { url = "https://files.pythonhosted.org/packages/ac/68/1666e3a4462f8202d836920114fa7a5ee9275d1fa45366d336c551a162dd/coverage-7.13.5-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:78e696e1cc714e57e8b25760b33a8b1026b7048d270140d25dafe1b0a1ee05a3", size = 253541, upload-time = "2026-03-17T10:31:14.247Z" }, - { url = "https://files.pythonhosted.org/packages/4e/5e/3ee3b835647be646dcf3c65a7c6c18f87c27326a858f72ab22c12730773d/coverage-7.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02ca0eed225b2ff301c474aeeeae27d26e2537942aa0f87491d3e147e784a82b", size = 254780, upload-time = "2026-03-17T10:31:16.193Z" }, - { url = "https://files.pythonhosted.org/packages/44/b3/cb5bd1a04cfcc49ede6cd8409d80bee17661167686741e041abc7ee1b9a9/coverage-7.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04690832cbea4e4663d9149e05dba142546ca05cb1848816760e7f58285c970a", size = 256912, upload-time = "2026-03-17T10:31:17.89Z" }, - { url = "https://files.pythonhosted.org/packages/1b/66/c1dceb7b9714473800b075f5c8a84f4588f887a90eb8645282031676e242/coverage-7.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0590e44dd2745c696a778f7bab6aa95256de2cbc8b8cff4f7db8ff09813d6969", size = 251165, upload-time = "2026-03-17T10:31:19.605Z" }, - { url = "https://files.pythonhosted.org/packages/b7/62/5502b73b97aa2e53ea22a39cf8649ff44827bef76d90bf638777daa27a9d/coverage-7.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d7cfad2d6d81dd298ab6b89fe72c3b7b05ec7544bdda3b707ddaecff8d25c161", size = 252908, upload-time = "2026-03-17T10:31:21.312Z" }, - { url = "https://files.pythonhosted.org/packages/7d/37/7792c2d69854397ca77a55c4646e5897c467928b0e27f2d235d83b5d08c6/coverage-7.13.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e092b9499de38ae0fbfbc603a74660eb6ff3e869e507b50d85a13b6db9863e15", size = 250873, upload-time = "2026-03-17T10:31:23.565Z" }, - { url = "https://files.pythonhosted.org/packages/a3/23/bc866fb6163be52a8a9e5d708ba0d3b1283c12158cefca0a8bbb6e247a43/coverage-7.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:48c39bc4a04d983a54a705a6389512883d4a3b9862991b3617d547940e9f52b1", size = 255030, upload-time = "2026-03-17T10:31:25.58Z" }, - { url = "https://files.pythonhosted.org/packages/7d/8b/ef67e1c222ef49860701d346b8bbb70881bef283bd5f6cbba68a39a086c7/coverage-7.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2d3807015f138ffea1ed9afeeb8624fd781703f2858b62a8dd8da5a0994c57b6", size = 250694, upload-time = "2026-03-17T10:31:27.316Z" }, - { url = "https://files.pythonhosted.org/packages/46/0d/866d1f74f0acddbb906db212e096dee77a8e2158ca5e6bb44729f9d93298/coverage-7.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee2aa19e03161671ec964004fb74b2257805d9710bf14a5c704558b9d8dbaf17", size = 252469, upload-time = "2026-03-17T10:31:29.472Z" }, - { url = "https://files.pythonhosted.org/packages/7a/f5/be742fec31118f02ce42b21c6af187ad6a344fed546b56ca60caacc6a9a0/coverage-7.13.5-cp313-cp313-win32.whl", hash = "sha256:ce1998c0483007608c8382f4ff50164bfc5bd07a2246dd272aa4043b75e61e85", size = 222112, upload-time = "2026-03-17T10:31:31.526Z" }, - { url = "https://files.pythonhosted.org/packages/66/40/7732d648ab9d069a46e686043241f01206348e2bbf128daea85be4d6414b/coverage-7.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:631efb83f01569670a5e866ceb80fe483e7c159fac6f167e6571522636104a0b", size = 222923, upload-time = "2026-03-17T10:31:33.633Z" }, - { url = "https://files.pythonhosted.org/packages/48/af/fea819c12a095781f6ccd504890aaddaf88b8fab263c4940e82c7b770124/coverage-7.13.5-cp313-cp313-win_arm64.whl", hash = "sha256:f4cd16206ad171cbc2470dbea9103cf9a7607d5fe8c242fdf1edf36174020664", size = 221540, upload-time = "2026-03-17T10:31:35.445Z" }, - { url = "https://files.pythonhosted.org/packages/23/d2/17879af479df7fbbd44bd528a31692a48f6b25055d16482fdf5cdb633805/coverage-7.13.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0428cbef5783ad91fe240f673cc1f76b25e74bbfe1a13115e4aa30d3f538162d", size = 220262, upload-time = "2026-03-17T10:31:37.184Z" }, - { url = "https://files.pythonhosted.org/packages/5b/4c/d20e554f988c8f91d6a02c5118f9abbbf73a8768a3048cb4962230d5743f/coverage-7.13.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e0b216a19534b2427cc201a26c25da4a48633f29a487c61258643e89d28200c0", size = 220617, upload-time = "2026-03-17T10:31:39.245Z" }, - { url = "https://files.pythonhosted.org/packages/29/9c/f9f5277b95184f764b24e7231e166dfdb5780a46d408a2ac665969416d61/coverage-7.13.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:972a9cd27894afe4bc2b1480107054e062df08e671df7c2f18c205e805ccd806", size = 261912, upload-time = "2026-03-17T10:31:41.324Z" }, - { url = "https://files.pythonhosted.org/packages/d5/f6/7f1ab39393eeb50cfe4747ae8ef0e4fc564b989225aa1152e13a180d74f8/coverage-7.13.5-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4b59148601efcd2bac8c4dbf1f0ad6391693ccf7a74b8205781751637076aee3", size = 263987, upload-time = "2026-03-17T10:31:43.724Z" }, - { url = "https://files.pythonhosted.org/packages/a0/d7/62c084fb489ed9c6fbdf57e006752e7c516ea46fd690e5ed8b8617c7d52e/coverage-7.13.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:505d7083c8b0c87a8fa8c07370c285847c1f77739b22e299ad75a6af6c32c5c9", size = 266416, upload-time = "2026-03-17T10:31:45.769Z" }, - { url = "https://files.pythonhosted.org/packages/a9/f6/df63d8660e1a0bff6125947afda112a0502736f470d62ca68b288ea762d8/coverage-7.13.5-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:60365289c3741e4db327e7baff2a4aaacf22f788e80fa4683393891b70a89fbd", size = 267558, upload-time = "2026-03-17T10:31:48.293Z" }, - { url = "https://files.pythonhosted.org/packages/5b/02/353ca81d36779bd108f6d384425f7139ac3c58c750dcfaafe5d0bee6436b/coverage-7.13.5-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b88c69c8ef5d4b6fe7dea66d6636056a0f6a7527c440e890cf9259011f5e606", size = 261163, upload-time = "2026-03-17T10:31:50.125Z" }, - { url = "https://files.pythonhosted.org/packages/2c/16/2e79106d5749bcaf3aee6d309123548e3276517cd7851faa8da213bc61bf/coverage-7.13.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5b13955d31d1633cf9376908089b7cebe7d15ddad7aeaabcbe969a595a97e95e", size = 263981, upload-time = "2026-03-17T10:31:51.961Z" }, - { url = "https://files.pythonhosted.org/packages/29/c7/c29e0c59ffa6942030ae6f50b88ae49988e7e8da06de7ecdbf49c6d4feae/coverage-7.13.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f70c9ab2595c56f81a89620e22899eea8b212a4041bd728ac6f4a28bf5d3ddd0", size = 261604, upload-time = "2026-03-17T10:31:53.872Z" }, - { url = "https://files.pythonhosted.org/packages/40/48/097cdc3db342f34006a308ab41c3a7c11c3f0d84750d340f45d88a782e00/coverage-7.13.5-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:084b84a8c63e8d6fc7e3931b316a9bcafca1458d753c539db82d31ed20091a87", size = 265321, upload-time = "2026-03-17T10:31:55.997Z" }, - { url = "https://files.pythonhosted.org/packages/bb/1f/4994af354689e14fd03a75f8ec85a9a68d94e0188bbdab3fc1516b55e512/coverage-7.13.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad14385487393e386e2ea988b09d62dd42c397662ac2dabc3832d71253eee479", size = 260502, upload-time = "2026-03-17T10:31:58.308Z" }, - { url = "https://files.pythonhosted.org/packages/22/c6/9bb9ef55903e628033560885f5c31aa227e46878118b63ab15dc7ba87797/coverage-7.13.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f2c47b36fe7709a6e83bfadf4eefb90bd25fbe4014d715224c4316f808e59a2", size = 262688, upload-time = "2026-03-17T10:32:00.141Z" }, - { url = "https://files.pythonhosted.org/packages/14/4f/f5df9007e50b15e53e01edea486814783a7f019893733d9e4d6caad75557/coverage-7.13.5-cp313-cp313t-win32.whl", hash = "sha256:67e9bc5449801fad0e5dff329499fb090ba4c5800b86805c80617b4e29809b2a", size = 222788, upload-time = "2026-03-17T10:32:02.246Z" }, - { url = "https://files.pythonhosted.org/packages/e1/98/aa7fccaa97d0f3192bec013c4e6fd6d294a6ed44b640e6bb61f479e00ed5/coverage-7.13.5-cp313-cp313t-win_amd64.whl", hash = "sha256:da86cdcf10d2519e10cabb8ac2de03da1bcb6e4853790b7fbd48523332e3a819", size = 223851, upload-time = "2026-03-17T10:32:04.416Z" }, - { url = "https://files.pythonhosted.org/packages/3d/8b/e5c469f7352651e5f013198e9e21f97510b23de957dd06a84071683b4b60/coverage-7.13.5-cp313-cp313t-win_arm64.whl", hash = "sha256:0ecf12ecb326fe2c339d93fc131816f3a7367d223db37817208905c89bded911", size = 222104, upload-time = "2026-03-17T10:32:06.65Z" }, - { url = "https://files.pythonhosted.org/packages/8e/77/39703f0d1d4b478bfd30191d3c14f53caf596fac00efb3f8f6ee23646439/coverage-7.13.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fbabfaceaeb587e16f7008f7795cd80d20ec548dc7f94fbb0d4ec2e038ce563f", size = 219621, upload-time = "2026-03-17T10:32:08.589Z" }, - { url = "https://files.pythonhosted.org/packages/e2/3e/51dff36d99ae14639a133d9b164d63e628532e2974d8b1edb99dd1ebc733/coverage-7.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9bb2a28101a443669a423b665939381084412b81c3f8c0fcfbac57f4e30b5b8e", size = 219953, upload-time = "2026-03-17T10:32:10.507Z" }, - { url = "https://files.pythonhosted.org/packages/6a/6c/1f1917b01eb647c2f2adc9962bd66c79eb978951cab61bdc1acab3290c07/coverage-7.13.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3a2fbc1c6cccb3c5106140d87cc6a8715110373ef42b63cf5aea29df8c217a", size = 250992, upload-time = "2026-03-17T10:32:12.41Z" }, - { url = "https://files.pythonhosted.org/packages/22/e5/06b1f88f42a5a99df42ce61208bdec3bddb3d261412874280a19796fc09c/coverage-7.13.5-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6c36ddb64ed9d7e496028d1d00dfec3e428e0aabf4006583bb1839958d280510", size = 253503, upload-time = "2026-03-17T10:32:14.449Z" }, - { url = "https://files.pythonhosted.org/packages/80/28/2a148a51e5907e504fa7b85490277734e6771d8844ebcc48764a15e28155/coverage-7.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:380e8e9084d8eb38db3a9176a1a4f3c0082c3806fa0dc882d1d87abc3c789247", size = 254852, upload-time = "2026-03-17T10:32:16.56Z" }, - { url = "https://files.pythonhosted.org/packages/61/77/50e8d3d85cc0b7ebe09f30f151d670e302c7ff4a1bf6243f71dd8b0981fa/coverage-7.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e808af52a0513762df4d945ea164a24b37f2f518cbe97e03deaa0ee66139b4d6", size = 257161, upload-time = "2026-03-17T10:32:19.004Z" }, - { url = "https://files.pythonhosted.org/packages/3b/c4/b5fd1d4b7bf8d0e75d997afd3925c59ba629fc8616f1b3aae7605132e256/coverage-7.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e301d30dd7e95ae068671d746ba8c34e945a82682e62918e41b2679acd2051a0", size = 251021, upload-time = "2026-03-17T10:32:21.344Z" }, - { url = "https://files.pythonhosted.org/packages/f8/66/6ea21f910e92d69ef0b1c3346ea5922a51bad4446c9126db2ae96ee24c4c/coverage-7.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:800bc829053c80d240a687ceeb927a94fd108bbdc68dfbe505d0d75ab578a882", size = 252858, upload-time = "2026-03-17T10:32:23.506Z" }, - { url = "https://files.pythonhosted.org/packages/9e/ea/879c83cb5d61aa2a35fb80e72715e92672daef8191b84911a643f533840c/coverage-7.13.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:0b67af5492adb31940ee418a5a655c28e48165da5afab8c7fa6fd72a142f8740", size = 250823, upload-time = "2026-03-17T10:32:25.516Z" }, - { url = "https://files.pythonhosted.org/packages/8a/fb/616d95d3adb88b9803b275580bdeee8bd1b69a886d057652521f83d7322f/coverage-7.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9136ff29c3a91e25b1d1552b5308e53a1e0653a23e53b6366d7c2dcbbaf8a16", size = 255099, upload-time = "2026-03-17T10:32:27.944Z" }, - { url = "https://files.pythonhosted.org/packages/1c/93/25e6917c90ec1c9a56b0b26f6cad6408e5f13bb6b35d484a0d75c9cf000d/coverage-7.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cff784eef7f0b8f6cb28804fbddcfa99f89efe4cc35fb5627e3ac58f91ed3ac0", size = 250638, upload-time = "2026-03-17T10:32:29.914Z" }, - { url = "https://files.pythonhosted.org/packages/fc/7b/dc1776b0464145a929deed214aef9fb1493f159b59ff3c7eeeedf91eddd0/coverage-7.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:68a4953be99b17ac3c23b6efbc8a38330d99680c9458927491d18700ef23ded0", size = 252295, upload-time = "2026-03-17T10:32:31.981Z" }, - { url = "https://files.pythonhosted.org/packages/ea/fb/99cbbc56a26e07762a2740713f3c8f9f3f3106e3a3dd8cc4474954bccd34/coverage-7.13.5-cp314-cp314-win32.whl", hash = "sha256:35a31f2b1578185fbe6aa2e74cea1b1d0bbf4c552774247d9160d29b80ed56cc", size = 222360, upload-time = "2026-03-17T10:32:34.233Z" }, - { url = "https://files.pythonhosted.org/packages/8d/b7/4758d4f73fb536347cc5e4ad63662f9d60ba9118cb6785e9616b2ce5d7fa/coverage-7.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:2aa055ae1857258f9e0045be26a6d62bdb47a72448b62d7b55f4820f361a2633", size = 223174, upload-time = "2026-03-17T10:32:36.369Z" }, - { url = "https://files.pythonhosted.org/packages/2c/f2/24d84e1dfe70f8ac9fdf30d338239860d0d1d5da0bda528959d0ebc9da28/coverage-7.13.5-cp314-cp314-win_arm64.whl", hash = "sha256:1b11eef33edeae9d142f9b4358edb76273b3bfd30bc3df9a4f95d0e49caf94e8", size = 221739, upload-time = "2026-03-17T10:32:38.736Z" }, - { url = "https://files.pythonhosted.org/packages/60/5b/4a168591057b3668c2428bff25dd3ebc21b629d666d90bcdfa0217940e84/coverage-7.13.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10a0c37f0b646eaff7cce1874c31d1f1ccb297688d4c747291f4f4c70741cc8b", size = 220351, upload-time = "2026-03-17T10:32:41.196Z" }, - { url = "https://files.pythonhosted.org/packages/f5/21/1fd5c4dbfe4a58b6b99649125635df46decdfd4a784c3cd6d410d303e370/coverage-7.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b5db73ba3c41c7008037fa731ad5459fc3944cb7452fc0aa9f822ad3533c583c", size = 220612, upload-time = "2026-03-17T10:32:43.204Z" }, - { url = "https://files.pythonhosted.org/packages/d6/fe/2a924b3055a5e7e4512655a9d4609781b0d62334fa0140c3e742926834e2/coverage-7.13.5-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:750db93a81e3e5a9831b534be7b1229df848b2e125a604fe6651e48aa070e5f9", size = 261985, upload-time = "2026-03-17T10:32:45.514Z" }, - { url = "https://files.pythonhosted.org/packages/d7/0d/c8928f2bd518c45990fe1a2ab8db42e914ef9b726c975facc4282578c3eb/coverage-7.13.5-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ddb4f4a5479f2539644be484da179b653273bca1a323947d48ab107b3ed1f29", size = 264107, upload-time = "2026-03-17T10:32:47.971Z" }, - { url = "https://files.pythonhosted.org/packages/ef/ae/4ae35bbd9a0af9d820362751f0766582833c211224b38665c0f8de3d487f/coverage-7.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8a7a2049c14f413163e2bdabd37e41179b1d1ccb10ffc6ccc4b7a718429c607", size = 266513, upload-time = "2026-03-17T10:32:50.1Z" }, - { url = "https://files.pythonhosted.org/packages/9c/20/d326174c55af36f74eac6ae781612d9492f060ce8244b570bb9d50d9d609/coverage-7.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1c85e0b6c05c592ea6d8768a66a254bfb3874b53774b12d4c89c481eb78cb90", size = 267650, upload-time = "2026-03-17T10:32:52.391Z" }, - { url = "https://files.pythonhosted.org/packages/7a/5e/31484d62cbd0eabd3412e30d74386ece4a0837d4f6c3040a653878bfc019/coverage-7.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:777c4d1eff1b67876139d24288aaf1817f6c03d6bae9c5cc8d27b83bcfe38fe3", size = 261089, upload-time = "2026-03-17T10:32:54.544Z" }, - { url = "https://files.pythonhosted.org/packages/e9/d8/49a72d6de146eebb0b7e48cc0f4bc2c0dd858e3d4790ab2b39a2872b62bd/coverage-7.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6697e29b93707167687543480a40f0db8f356e86d9f67ddf2e37e2dfd91a9dab", size = 263982, upload-time = "2026-03-17T10:32:56.803Z" }, - { url = "https://files.pythonhosted.org/packages/06/3b/0351f1bd566e6e4dd39e978efe7958bde1d32f879e85589de147654f57bb/coverage-7.13.5-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8fdf453a942c3e4d99bd80088141c4c6960bb232c409d9c3558e2dbaa3998562", size = 261579, upload-time = "2026-03-17T10:32:59.466Z" }, - { url = "https://files.pythonhosted.org/packages/5d/ce/796a2a2f4017f554d7810f5c573449b35b1e46788424a548d4d19201b222/coverage-7.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:32ca0c0114c9834a43f045a87dcebd69d108d8ffb666957ea65aa132f50332e2", size = 265316, upload-time = "2026-03-17T10:33:01.847Z" }, - { url = "https://files.pythonhosted.org/packages/3d/16/d5ae91455541d1a78bc90abf495be600588aff8f6db5c8b0dae739fa39c9/coverage-7.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8769751c10f339021e2638cd354e13adeac54004d1941119b2c96fe5276d45ea", size = 260427, upload-time = "2026-03-17T10:33:03.945Z" }, - { url = "https://files.pythonhosted.org/packages/48/11/07f413dba62db21fb3fad5d0de013a50e073cc4e2dc4306e770360f6dfc8/coverage-7.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cec2d83125531bd153175354055cdb7a09987af08a9430bd173c937c6d0fba2a", size = 262745, upload-time = "2026-03-17T10:33:06.285Z" }, - { url = "https://files.pythonhosted.org/packages/91/15/d792371332eb4663115becf4bad47e047d16234b1aff687b1b18c58d60ae/coverage-7.13.5-cp314-cp314t-win32.whl", hash = "sha256:0cd9ed7a8b181775459296e402ca4fb27db1279740a24e93b3b41942ebe4b215", size = 223146, upload-time = "2026-03-17T10:33:08.756Z" }, - { url = "https://files.pythonhosted.org/packages/db/51/37221f59a111dca5e85be7dbf09696323b5b9f13ff65e0641d535ed06ea8/coverage-7.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:301e3b7dfefecaca37c9f1aa6f0049b7d4ab8dd933742b607765d757aca77d43", size = 224254, upload-time = "2026-03-17T10:33:11.174Z" }, - { url = "https://files.pythonhosted.org/packages/54/83/6acacc889de8987441aa7d5adfbdbf33d288dad28704a67e574f1df9bcbb/coverage-7.13.5-cp314-cp314t-win_arm64.whl", hash = "sha256:9dacc2ad679b292709e0f5fc1ac74a6d4d5562e424058962c7bb0c658ad25e45", size = 222276, upload-time = "2026-03-17T10:33:13.466Z" }, - { url = "https://files.pythonhosted.org/packages/9e/ee/a4cf96b8ce1e566ed238f0659ac2d3f007ed1d14b181bcb684e19561a69a/coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61", size = 211346, upload-time = "2026-03-17T10:33:15.691Z" }, + { url = "https://files.pythonhosted.org/packages/8a/9e/5f6d56327c62b185225d145191c607e07515294a0aa6338e58805cd4a5ac/coverage-7.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:be9f2c802dcfce3f71298303aa5dad0dce440a76c52f2f60dacd8656dab78793", size = 220044, upload-time = "2026-05-26T20:39:29.902Z" }, + { url = "https://files.pythonhosted.org/packages/75/92/e82aca356744cbbc0f77a0b623e38918c1872361963413a3bab5d0340393/coverage-7.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6223a72fd0e4c7156353ec0f08a5f93623e1d3034d0e2683b9bb8ea674131b1d", size = 220412, upload-time = "2026-05-26T20:39:31.561Z" }, + { url = "https://files.pythonhosted.org/packages/27/c9/385bde0bf7ed0f4bf3a7ee5367060a86b5d218718cfd6fb943c0f836b34f/coverage-7.14.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7279d2110a28cebc738b6459ecda2771735a4c18465fbbd36b3288fe5ed92247", size = 251412, upload-time = "2026-05-26T20:39:33.337Z" }, + { url = "https://files.pythonhosted.org/packages/51/8c/23faf6a2343a0d17f960a4bd56c43bc7eb4cf312f774dd6ceebd82c7d8fc/coverage-7.14.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9eeb3fcbc13ba40dfbdb22d01d196a28e9cef9ed4c29b60061a1e0e823a9929d", size = 254008, upload-time = "2026-05-26T20:39:35.009Z" }, + { url = "https://files.pythonhosted.org/packages/42/06/36f4aa9ca8a815e6036156e80706a67828bb97bd826948244f6996dda957/coverage-7.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f0cfc27c539f07cf5c0a4cfe211d0b6cae039f8f40526dbaa71944e64b50a7b", size = 255241, upload-time = "2026-05-26T20:39:36.71Z" }, + { url = "https://files.pythonhosted.org/packages/ca/79/95266316352f90f6b1c6736bb413302edfde2453fb32422d3911642691b3/coverage-7.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:221c70f316241a78e77e607c227cefc8808d4e08f28d99c04f35694690e940be", size = 257373, upload-time = "2026-05-26T20:39:38.412Z" }, + { url = "https://files.pythonhosted.org/packages/e3/9c/58316d1f66c488b5fca8a0eb3e98348807813efa8a0d0833b9021be27488/coverage-7.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:da028256b04ec30e5e0114b6f76172938c313991f0a2d3d894271315cf5d5e43", size = 251635, upload-time = "2026-05-26T20:39:40.268Z" }, + { url = "https://files.pythonhosted.org/packages/ef/5a/ca2398a568e16fed7bb713e84ba3603a7164fb65779abe645c565ec890d5/coverage-7.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:76a085d7005236a767e3426148b2c407e53ad61695c562f8a81da2d373324901", size = 253373, upload-time = "2026-05-26T20:39:42.145Z" }, + { url = "https://files.pythonhosted.org/packages/6e/2c/0396562c32deaebe7be51d865b3a41e9a87d7561acafe1a28f53b07e019a/coverage-7.14.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b553d04b5e778a8e56d57eb134aff42a92718ecba45e79c4764ecfa40efd92ff", size = 251341, upload-time = "2026-05-26T20:39:43.907Z" }, + { url = "https://files.pythonhosted.org/packages/fd/8f/a94f9221184c9cae1ee115820e3798e48b6b17777a9f19e46fb9a0c8dc74/coverage-7.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:46f714d2fb8ae2f4f29f23ada7f1e79b759fff5a70f94a1dac23af204c3ec9e4", size = 255497, upload-time = "2026-05-26T20:39:46.166Z" }, + { url = "https://files.pythonhosted.org/packages/71/69/505d70e47db1eaebcd002c39759707621ef184cd6b1ae084d9f41293f323/coverage-7.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:1896f5e19ff3f0431c7ce2172adc54890fd97f86b59ced8ca1649145d9ffe35d", size = 251159, upload-time = "2026-05-26T20:39:48.03Z" }, + { url = "https://files.pythonhosted.org/packages/e0/aa/58681c383aa33a9d2ed40a02d7a22fbf780d1fa4d575396365777828198c/coverage-7.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:62fd185ef9df3c33d1c8178c5af105f762afbad96038de9a4ae100aa6297ca33", size = 252934, upload-time = "2026-05-26T20:39:49.872Z" }, + { url = "https://files.pythonhosted.org/packages/eb/fd/11c928cd6bdffc7074bb5965c173d9ebf517fb00205e1da524b98d29ef92/coverage-7.14.1-cp313-cp313-win32.whl", hash = "sha256:ab4af6352741a604c431c6072fce5bee33bf0f20dc7a56618d6bf6bb89e9810c", size = 222584, upload-time = "2026-05-26T20:39:51.68Z" }, + { url = "https://files.pythonhosted.org/packages/6f/92/fb416fc26d340dcba19518c418d6048e913186e17243982c5e435e41fa7a/coverage-7.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:7af486dabe8954d03b087f0021540897afe084f04e16ff5579e08cc46f871416", size = 223394, upload-time = "2026-05-26T20:39:53.472Z" }, + { url = "https://files.pythonhosted.org/packages/73/c6/02d56e3867972f77d5036de924643f26c056e848f00452cafb4dbc3c29b4/coverage-7.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:2224f89ffd0c5605ccce1ed7a584da162bc7c55f601ab1c946bc9de31a486b42", size = 222015, upload-time = "2026-05-26T20:39:55.374Z" }, + { url = "https://files.pythonhosted.org/packages/4d/9e/fcc77914050df73f7662fa1f00902774c79c075a8388ab334074574bf77e/coverage-7.14.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:de286598cc65d2b489411174b1faec2f5a7775fb3201fd925db2a76b4030f37d", size = 220733, upload-time = "2026-05-26T20:39:57.189Z" }, + { url = "https://files.pythonhosted.org/packages/f7/67/2963cbdaf5cbadec44efa3a1e39eaa1f02df4079585f05387607a221e126/coverage-7.14.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:042c46ded7c288aeb07cf14a28b6c1e10b78fcba40171c3fa1e939377eeef0b5", size = 221086, upload-time = "2026-05-26T20:39:59.019Z" }, + { url = "https://files.pythonhosted.org/packages/c8/c5/8701645574e11881f2f47d8930f98bc48b5d43b25eb5b4430dfc4a2f9f48/coverage-7.14.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f4ddbe407477f04c45115d1a4e5bc480f753553b534d338d4c3358b1cdd0ea52", size = 262381, upload-time = "2026-05-26T20:40:00.822Z" }, + { url = "https://files.pythonhosted.org/packages/7c/28/7a64d73598263e0c5abd5084211a8474488d31b3c552ff531c719dfcff62/coverage-7.14.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d13e6725992e2d2fd7d81d4f5241952d13740121dfd501da09201be39b2c003a", size = 264458, upload-time = "2026-05-26T20:40:02.506Z" }, + { url = "https://files.pythonhosted.org/packages/fa/d8/4969179db9f7eb4df218e69540adf829d1c835f59452513d065d15446802/coverage-7.14.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f747dc8edcfe740130f28f32f3995e955494285717e86ee25af51db2219df08a", size = 266884, upload-time = "2026-05-26T20:40:04.421Z" }, + { url = "https://files.pythonhosted.org/packages/a6/78/a45d5794dbc9bafd97afc96a4377c86c7820d78b6cf51b89bc1d4e919275/coverage-7.14.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ced2f09ef276fd58611a1ef502164ad266d2b75174e5a40cabbdb4033f9f6cf2", size = 268022, upload-time = "2026-05-26T20:40:06.298Z" }, + { url = "https://files.pythonhosted.org/packages/21/cb/4f5e354e9e3e67af96bd4e57113e6db6b22298c7168b13eec408a549903d/coverage-7.14.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b84800013769a78ccb9ef4659402e26d06867e337b61ec365f77ad008adea80e", size = 261631, upload-time = "2026-05-26T20:40:08.226Z" }, + { url = "https://files.pythonhosted.org/packages/ec/49/eced49af4cb996d5d8b7e94e736175c513e4facd3398507b89892b4326d8/coverage-7.14.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ea8cd6ca0ee9f616aaef3afc6882e32c2cbf18b00d96313ffd76af650574034d", size = 264443, upload-time = "2026-05-26T20:40:10.137Z" }, + { url = "https://files.pythonhosted.org/packages/f1/d8/5603a88a7c5913a6b54f6cb1a8c46f7b39cbb30f27cd3f492908da09b2d7/coverage-7.14.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:aa5e304a873fabddc11e484e9b6b738bd38bd7bed17b09aa84eecf5332e8b8bb", size = 262069, upload-time = "2026-05-26T20:40:11.999Z" }, + { url = "https://files.pythonhosted.org/packages/f0/59/2ae3cb79da554a06c8619d6c88ea19dd1e4aed4b834b6a83bb1fa243bdc5/coverage-7.14.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:5a1c5215be81035e629d5bc756650634d0bf31991038db7a0eccb90f025ce16d", size = 265780, upload-time = "2026-05-26T20:40:13.858Z" }, + { url = "https://files.pythonhosted.org/packages/af/5f/b130c1dc999031f2648bd25317fbce505ad8d5562079b4ed81e736a84967/coverage-7.14.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:79058c47dae6788504b5effb319961bcd72d7240551464b91d474bc0ed186d69", size = 260970, upload-time = "2026-05-26T20:40:16.142Z" }, + { url = "https://files.pythonhosted.org/packages/87/d1/ec13ccddeb48ec963bdfa72a11224bac2584bd045ba13beca82f8113e9c7/coverage-7.14.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:370c5afae3fa0658e11694a32b24c2778f6bc2d17718121f94ee185e69f26b54", size = 263157, upload-time = "2026-05-26T20:40:18.382Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c2/cd91ead503045161092d3845f7bb95ea2f25131ce96d3e314dd835d91b9c/coverage-7.14.1-cp313-cp313t-win32.whl", hash = "sha256:3758dd0a7f1fa57365ef2e781df0f0731d38b6e3772259d13dae4bd8a958d4b1", size = 223259, upload-time = "2026-05-26T20:40:20.381Z" }, + { url = "https://files.pythonhosted.org/packages/71/9f/1e28d97e6bd2c76b07f38b7c02870f1371255ff6717f54eca578fcbbdd0e/coverage-7.14.1-cp313-cp313t-win_amd64.whl", hash = "sha256:6ff665fb023a77386fe11685190cee1f60a7d635994a30d9b0a061533d470fce", size = 224320, upload-time = "2026-05-26T20:40:22.316Z" }, + { url = "https://files.pythonhosted.org/packages/a9/e0/d936e908f0e1efa55e52b91e01b52f1055cef5e1ab2718493390ed8e2fb8/coverage-7.14.1-cp313-cp313t-win_arm64.whl", hash = "sha256:17a5a241e5997621a956a7f402a7433ef4221e5152809b785bec79e2323799f1", size = 222577, upload-time = "2026-05-26T20:40:24.894Z" }, + { url = "https://files.pythonhosted.org/packages/d6/34/fc2f101b151af3799a101f0550b0454aa008afdc0add677394ec4aa8ea10/coverage-7.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d5ed429d0b8edaac649e889b4ffcedb6c80b06629a3f93050e3dddfb99235bee", size = 220091, upload-time = "2026-05-26T20:40:27.249Z" }, + { url = "https://files.pythonhosted.org/packages/3d/a7/1ebae2ab5b961b5c79bb09fe7b3ac99edb190d8be4a8c510b2cf66f46468/coverage-7.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8011224a62280e50dab346960c03cf47aca1a1e09e608c0fb33fd6e0cc8e9500", size = 220421, upload-time = "2026-05-26T20:40:30.084Z" }, + { url = "https://files.pythonhosted.org/packages/5e/90/92aca9cf0acc95123c96cd1eb1f08917897a7f5dee01e15738922971ec31/coverage-7.14.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:12c42ec1e14f553c4f817e989365982e646e27211f10a0f717855b94a79c8906", size = 251466, upload-time = "2026-05-26T20:40:32.542Z" }, + { url = "https://files.pythonhosted.org/packages/26/2b/78048cbe3b999f6cbf9cc0d90abba6a88a3e0863a8c1c6cbc762f3f8802f/coverage-7.14.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:06144cd511cf2624873a035c5069cf297144f6e77a73ee3d7a55b605ec5efb42", size = 253973, upload-time = "2026-05-26T20:40:34.473Z" }, + { url = "https://files.pythonhosted.org/packages/8e/21/c2e33b29d1cfde484a19d437afc343c6cd30b08d78cbbf9f5aff14e57b2b/coverage-7.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a311d8e1da24be5c1ccf85cbfb06315dbaa1703d5a1eab3f6432c72b837917c8", size = 255318, upload-time = "2026-05-26T20:40:38.154Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ee/aad2f108d63b769121005302f16bf66db8625c88ceaba466942e09a2607e/coverage-7.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c79cead5b5bc584d9c71451cb984d0e3a84e0c0937379c8efcbf27c8d661b851", size = 257633, upload-time = "2026-05-26T20:40:40.164Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f8/11a2c29b4fd76d9849f81d0bb812ec0017a9396df3217214e38934a8c837/coverage-7.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dcbf65f1f66a26cdd88c35cf68fb4729c5d1cd2e88added72420541dfb212034", size = 251488, upload-time = "2026-05-26T20:40:42.631Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b8/9a5820de4b8ac2b71d85e3b5fb49108d7469c665f0e2ad0dd7569023e305/coverage-7.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fd86572566fb40189a8260446158235159bc7a82dfbc87a3b39cf4fb57fcec1c", size = 253329, upload-time = "2026-05-26T20:40:45.208Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ff/f33e4823667e27548e8fd8df44217515303f9808d0ff29817db56f87d990/coverage-7.14.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:7771b601718fdde84832c3a434ca9bbf4ae9adbc49d84198b4110700c3c77c36", size = 251291, upload-time = "2026-05-26T20:40:47.502Z" }, + { url = "https://files.pythonhosted.org/packages/68/9b/489db0ebb209054766b90a9014a45f6d26eb724c02ec21311c3733b5a644/coverage-7.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:39b21e212c55af06fa375e3dbf90a8a8e38792f3a910c580066d23563830ddd5", size = 255564, upload-time = "2026-05-26T20:40:49.372Z" }, + { url = "https://files.pythonhosted.org/packages/27/b5/16bc2d4c2409b23c7737edb68c83bc89e345f378050549fe1d75ac7d34d5/coverage-7.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f2302660e32562a532b442480121aef8aa61a5bdb20b30bf0adab29f10a5a4b4", size = 251107, upload-time = "2026-05-26T20:40:51.677Z" }, + { url = "https://files.pythonhosted.org/packages/7d/0c/2629997469a00cd069d588a41c9dc887610f2775ae89d250c4791e65272a/coverage-7.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03a6f93c1ec3b7f2e77b5dbcc5573a2c21f12529a5c6bbe0f16f72303cc2fa4d", size = 252764, upload-time = "2026-05-26T20:40:54.267Z" }, + { url = "https://files.pythonhosted.org/packages/d2/ee/f78d63c8f079e0d7211c7e2401fa17e311514534ba61bae03e4b287ce4ab/coverage-7.14.1-cp314-cp314-win32.whl", hash = "sha256:8a3ce026d73290f42f08dafecbd82c193a74df280461fbf97300fec51fd133ee", size = 222837, upload-time = "2026-05-26T20:40:56.496Z" }, + { url = "https://files.pythonhosted.org/packages/dc/b9/be539854f93a70dfbeec69117f33ec70dc42ff0b65b5b07ab8d40d04228e/coverage-7.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:114c95ef29302423b87d159075805f4ab973254a2638a5d7d046c94887cc87d7", size = 223650, upload-time = "2026-05-26T20:40:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/fe/9e/24e2842fef40f35ac82ba3a7719c8023d011bf3bf652d0675316a9d088a1/coverage-7.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:a07891c3f4805442b31b71e84ba3cf29ed1aa9a428284e06deeb4b23e5b46343", size = 222218, upload-time = "2026-05-26T20:41:00.321Z" }, + { url = "https://files.pythonhosted.org/packages/0a/1d/ac0a9df5fe31c1e8bdd658074905fc12844a05c1a7e3fdb8417e97c31e23/coverage-7.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:1101a5ebb083aecb625ebb6209d4105b58f647b093cb2dc8122d7b33f743cfe1", size = 220822, upload-time = "2026-05-26T20:41:02.281Z" }, + { url = "https://files.pythonhosted.org/packages/32/cf/f964fd9aff20323f9f1a726c97135f8a76bcd87b92dad141a456a43f3c64/coverage-7.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:851b9e1e4e8a4608e77c79714b2e77c0970d2ed7202a05e92ae407817481887b", size = 221084, upload-time = "2026-05-26T20:41:04.593Z" }, + { url = "https://files.pythonhosted.org/packages/d8/5e/7e5ef2aba844de2b80d678619fcf0841b42e3f37f16411226f3fe4c1016f/coverage-7.14.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d5b89cdfb2ee051b71e8c3c70bd81a9eff81100f736a269136fe1a68efe00474", size = 262454, upload-time = "2026-05-26T20:41:06.641Z" }, + { url = "https://files.pythonhosted.org/packages/64/62/75809bded87015cc4935524218a2a8ed8dd1a8498bfed30a2f4f7a4b4d34/coverage-7.14.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0177614a0370f227888b4e436a7c55686d6a9f90eb1ade2b624ba685a1686e86", size = 264578, upload-time = "2026-05-26T20:41:08.556Z" }, + { url = "https://files.pythonhosted.org/packages/f3/42/d33392dc14633525012d2d504fa1a33b05538bf535f5c1d64675e5754b78/coverage-7.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2d69af5dea2de76fc485a83032a630523f985198b7e25be901ec60181587b01e", size = 266981, upload-time = "2026-05-26T20:41:10.824Z" }, + { url = "https://files.pythonhosted.org/packages/2a/49/0157c4428c2aca7f1e09d5565930586fd5ae36f1655f08b0daa7cf1fcae1/coverage-7.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:35ab22d91de736e8966b980dc355cbcdd2c6dbbcfe275f9a2991bc8a91b3df65", size = 268112, upload-time = "2026-05-26T20:41:12.966Z" }, + { url = "https://files.pythonhosted.org/packages/96/26/86b9ce71f4092b1ed325ce1421698081df1286b833400b6836912834d6e0/coverage-7.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:357d4e32935c36588aaba057d734fa32428c360c9fc2e4442afbf1b646beee6e", size = 261558, upload-time = "2026-05-26T20:41:15Z" }, + { url = "https://files.pythonhosted.org/packages/20/4c/c311210c5472cf5401d8422b0d7812cdd520f24417673afabda6c323faca/coverage-7.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:51bd64741cc6fa065abd300ede1afe5a5291ece9c31da8b24884deda48bcc3f8", size = 264447, upload-time = "2026-05-26T20:41:17.369Z" }, + { url = "https://files.pythonhosted.org/packages/fb/71/59513f8710ed3e6b0ac0a050a5b7e977bb9c9e880354863b5d00d8809256/coverage-7.14.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:9132cd363a68a4c3daa7c8704a654b1e39d3360f6f5b8ddd470608a945236c07", size = 262048, upload-time = "2026-05-26T20:41:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/84/8d/bceed32dc494f5bbf50f775cd2e78ca814953942b5ea28d3c1c3ac316f14/coverage-7.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:07c6290b1697b862c0478eab545eec949a0d0e4d6d03497f446d706da3b4f2de", size = 265781, upload-time = "2026-05-26T20:41:21.559Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c5/9348fe40dbfd4991aaf78df2c6c3098bfb2cc834d1fd362a64b4efef855a/coverage-7.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5ea0c297e27133853b4d8a3eb799bff5a2dbd9f2f41537a240d337ac9b4df890", size = 260896, upload-time = "2026-05-26T20:41:23.428Z" }, + { url = "https://files.pythonhosted.org/packages/ca/92/1ea0f03929da7cf87206b1fa24f4c8e9c158be0455481af29ec0a1f3503f/coverage-7.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:01b7733daad0237daa01ef80fe2dfceffc911e6a17fa7b55d14aa8214eaaaecd", size = 263214, upload-time = "2026-05-26T20:41:25.419Z" }, + { url = "https://files.pythonhosted.org/packages/f6/a9/b2493c054c0e01a643266742ab45e15744e60743f9260cd930c7142b1124/coverage-7.14.1-cp314-cp314t-win32.whl", hash = "sha256:6adc5a36984624a70bf11d7184e20fa0a49aa7c47ffab43804106a1a695ea22e", size = 223624, upload-time = "2026-05-26T20:41:27.795Z" }, + { url = "https://files.pythonhosted.org/packages/fc/bd/3e1e6a57fccd2d7c83fcdf338e93ba98eb85c6e877dd34731ac585375490/coverage-7.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:ddf799247318f34dbcd2efa8c95a8d0642674e926bb1774cf9b63dfd2a389d1c", size = 224728, upload-time = "2026-05-26T20:41:30.098Z" }, + { url = "https://files.pythonhosted.org/packages/bb/d7/31066cf1d2f0c6c797fce911bcfa01dd35642dc6da992a950256097c5860/coverage-7.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:145986fe66647eb489f18d9a997567a3fd358584c4b5a808769113abc07466af", size = 222752, upload-time = "2026-05-26T20:41:32.123Z" }, + { url = "https://files.pythonhosted.org/packages/8a/3c/1a983b9a745d7f83d53f057bcc5bf79ba6a2bbc08266b3f0c7d6fe630c9b/coverage-7.14.1-py3-none-any.whl", hash = "sha256:a252f21c27e38347e60111a3266b03827422a7d5525951aceee313aa68bab1d2", size = 211815, upload-time = "2026-05-26T20:41:34.078Z" }, ] [[package]] @@ -243,29 +243,29 @@ wheels = [ [[package]] name = "duckdb" -version = "1.5.2" +version = "1.5.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0c/66/744b4931b799a42f8cb9bc7a6f169e7b8e51195b62b246db407fd90bf15f/duckdb-1.5.2.tar.gz", hash = "sha256:638da0d5102b6cb6f7d47f83d0600708ac1d3cb46c5e9aaabc845f9ba4d69246", size = 18017166, upload-time = "2026-04-13T11:30:09.065Z" } +sdist = { url = "https://files.pythonhosted.org/packages/69/00/d579dcb2a536b6ea3a2563cdad6844f77d81a9b2d4b22a858097f2468acf/duckdb-1.5.3.tar.gz", hash = "sha256:df39428eb130faa35ae96fd35245bdeae6ecf43936250b116b5fead568eb9f16", size = 18026640, upload-time = "2026-05-20T11:55:31.901Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/f2/e3d742808f138d374be4bb516fade3d1f33749b813650810ab7885cdc363/duckdb-1.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4420b3f47027a7849d0e1815532007f377fa95ee5810b47ea717d35525c12f79", size = 30064879, upload-time = "2026-04-13T11:29:30.763Z" }, - { url = "https://files.pythonhosted.org/packages/72/0d/f3dc1cf97e1267ca15e4307d456f96ce583961f0703fd75e62b2ad8d64fa/duckdb-1.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:bb42e6ed543902e14eae647850da24103a89f0bc2587dec5601b1c1f213bd2ed", size = 15969327, upload-time = "2026-04-13T11:29:33.481Z" }, - { url = "https://files.pythonhosted.org/packages/b1/e0/d5418def53ae4e05a63075705ff44ed5af5a1a5932627eb2b600c5df1c93/duckdb-1.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:98c0535cd6d901f61a5ea3c2e26a1fd28482953d794deb183daf568e3aa5dda6", size = 14225107, upload-time = "2026-04-13T11:29:35.882Z" }, - { url = "https://files.pythonhosted.org/packages/16/a7/15aaa59dbecc35e9711980fcdbf525b32a52470b32d18ef678193a146213/duckdb-1.5.2-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:486c862bf7f163c0110b6d85b3e5c031d224a671cca468f12ebb1d3a348f6b39", size = 19313433, upload-time = "2026-04-13T11:29:38.367Z" }, - { url = "https://files.pythonhosted.org/packages/bd/21/d903cc63a5140c822b7b62b373a87dc557e60c29b321dfb435061c5e67cf/duckdb-1.5.2-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:70631c847ca918ee710ec874241b00cf9d2e5be90762cbb2a0389f17823c08f7", size = 21429837, upload-time = "2026-04-13T11:29:41.135Z" }, - { url = "https://files.pythonhosted.org/packages/e3/0a/b770d1f60c70597302130d6247f418549b7094251a02348fbaf1c7e147ae/duckdb-1.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:52a21823f3fbb52f0f0e5425e20b07391ad882464b955879499b5ff0b45a376b", size = 13107699, upload-time = "2026-04-13T11:29:43.905Z" }, - { url = "https://files.pythonhosted.org/packages/d9/cf/e200fe431d700962d1a908d2ce89f53ccee1cc8db260174ae663ba09686b/duckdb-1.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:411ad438bd4140f189a10e7f515781335962c5d18bd07837dc6d202e3985253d", size = 13927646, upload-time = "2026-04-13T11:29:46.598Z" }, - { url = "https://files.pythonhosted.org/packages/83/a1/f6286c67726cc1ea60a6e3c0d9fbc66527dde24ae089a51bbe298b13ca78/duckdb-1.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:6b0fe75c148000f060aa1a27b293cacc0ea08cc1cad724fbf2143d56070a3785", size = 30078598, upload-time = "2026-04-13T11:29:49.828Z" }, - { url = "https://files.pythonhosted.org/packages/de/6a/59febb02f21a4a5c6b0b0099ef7c965fdd5e61e4904cf813809bb792e35f/duckdb-1.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:35579b8e3a064b5eaf15b0eafc558056a13f79a0a62e34cc4baf57119daecfec", size = 15975120, upload-time = "2026-04-13T11:29:52.631Z" }, - { url = "https://files.pythonhosted.org/packages/09/70/ce750854d37bb5a45cccbb2c3cb04df4af56aea8fc30a2499bb643b4a9c0/duckdb-1.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ea58ff5b0880593a280cf5511734b17711b32ee1f58b47d726e8600848358160", size = 14227762, upload-time = "2026-04-13T11:29:55.564Z" }, - { url = "https://files.pythonhosted.org/packages/28/dc/ad45ac3c0b6c4687dc649e8f6cf01af1c8b0443932a39b2abb4ebcb3babd/duckdb-1.5.2-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef461bca07313412dc09961c4a4757a851f56b95ac01c58fac6007632b7b94f2", size = 19315668, upload-time = "2026-04-13T11:29:58.427Z" }, - { url = "https://files.pythonhosted.org/packages/cc/b1/1464f468d2e5813f5808de95df9d3113a645a5bfa2ffcaecbc542ddae272/duckdb-1.5.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be37680ddb380015cb37318e378c53511c45c4f0d8fac5599d22b7d092b9217a", size = 21434056, upload-time = "2026-04-13T11:30:01.238Z" }, - { url = "https://files.pythonhosted.org/packages/ce/32/6673607e024722473fa7aafdd29c0e3dd231dd528f6cd8b5797fbeeb229d/duckdb-1.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:0b291786014df1133f8f18b9df4d004484613146e858d71a21791e0fcca16cf4", size = 13633667, upload-time = "2026-04-13T11:30:04.05Z" }, - { url = "https://files.pythonhosted.org/packages/7a/e3/9d34173ec068631faea3ea6e73050700729363e7e33306a9a3218e5cdc61/duckdb-1.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:c9f3e0b71b8a50fccfb42794899285d9d318ce2503782b9dd54868e5ecd0ad31", size = 14402513, upload-time = "2026-04-13T11:30:06.609Z" }, + { url = "https://files.pythonhosted.org/packages/cc/9c/a528eb09d8be51954c485864bd06753e616939a080cbc3dd4417e8c94a57/duckdb-1.5.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e75a6122c12579a99848517f6f00a4e342aebda3590c30fe9b5cc5f39d5e6afc", size = 32626254, upload-time = "2026-05-20T11:54:53.65Z" }, + { url = "https://files.pythonhosted.org/packages/ec/3c/1534c0a6db347c05eb7d0f6ecfb7aefbe74cbff398e4892a8fd1903a20e8/duckdb-1.5.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fd3963c1cb9d9567777f4a898a9dbe388a2fe9724681801b1e7d6d93eecf1b76", size = 17300917, upload-time = "2026-05-20T11:54:56.628Z" }, + { url = "https://files.pythonhosted.org/packages/23/fa/beafb91e6e152d2161c4a9cbc472334c87607eb61ad7104b5a7fa8d8d7b1/duckdb-1.5.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3d5db8c0b55e072cf437948ebb5d7e23d7b9d03d905fa5f9145583e65aa447f7", size = 15449411, upload-time = "2026-05-20T11:54:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/50/0a/49b6fe04e2fcd63729eb607dadd44818dde77342a4f5ce086c6c92f1dd4d/duckdb-1.5.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ce80aed7a538422129a57eaca9141e3afb51f8bf562b1908b1576c9725b5b22", size = 19333120, upload-time = "2026-05-20T11:55:01.727Z" }, + { url = "https://files.pythonhosted.org/packages/63/4c/0907c3f76adb9dd90e67610b31e0304a35814e65c4c41a354a262c09b885/duckdb-1.5.3-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:787df63824f07bf18022dbc3b8ca4b2bfab0ebe616464f55c6e8cd0f59ea762e", size = 21453266, upload-time = "2026-05-20T11:55:04.5Z" }, + { url = "https://files.pythonhosted.org/packages/6d/9c/d2f23a7803ddbbd9413f7572ecf66a15120ed5ced7ce5c73e698c1406b76/duckdb-1.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:bb5bb5dcdd09d62ee60f0ddbbef918e71cce304ffe28428b1131949d39ffaabf", size = 13118640, upload-time = "2026-05-20T11:55:07.389Z" }, + { url = "https://files.pythonhosted.org/packages/27/d5/7ba2316415bcdab6edd765bbbe35c2ca8a3800f2fe695cd70e3cdb997f09/duckdb-1.5.3-cp313-cp313-win_arm64.whl", hash = "sha256:2fa17ecdd5d3db122836cb71bb93601c2106a3be883c17dffddc02fbf3fa7888", size = 13926409, upload-time = "2026-05-20T11:55:10.166Z" }, + { url = "https://files.pythonhosted.org/packages/a5/c2/d4b6f8a5e4d3bc25773be6da76a99d9661ebbf3552c007c460d2dd59dbf8/duckdb-1.5.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:4bfa9a4dadf71e83e2c4eaca2f9421c82a54defecc1b0b4c0be95e2389dec4fe", size = 32636685, upload-time = "2026-05-20T11:55:13.158Z" }, + { url = "https://files.pythonhosted.org/packages/42/58/e835c8298979d29db7a62cb5acc29e9b57aeaca7cdde2fcd3ac980f5cb18/duckdb-1.5.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:aea7baf67ad7e1829ac76f67d7dcbd7fb1f57c3eb179d55ac30952df4709ae30", size = 17308134, upload-time = "2026-05-20T11:55:16.194Z" }, + { url = "https://files.pythonhosted.org/packages/c9/46/617b51363f5613418c8b224b3cce16b58e6dde80904566bec232579c1d4e/duckdb-1.5.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0b0b4f088a65d77e1217ce5d7eff889e63fedc44281200d899ff47c84d8ff836", size = 15449891, upload-time = "2026-05-20T11:55:18.687Z" }, + { url = "https://files.pythonhosted.org/packages/b3/72/354146656e8d9ba3853d3a5ee80a481b8c5f70edfc3d5ae80a8c4479c967/duckdb-1.5.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe8d0c1f6a120aa03fa6e0d03897c71a1842e6cf7afd31d181348391f7108fe1", size = 19338499, upload-time = "2026-05-20T11:55:21.34Z" }, + { url = "https://files.pythonhosted.org/packages/56/8f/65fc623b51448f2bfba1a9ec6ab3debb4664c0876c0113a5e782600b53ac/duckdb-1.5.3-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0405eae18ec6e8210a471c97dbfe87a7e4d605274b7fe572a1f276e92158f13", size = 21455828, upload-time = "2026-05-20T11:55:23.847Z" }, + { url = "https://files.pythonhosted.org/packages/2b/db/d0274cbe9f5fe219f77c0bdf900ac77103569e83c102a4225ce04cbc607d/duckdb-1.5.3-cp314-cp314-win_amd64.whl", hash = "sha256:33ae08b3e818d7613d8936744b67718c2062c2f530376895bfd89efb51b81538", size = 13640011, upload-time = "2026-05-20T11:55:26.276Z" }, + { url = "https://files.pythonhosted.org/packages/07/5d/8f1899b8bef291caf953992fcd6c24df9f29387a35645e58c2504a5ca473/duckdb-1.5.3-cp314-cp314-win_arm64.whl", hash = "sha256:746433e49bbc667b4df283153415fbe37e9083e0eff6c3cd6e54de7536869cd4", size = 14411554, upload-time = "2026-05-20T11:55:29.037Z" }, ] [[package]] name = "fastapi" -version = "0.136.1" +version = "0.136.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, @@ -274,9 +274,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5d/45/c130091c2dfa061bbfe3150f2a5091ef1adf149f2a8d2ae769ecaf6e99a2/fastapi-0.136.1.tar.gz", hash = "sha256:7af665ad7acfa0a3baf8983d393b6b471b9da10ede59c60045f49fbc89a0fa7f", size = 397448, upload-time = "2026-04-23T16:49:44.046Z" } +sdist = { url = "https://files.pythonhosted.org/packages/81/2d/ff8d91d7b564d464629a0fd50a4489c97fcb836ac230bf3a7269232a9b1f/fastapi-0.136.3.tar.gz", hash = "sha256:e487fae93ad408e6f47641ee4dfe389864fd7bec92e547ea8498fc13f43e83ab", size = 396410, upload-time = "2026-05-23T18:53:15.192Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/ff/2e4eca3ade2c22fe1dea7043b8ee9dabe47753349eb1b56a202de8af6349/fastapi-0.136.1-py3-none-any.whl", hash = "sha256:a6e9d7eeada96c93a4d69cb03836b44fa34e2854accb7244a1ece36cd4781c3f", size = 117683, upload-time = "2026-04-23T16:49:42.437Z" }, + { url = "https://files.pythonhosted.org/packages/e0/82/45359b62a067409bd929ae8a56b8ed13e5a8c8a61194b3c236920999ab83/fastapi-0.136.3-py3-none-any.whl", hash = "sha256:3d2a69bdf04b7e9f3afa292c3bc7a98816bbfafa10bc9b45f3f3700d2f761620", size = 117481, upload-time = "2026-05-23T18:53:16.924Z" }, ] [[package]] @@ -290,11 +290,11 @@ wheels = [ [[package]] name = "fsspec" -version = "2026.3.0" +version = "2026.4.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e1/cf/b50ddf667c15276a9ab15a70ef5f257564de271957933ffea49d2cdbcdfb/fsspec-2026.3.0.tar.gz", hash = "sha256:1ee6a0e28677557f8c2f994e3eea77db6392b4de9cd1f5d7a9e87a0ae9d01b41", size = 313547, upload-time = "2026-03-27T19:11:14.892Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/8d/1c51c094345df128ca4a990d633fe1a0ff28726c9e6b3c41ba65087bba1d/fsspec-2026.4.0.tar.gz", hash = "sha256:301d8ac70ae90ef3ad05dcf94d6c3754a097f9b5fe4667d2787aa359ec7df7e4", size = 312760, upload-time = "2026-04-29T20:42:38.635Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/1f/5f4a3cd9e4440e9d9bc78ad0a91a1c8d46b4d429d5239ebe6793c9fe5c41/fsspec-2026.3.0-py3-none-any.whl", hash = "sha256:d2ceafaad1b3457968ed14efa28798162f1638dbb5d2a6868a2db002a5ee39a4", size = 202595, upload-time = "2026-03-27T19:11:13.595Z" }, + { url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" }, ] [[package]] @@ -308,34 +308,34 @@ wheels = [ [[package]] name = "hf-xet" -version = "1.4.3" +version = "1.5.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/92/ec9ad04d0b5728dca387a45af7bc98fbb0d73b2118759f5f6038b61a57e8/hf_xet-1.4.3.tar.gz", hash = "sha256:8ddedb73c8c08928c793df2f3401ec26f95be7f7e516a7bee2fbb546f6676113", size = 670477, upload-time = "2026-03-31T22:40:07.874Z" } +sdist = { url = "https://files.pythonhosted.org/packages/74/d8/5c06fc76461418326a7decf8367480c35be11a41fd938633929c60a9ec6b/hf_xet-1.5.0.tar.gz", hash = "sha256:e0fb0a34d9f406eed88233e829a67ec016bec5af19e480eac65a233ea289a948", size = 837196, upload-time = "2026-05-06T06:18:15.583Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/72/43/724d307b34e353da0abd476e02f72f735cdd2bc86082dee1b32ea0bfee1d/hf_xet-1.4.3-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:7551659ba4f1e1074e9623996f28c3873682530aee0a846b7f2f066239228144", size = 3800935, upload-time = "2026-03-31T22:39:49.618Z" }, - { url = "https://files.pythonhosted.org/packages/2b/d2/8bee5996b699262edb87dbb54118d287c0e1b2fc78af7cdc41857ba5e3c4/hf_xet-1.4.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bee693ada985e7045997f05f081d0e12c4c08bd7626dc397f8a7c487e6c04f7f", size = 3558942, upload-time = "2026-03-31T22:39:47.938Z" }, - { url = "https://files.pythonhosted.org/packages/c3/a1/e993d09cbe251196fb60812b09a58901c468127b7259d2bf0f68bf6088eb/hf_xet-1.4.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:21644b404bb0100fe3857892f752c4d09642586fd988e61501c95bbf44b393a3", size = 4207657, upload-time = "2026-03-31T22:39:39.69Z" }, - { url = "https://files.pythonhosted.org/packages/64/44/9eb6d21e5c34c63e5e399803a6932fa983cabdf47c0ecbcfe7ea97684b8c/hf_xet-1.4.3-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:987f09cfe418237812896a6736b81b1af02a3a6dcb4b4944425c4c4fca7a7cf8", size = 3986765, upload-time = "2026-03-31T22:39:37.936Z" }, - { url = "https://files.pythonhosted.org/packages/ea/7b/8ad6f16fdb82f5f7284a34b5ec48645bd575bdcd2f6f0d1644775909c486/hf_xet-1.4.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:60cf7fc43a99da0a853345cf86d23738c03983ee5249613a6305d3e57a5dca74", size = 4188162, upload-time = "2026-03-31T22:39:58.382Z" }, - { url = "https://files.pythonhosted.org/packages/1b/c4/39d6e136cbeea9ca5a23aad4b33024319222adbdc059ebcda5fc7d9d5ff4/hf_xet-1.4.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2815a49a7a59f3e2edf0cf113ae88e8cb2ca2a221bf353fb60c609584f4884d4", size = 4424525, upload-time = "2026-03-31T22:40:00.225Z" }, - { url = "https://files.pythonhosted.org/packages/46/f2/adc32dae6bdbc367853118b9878139ac869419a4ae7ba07185dc31251b76/hf_xet-1.4.3-cp313-cp313t-win_amd64.whl", hash = "sha256:42ee323265f1e6a81b0e11094564fb7f7e0ec75b5105ffd91ae63f403a11931b", size = 3671610, upload-time = "2026-03-31T22:40:10.42Z" }, - { url = "https://files.pythonhosted.org/packages/e2/19/25d897dcc3f81953e0c2cde9ec186c7a0fee413eb0c9a7a9130d87d94d3a/hf_xet-1.4.3-cp313-cp313t-win_arm64.whl", hash = "sha256:27c976ba60079fb8217f485b9c5c7fcd21c90b0367753805f87cb9f3cdc4418a", size = 3528529, upload-time = "2026-03-31T22:40:09.106Z" }, - { url = "https://files.pythonhosted.org/packages/ec/36/3e8f85ca9fe09b8de2b2e10c63b3b3353d7dda88a0b3d426dffbe7b8313b/hf_xet-1.4.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:5251d5ece3a81815bae9abab41cf7ddb7bcb8f56411bce0827f4a3071c92fdc6", size = 3801019, upload-time = "2026-03-31T22:39:56.651Z" }, - { url = "https://files.pythonhosted.org/packages/b5/9c/defb6cb1de28bccb7bd8d95f6e60f72a3d3fa4cb3d0329c26fb9a488bfe7/hf_xet-1.4.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1feb0f3abeacee143367c326a128a2e2b60868ec12a36c225afb1d6c5a05e6d2", size = 3558746, upload-time = "2026-03-31T22:39:54.766Z" }, - { url = "https://files.pythonhosted.org/packages/c1/bd/8d001191893178ff8e826e46ad5299446e62b93cd164e17b0ffea08832ec/hf_xet-1.4.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8b301fc150290ca90b4fccd079829b84bb4786747584ae08b94b4577d82fb791", size = 4207692, upload-time = "2026-03-31T22:39:46.246Z" }, - { url = "https://files.pythonhosted.org/packages/ce/48/6790b402803250e9936435613d3a78b9aaeee7973439f0918848dde58309/hf_xet-1.4.3-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:d972fbe95ddc0d3c0fc49b31a8a69f47db35c1e3699bf316421705741aab6653", size = 3986281, upload-time = "2026-03-31T22:39:44.648Z" }, - { url = "https://files.pythonhosted.org/packages/51/56/ea62552fe53db652a9099eda600b032d75554d0e86c12a73824bfedef88b/hf_xet-1.4.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c5b48db1ee344a805a1b9bd2cda9b6b65fe77ed3787bd6e87ad5521141d317cd", size = 4187414, upload-time = "2026-03-31T22:40:04.951Z" }, - { url = "https://files.pythonhosted.org/packages/7d/f5/bc1456d4638061bea997e6d2db60a1a613d7b200e0755965ec312dc1ef79/hf_xet-1.4.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:22bdc1f5fb8b15bf2831440b91d1c9bbceeb7e10c81a12e8d75889996a5c9da8", size = 4424368, upload-time = "2026-03-31T22:40:06.347Z" }, - { url = "https://files.pythonhosted.org/packages/e4/76/ab597bae87e1f06d18d3ecb8ed7f0d3c9a37037fc32ce76233d369273c64/hf_xet-1.4.3-cp314-cp314t-win_amd64.whl", hash = "sha256:0392c79b7cf48418cd61478c1a925246cf10639f4cd9d94368d8ca1e8df9ea07", size = 3672280, upload-time = "2026-03-31T22:40:16.401Z" }, - { url = "https://files.pythonhosted.org/packages/62/05/2e462d34e23a09a74d73785dbed71cc5dbad82a72eee2ad60a72a554155d/hf_xet-1.4.3-cp314-cp314t-win_arm64.whl", hash = "sha256:681c92a07796325778a79d76c67011764ecc9042a8c3579332b61b63ae512075", size = 3528945, upload-time = "2026-03-31T22:40:14.995Z" }, - { url = "https://files.pythonhosted.org/packages/ac/9f/9c23e4a447b8f83120798f9279d0297a4d1360bdbf59ef49ebec78fe2545/hf_xet-1.4.3-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d0da85329eaf196e03e90b84c2d0aca53bd4573d097a75f99609e80775f98025", size = 3805048, upload-time = "2026-03-31T22:39:53.105Z" }, - { url = "https://files.pythonhosted.org/packages/0b/f8/7aacb8e5f4a7899d39c787b5984e912e6c18b11be136ef13947d7a66d265/hf_xet-1.4.3-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:e23717ce4186b265f69afa66e6f0069fe7efbf331546f5c313d00e123dc84583", size = 3562178, upload-time = "2026-03-31T22:39:51.295Z" }, - { url = "https://files.pythonhosted.org/packages/df/9a/a24b26dc8a65f0ecc0fe5be981a19e61e7ca963b85e062c083f3a9100529/hf_xet-1.4.3-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc360b70c815bf340ed56c7b8c63aacf11762a4b099b2fe2c9bd6d6068668c08", size = 4212320, upload-time = "2026-03-31T22:39:42.922Z" }, - { url = "https://files.pythonhosted.org/packages/53/60/46d493db155d2ee2801b71fb1b0fd67696359047fdd8caee2c914cc50c79/hf_xet-1.4.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:39f2d2e9654cd9b4319885733993807aab6de9dfbd34c42f0b78338d6617421f", size = 3991546, upload-time = "2026-03-31T22:39:41.335Z" }, - { url = "https://files.pythonhosted.org/packages/bc/f5/067363e1c96c6b17256910830d1b54099d06287e10f4ec6ec4e7e08371fc/hf_xet-1.4.3-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:49ad8a8cead2b56051aa84d7fce3e1335efe68df3cf6c058f22a65513885baac", size = 4193200, upload-time = "2026-03-31T22:40:01.936Z" }, - { url = "https://files.pythonhosted.org/packages/42/4b/53951592882d9c23080c7644542fda34a3813104e9e11fa1a7d82d419cb8/hf_xet-1.4.3-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7716d62015477a70ea272d2d68cd7cad140f61c52ee452e133e139abfe2c17ba", size = 4429392, upload-time = "2026-03-31T22:40:03.492Z" }, - { url = "https://files.pythonhosted.org/packages/8a/21/75a6c175b4e79662ad8e62f46a40ce341d8d6b206b06b4320d07d55b188c/hf_xet-1.4.3-cp37-abi3-win_amd64.whl", hash = "sha256:6b591fcad34e272a5b02607485e4f2a1334aebf1bc6d16ce8eb1eb8978ac2021", size = 3677359, upload-time = "2026-03-31T22:40:13.619Z" }, - { url = "https://files.pythonhosted.org/packages/8a/7c/44314ecd0e89f8b2b51c9d9e5e7a60a9c1c82024ac471d415860557d3cd8/hf_xet-1.4.3-cp37-abi3-win_arm64.whl", hash = "sha256:7c2c7e20bcfcc946dc67187c203463f5e932e395845d098cc2a93f5b67ca0b47", size = 3533664, upload-time = "2026-03-31T22:40:12.152Z" }, + { url = "https://files.pythonhosted.org/packages/68/9b/6912c99070915a4f28119e3c5b52a9abd1eec0ad5cb293b8c967a0c6f5a2/hf_xet-1.5.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:7d70fe2ce97b9db73b9c9b9c81fe3693640aec83416a966c446afea54acfae3c", size = 4023383, upload-time = "2026-05-06T06:17:53.947Z" }, + { url = "https://files.pythonhosted.org/packages/0f/6d/9563cfde59b5d8128a9c7ec972a087f4c782e4f7bac5a85234edfd5d5e49/hf_xet-1.5.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:73a0dae8c71de3b0633a45c73f4a4a5ed09e94b43441d82981a781d4f12baa42", size = 3792751, upload-time = "2026-05-06T06:17:51.791Z" }, + { url = "https://files.pythonhosted.org/packages/07/a5/ed5a0cf35b49a0571af5a8f53416dad1877a718c021c9937c3a53cb45781/hf_xet-1.5.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a60290ec57e9b71767fba7c3645ddafdd0759974b540441510c629c6db6db24a", size = 4456058, upload-time = "2026-05-06T06:17:40.735Z" }, + { url = "https://files.pythonhosted.org/packages/60/fb/3ae8bf2a7a37a4197d0195d7247fd25b3952e15cb8a599e285dfaa6f52b3/hf_xet-1.5.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e5de0f6deada0dada870bb376a11bcd1f08abf3a968a6d118f33e72d1b1eb480", size = 4250783, upload-time = "2026-05-06T06:17:38.412Z" }, + { url = "https://files.pythonhosted.org/packages/a2/9b/8bae40d4d91525085137196e84eb0ed49cf65b5e96e5c3ecdadd8bd0fac2/hf_xet-1.5.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c799d49f1a5544a0ef7591c0ee75e0d6b93d6f56dc7a4979f59f7518d2872216", size = 4445594, upload-time = "2026-05-06T06:18:04.219Z" }, + { url = "https://files.pythonhosted.org/packages/13/59/c74efbbd4e8728172b2cc72a2bc014d2947a4b7bdced932fbd3f5da1a4e5/hf_xet-1.5.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2baea1b0b989e5c152fe81425f7745ddc8901280ba3d97c98d8cdece7b706c60", size = 4663995, upload-time = "2026-05-06T06:18:06.1Z" }, + { url = "https://files.pythonhosted.org/packages/73/32/8e1e0410af64cda9b139d1dcebdc993a8ff9c8c7c0e2696ae356d75ccc0d/hf_xet-1.5.0-cp313-cp313t-win_amd64.whl", hash = "sha256:526345b3ed45f374f6317349df489167606736c876241ba984105afe7fd4839d", size = 3966608, upload-time = "2026-05-06T06:18:19.74Z" }, + { url = "https://files.pythonhosted.org/packages/fc/34/a8febc8f4edbea8b3e21b02ebc8b628679b84ba7e45cde624a7736b51500/hf_xet-1.5.0-cp313-cp313t-win_arm64.whl", hash = "sha256:786d28e2eb8315d5035544b9d137b4a842d600c434bb91bf7d0d953cce906ad4", size = 3796946, upload-time = "2026-05-06T06:18:17.568Z" }, + { url = "https://files.pythonhosted.org/packages/2a/20/8fc8996afe5815fa1a6be8e9e5c02f24500f409d599e905800d498a4e14d/hf_xet-1.5.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:872d5601e6deea30d15865ede55d29eac6daf5a534ab417b99b6ef6b076dd96c", size = 4023495, upload-time = "2026-05-06T06:18:01.94Z" }, + { url = "https://files.pythonhosted.org/packages/32/6a/93d84463c00cecb561a7508aa6303e35ee2894294eac14245526924415fe/hf_xet-1.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9929561f5abf4581c8ea79587881dfef6b8abb2a0d8a51915936fc2a614f4e73", size = 3792731, upload-time = "2026-05-06T06:18:00.021Z" }, + { url = "https://files.pythonhosted.org/packages/9d/5a/8ec8e0c863b382d00b3c2e2af6ded6b06371be617144a625903a6d562f4b/hf_xet-1.5.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f7b7bbae318e583a86fb21e5a4a175d6721d628a2874f4bd022d0e660c32a682", size = 4456738, upload-time = "2026-05-06T06:17:49.574Z" }, + { url = "https://files.pythonhosted.org/packages/c5/ca/f7effa1a67717da2bcc6b6c28f71c6ca648c77acaec4e2c32f40cbe16d85/hf_xet-1.5.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:cf7b2dc6f31a4ea754bb50f74cde482dcf5d366d184076d8530b9872787f3761", size = 4251622, upload-time = "2026-05-06T06:17:47.096Z" }, + { url = "https://files.pythonhosted.org/packages/65/f2/19247dba3e231cf77dec59ddfb878f00057635ff773d099c9b59d37812c3/hf_xet-1.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8dbcbab554c9ef158ef2c991545c3e970ddd8cc7acdcd0a78c5a41095dab4ded", size = 4445667, upload-time = "2026-05-06T06:18:11.983Z" }, + { url = "https://files.pythonhosted.org/packages/7f/64/6f116801a3bcfb6f59f5c251f48cadc47ea54026441c4a385079286a94fa/hf_xet-1.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5906bf7718d3636dc13402914736abe723492cb730f744834f5f5b67d3a12702", size = 4664619, upload-time = "2026-05-06T06:18:13.771Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e8/069542d37946ed08669b127e1496fa99e78196d71de8d41eda5e9f1b7a58/hf_xet-1.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5f3dc2248fc01cc0a00cd392ab497f1ca373fcbc7e3f2da1f452480b384e839e", size = 3966802, upload-time = "2026-05-06T06:18:28.162Z" }, + { url = "https://files.pythonhosted.org/packages/f9/91/fc6fdec27b14d04e88c386ac0a0129732b53fa23f7c4a78f4b83a039c567/hf_xet-1.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b285cea1b5bab46b758772716ba8d6854a1a0310fed1c249d678a8b38601e5a0", size = 3797168, upload-time = "2026-05-06T06:18:26.287Z" }, + { url = "https://files.pythonhosted.org/packages/3d/fb/69ff198a82cae7eb1a69fb84d93b3a3e4816564d76817fe541ddc96874eb/hf_xet-1.5.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:dad0dc84e941b8ba3c860659fe1fdc35c049d47cce293f003287757e971a8f56", size = 4030814, upload-time = "2026-05-06T06:17:57.933Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ff/edcc2b40162bef3ff78e14ab637e5f3b89243d6aee72f5949d3bb6a5af83/hf_xet-1.5.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fd6e5a9b0fdac4ed03ed45ef79254a655b1aaab514a02202617fbf643f5fdf7a", size = 3798444, upload-time = "2026-05-06T06:17:55.79Z" }, + { url = "https://files.pythonhosted.org/packages/49/4d/103f76b04310e5e57656696cc184690d20c466af0bca3ca88f8c8ea5d4f3/hf_xet-1.5.0-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3531b1823a0e6d77d80f9ed15ca0e00f0d115094f8ac033d5cae88f4564cc949", size = 4465986, upload-time = "2026-05-06T06:17:44.886Z" }, + { url = "https://files.pythonhosted.org/packages/c4/a2/546f47f464737b3edbab6f8ddb57f2599b93d2cbb66f06abb475ccb48651/hf_xet-1.5.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9a0ee58cd18d5ea799f7ed11290bbccbe56bdd8b1d97ca74b9cc49a3945d7a3b", size = 4259865, upload-time = "2026-05-06T06:17:42.639Z" }, + { url = "https://files.pythonhosted.org/packages/95/7f/1be593c1f28613be2e196473481cd81bfc5910795e30a34e8f744f6cac4f/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e60df5a42e9bed8628b6416af2cba4cba57ae9f02de226a06b020d98e1aab18", size = 4459835, upload-time = "2026-05-06T06:18:08.026Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b2/703569fc881f3284487e68cda7b42179978480da3c438042a6bbbb4a671c/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4b35549ce62601b84da4ff9b24d970032ace3d4430f52d91bcbb26c901d6c690", size = 4672414, upload-time = "2026-05-06T06:18:09.864Z" }, + { url = "https://files.pythonhosted.org/packages/af/37/1b6def445c567286b50aa3b33828158e135b1be44938dde59f11382a500c/hf_xet-1.5.0-cp37-abi3-win_amd64.whl", hash = "sha256:2806c7c17b4d23f8d88f7c4814f838c3b6150773fe339c20af23e1cfaf2797e4", size = 3977238, upload-time = "2026-05-06T06:18:23.621Z" }, + { url = "https://files.pythonhosted.org/packages/62/94/3b66b148778ee100dcfd69c2ca22b57b41b44d3063ceec934f209e9184ce/hf_xet-1.5.0-cp37-abi3-win_arm64.whl", hash = "sha256:b6c9df403040248c76d808d3e047d64db2d923bae593eb244c41e425cf6cd7be", size = 3806916, upload-time = "2026-05-06T06:18:21.7Z" }, ] [[package]] @@ -353,24 +353,31 @@ wheels = [ [[package]] name = "httptools" -version = "0.7.1" +version = "0.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b5/46/120a669232c7bdedb9d52d4aeae7e6c7dfe151e99dc70802e2fc7a5e1993/httptools-0.7.1.tar.gz", hash = "sha256:abd72556974f8e7c74a259655924a717a2365b236c882c3f6f8a45fe94703ac9", size = 258961, upload-time = "2025-10-10T03:55:08.559Z" } +sdist = { url = "https://files.pythonhosted.org/packages/43/e5/d471fcb0e14523fe1c3f4ba58ca52480e7bd70ad7109a3846bc75892f7fb/httptools-0.8.0.tar.gz", hash = "sha256:6b2a32f18d97e16e90827d7a819ffa8dbd8cc245fc4e1fa9d1095b54ef4bd999", size = 271342, upload-time = "2026-05-25T22:17:48.841Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/09/8f/c77b1fcbfd262d422f12da02feb0d218fa228d52485b77b953832105bb90/httptools-0.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6babce6cfa2a99545c60bfef8bee0cc0545413cb0018f617c8059a30ad985de3", size = 202889, upload-time = "2025-10-10T03:54:47.089Z" }, - { url = "https://files.pythonhosted.org/packages/0a/1a/22887f53602feaa066354867bc49a68fc295c2293433177ee90870a7d517/httptools-0.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:601b7628de7504077dd3dcb3791c6b8694bbd967148a6d1f01806509254fb1ca", size = 108180, upload-time = "2025-10-10T03:54:48.052Z" }, - { url = "https://files.pythonhosted.org/packages/32/6a/6aaa91937f0010d288d3d124ca2946d48d60c3a5ee7ca62afe870e3ea011/httptools-0.7.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:04c6c0e6c5fb0739c5b8a9eb046d298650a0ff38cf42537fc372b28dc7e4472c", size = 478596, upload-time = "2025-10-10T03:54:48.919Z" }, - { url = "https://files.pythonhosted.org/packages/6d/70/023d7ce117993107be88d2cbca566a7c1323ccbaf0af7eabf2064fe356f6/httptools-0.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:69d4f9705c405ae3ee83d6a12283dc9feba8cc6aaec671b412917e644ab4fa66", size = 473268, upload-time = "2025-10-10T03:54:49.993Z" }, - { url = "https://files.pythonhosted.org/packages/32/4d/9dd616c38da088e3f436e9a616e1d0cc66544b8cdac405cc4e81c8679fc7/httptools-0.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:44c8f4347d4b31269c8a9205d8a5ee2df5322b09bbbd30f8f862185bb6b05346", size = 455517, upload-time = "2025-10-10T03:54:51.066Z" }, - { url = "https://files.pythonhosted.org/packages/1d/3a/a6c595c310b7df958e739aae88724e24f9246a514d909547778d776799be/httptools-0.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:465275d76db4d554918aba40bf1cbebe324670f3dfc979eaffaa5d108e2ed650", size = 458337, upload-time = "2025-10-10T03:54:52.196Z" }, - { url = "https://files.pythonhosted.org/packages/fd/82/88e8d6d2c51edc1cc391b6e044c6c435b6aebe97b1abc33db1b0b24cd582/httptools-0.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:322d00c2068d125bd570f7bf78b2d367dad02b919d8581d7476d8b75b294e3e6", size = 85743, upload-time = "2025-10-10T03:54:53.448Z" }, - { url = "https://files.pythonhosted.org/packages/34/50/9d095fcbb6de2d523e027a2f304d4551855c2f46e0b82befd718b8b20056/httptools-0.7.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:c08fe65728b8d70b6923ce31e3956f859d5e1e8548e6f22ec520a962c6757270", size = 203619, upload-time = "2025-10-10T03:54:54.321Z" }, - { url = "https://files.pythonhosted.org/packages/07/f0/89720dc5139ae54b03f861b5e2c55a37dba9a5da7d51e1e824a1f343627f/httptools-0.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7aea2e3c3953521c3c51106ee11487a910d45586e351202474d45472db7d72d3", size = 108714, upload-time = "2025-10-10T03:54:55.163Z" }, - { url = "https://files.pythonhosted.org/packages/b3/cb/eea88506f191fb552c11787c23f9a405f4c7b0c5799bf73f2249cd4f5228/httptools-0.7.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0e68b8582f4ea9166be62926077a3334064d422cf08ab87d8b74664f8e9058e1", size = 472909, upload-time = "2025-10-10T03:54:56.056Z" }, - { url = "https://files.pythonhosted.org/packages/e0/4a/a548bdfae6369c0d078bab5769f7b66f17f1bfaa6fa28f81d6be6959066b/httptools-0.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df091cf961a3be783d6aebae963cc9b71e00d57fa6f149025075217bc6a55a7b", size = 470831, upload-time = "2025-10-10T03:54:57.219Z" }, - { url = "https://files.pythonhosted.org/packages/4d/31/14df99e1c43bd132eec921c2e7e11cda7852f65619bc0fc5bdc2d0cb126c/httptools-0.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f084813239e1eb403ddacd06a30de3d3e09a9b76e7894dcda2b22f8a726e9c60", size = 452631, upload-time = "2025-10-10T03:54:58.219Z" }, - { url = "https://files.pythonhosted.org/packages/22/d2/b7e131f7be8d854d48cb6d048113c30f9a46dca0c9a8b08fcb3fcd588cdc/httptools-0.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7347714368fb2b335e9063bc2b96f2f87a9ceffcd9758ac295f8bbcd3ffbc0ca", size = 452910, upload-time = "2025-10-10T03:54:59.366Z" }, - { url = "https://files.pythonhosted.org/packages/53/cf/878f3b91e4e6e011eff6d1fa9ca39f7eb17d19c9d7971b04873734112f30/httptools-0.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:cfabda2a5bb85aa2a904ce06d974a3f30fb36cc63d7feaddec05d2050acede96", size = 88205, upload-time = "2025-10-10T03:55:00.389Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e5/8cfcabc5546e8022f168be28bcdaa128a240a0befdd03b59d558b4f18bd6/httptools-0.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:614ceea8ea606848bece2338ac03b3ce5324bcb4be8dc7d377ed708012fa4db8", size = 205148, upload-time = "2026-05-25T22:17:16.333Z" }, + { url = "https://files.pythonhosted.org/packages/2a/0e/0fb14848c19a686c8062ff9067c1a48793e3224b47bc5b201535b6036fce/httptools-0.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2d689918c15a013c65ef52d9fd495d766893ab831a2c8d89f2ac5940a5df847c", size = 111368, upload-time = "2026-05-25T22:17:17.586Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/46f1cecf06b9bbde8e4b8c88034ac7908989e5ff7a3a388ef38392949c1f/httptools-0.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:eb3028cca2fc0a6d720e52ef61d8ebb62fcbfeb1de56874546d858d3f25a26b7", size = 486447, upload-time = "2026-05-25T22:17:18.564Z" }, + { url = "https://files.pythonhosted.org/packages/77/00/258bfc0837221f81d9725c45f9b948a6a6b2994a147a4fb66e85100c668f/httptools-0.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:88bdd940f2b5d487b4d032c6afa5489a7dc4694410d43de3c38c4fb3af0dc45d", size = 482448, upload-time = "2026-05-25T22:17:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/04/ab/d1cef3b5523f4d272a70f42a776c3169a2dddfe3a54de4b2ce4a36341528/httptools-0.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6a43c9dd399758ccc0531acb0a3c4a6c299ee893ee9400e9c893b7bdcfae0681", size = 464460, upload-time = "2026-05-25T22:17:20.882Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/5d1d072442277bb2b3434e0e60690b8e8c23840ef7de8b6ea54040a536d3/httptools-0.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0770728beb05094c809b98e814edff5fef69d26ad7d21185f2f6d5884a0ba683", size = 471312, upload-time = "2026-05-25T22:17:22.085Z" }, + { url = "https://files.pythonhosted.org/packages/0d/66/b96623b27e51a68199ef4efdda0613cced9233fe3062ac74e50749c5ad37/httptools-0.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:7685df791fad561384bfb139e77fde27a1ffd93134e016f95a0db424ffbf77b1", size = 90117, upload-time = "2026-05-25T22:17:23.074Z" }, + { url = "https://files.pythonhosted.org/packages/1a/12/fa3fbf5f9517b273edea2dc982aa82a8c634091e67c590792b729017bc6f/httptools-0.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:de242a49b5d18e0a8776e654e9f6bf6d89f3875a5c35b425a0e7ce940feb3fd6", size = 206183, upload-time = "2026-05-25T22:17:24.004Z" }, + { url = "https://files.pythonhosted.org/packages/30/fc/5e7c4cb443370f2090a3aba0453a07384d29ff66b7435bb90e77e1037599/httptools-0.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:159e9ab5f701ccd42e555a12f1ad8ff69702910fc1c996cf2bb66e5fcb7a231b", size = 112079, upload-time = "2026-05-25T22:17:25.216Z" }, + { url = "https://files.pythonhosted.org/packages/ba/53/771bd891eb0f236f32145d6a1775777ec85745f3cc983a1f23d1a3b8ddfe/httptools-0.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c4a9f1707e4823d54dfec6c33fa3697d302aed536ed352a7ebb5a061ddb869d0", size = 481596, upload-time = "2026-05-25T22:17:26.186Z" }, + { url = "https://files.pythonhosted.org/packages/62/42/94e15bc68ce3d423243c45d7f1b0c7561f13844f97dc52ae23182fb65628/httptools-0.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d76ad7b951387e3632c8716a9bb03ac5b45c5f16119aa409db0459520887944e", size = 480865, upload-time = "2026-05-25T22:17:27.542Z" }, + { url = "https://files.pythonhosted.org/packages/1c/7c/fe2980fc03723272e30f135b62360b075f513dfe7cc73aef36c7f04012bd/httptools-0.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a3b7387147361c3fd47a0bde763c5c91b5b4cd4dc9989b8ece84ff436c99843b", size = 463189, upload-time = "2026-05-25T22:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/15/1b/47fc5fff68acd1bfa20b4734059c9a06cadb88119dcd5258b5b0d21d91c8/httptools-0.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f256d6ce930c52ca1cb2a960b7da03548c454e7d28b06059ad41bfe789036ce0", size = 466610, upload-time = "2026-05-25T22:17:29.816Z" }, + { url = "https://files.pythonhosted.org/packages/60/bd/07b13c93ffd9bec9546e0d43f8e19378dd696dbd278511406bc07371ef1f/httptools-0.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:19d1ee275bb59ba2643ba9a3a1e51cc0c788caf2b8df506368e03f56fdd08527", size = 92705, upload-time = "2026-05-25T22:17:31.133Z" }, + { url = "https://files.pythonhosted.org/packages/fd/c4/121648f68ce066d7bd762d6b6d97e620847642d38d54f3d90ff11d947629/httptools-0.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:de1ed58a974e75d56560acc7e7fed01a454994429456f65209789992e41f2568", size = 215023, upload-time = "2026-05-25T22:17:32.401Z" }, + { url = "https://files.pythonhosted.org/packages/b9/b0/312a062ae741ae3e8baa8c8bf20be81b2e67337b259ab4349bebc7b6142e/httptools-0.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e93c227b595c6926c1acee96891dd9da4be338cfbe82e5cd3bb9d8dd7dc4ac0b", size = 117405, upload-time = "2026-05-25T22:17:33.742Z" }, + { url = "https://files.pythonhosted.org/packages/fc/37/fccd705f795386bb05bf413012fecff2a33e5aa8c2f069096de3e9fd8702/httptools-0.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2a021c3a8e65cc125390d72f59b968afca3bdcaff25bd67965e0a055a14946ca", size = 558497, upload-time = "2026-05-25T22:17:34.732Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/f172e8003576de35f5ba77ff417cf0e34429d35dc014deef15afa337a72c/httptools-0.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48774d39cbb70e2b1f71f88852a3087ae1d3a1eb80482bb48c13067ab080c14f", size = 571585, upload-time = "2026-05-25T22:17:35.813Z" }, + { url = "https://files.pythonhosted.org/packages/3e/b9/f5564760af99f3dbbf3f9104dc00e5da27e96cf433c6bdcf77617f70bf3f/httptools-0.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:88eead8ec8680a9f146c655bc88445a325bd7921cfd8194c7337e9467282427d", size = 543297, upload-time = "2026-05-25T22:17:37.08Z" }, + { url = "https://files.pythonhosted.org/packages/99/67/8d9f2c313618e161b82f3873188e7196126da1d6e29688df40eb3997c77a/httptools-0.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2c032fa028f46871ec7e1fc59fc15e8023eab3e6bbe6ece786a1611719a5d081", size = 539535, upload-time = "2026-05-25T22:17:38.032Z" }, + { url = "https://files.pythonhosted.org/packages/48/63/b906c01e53f50d432c0defe43ce52764a111dc1bdd028bafbeb54dcfd008/httptools-0.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:384c17174464c8e873398b7af24f0b1f44d992c820328413951a625323155d77", size = 108209, upload-time = "2026-05-25T22:17:39.473Z" }, ] [[package]] @@ -390,9 +397,10 @@ wheels = [ [[package]] name = "huggingface-hub" -version = "1.12.0" +version = "1.16.4" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "click" }, { name = "filelock" }, { name = "fsspec" }, { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, @@ -403,9 +411,9 @@ dependencies = [ { name = "typer" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/56/52/1b54cb569509c725a32c1315261ac9fd0e6b91bbbf74d86fca10d3376164/huggingface_hub-1.12.0.tar.gz", hash = "sha256:7c3fe85e24b652334e5d456d7a812cd9a071e75630fac4365d9165ab5e4a34b6", size = 763091, upload-time = "2026-04-24T13:32:08.674Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ab/11/9b6e439cb2417c479c3da108b38363232a1554721de9f8ef4836cb07422b/huggingface_hub-1.16.4.tar.gz", hash = "sha256:023bacd155f837d3fa56379ac8e23dababe6d6d87b04f8dacc258a44a38abe01", size = 792585, upload-time = "2026-05-26T17:19:09.971Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/2b/ef03ddb96bd1123503c2bd6932001020292deea649e9bf4caa2cb65a85bf/huggingface_hub-1.12.0-py3-none-any.whl", hash = "sha256:d74939969585ee35748bd66de09baf84099d461bda7287cd9043bfb99b0e424d", size = 646806, upload-time = "2026-04-24T13:32:06.717Z" }, + { url = "https://files.pythonhosted.org/packages/da/86/e05d58ea272089151ba9f6fcc7b44a97aa2533d5a5bce46611220c23c6d6/huggingface_hub-1.16.4-py3-none-any.whl", hash = "sha256:994ec184c3330952d7b5f131ea0b1a6ba1047bd05461f5dec191f8fc1099fbd7", size = 668190, upload-time = "2026-05-26T17:19:08.228Z" }, ] [[package]] @@ -419,11 +427,11 @@ wheels = [ [[package]] name = "idna" -version = "3.15" +version = "3.16" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/88/bcf9709822fe69d02c2a6a77956c98ce6ea8ca8767a9aadcedc7eb6a2390/idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d", size = 203770, upload-time = "2026-05-22T00:16:18.781Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, + { url = "https://files.pythonhosted.org/packages/94/16/70255075a9859a0e3adb789b68ceb0e210dec03934245fd98d248226572f/idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5", size = 74165, upload-time = "2026-05-22T00:16:16.698Z" }, ] [[package]] @@ -458,7 +466,7 @@ wheels = [ [[package]] name = "ktx-daemon" -version = "0.4.1" +version = "0.9.0" source = { editable = "python/ktx-daemon" } dependencies = [ { name = "fastapi" }, @@ -478,8 +486,8 @@ dependencies = [ [package.optional-dependencies] local-embeddings = [ { name = "sentence-transformers" }, - { name = "torch", version = "2.11.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin'" }, - { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin'" }, + { name = "torch", version = "2.12.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin'" }, + { name = "torch", version = "2.12.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin'" }, ] [package.dev-dependencies] @@ -490,20 +498,20 @@ dev = [ [package.metadata] requires-dist = [ - { name = "fastapi", specifier = ">=0.115.0" }, + { name = "fastapi", specifier = ">=0.136.3" }, { name = "ktx-sl", editable = "python/ktx-sl" }, { name = "lkml", specifier = ">=1.3.7" }, - { name = "numpy", specifier = ">=2.2.6" }, - { name = "orjson", specifier = ">=3.11.4" }, - { name = "pandas", specifier = ">=2.2.3" }, - { name = "posthog", specifier = ">=7.0.0" }, - { name = "psycopg", extras = ["binary"], specifier = ">=3.2.0" }, - { name = "pydantic", specifier = ">=2.9.0" }, - { name = "requests", specifier = ">=2.32.0" }, + { name = "numpy", specifier = ">=2.4.6" }, + { name = "orjson", specifier = ">=3.11.9" }, + { name = "pandas", specifier = ">=3.0.3" }, + { name = "posthog", specifier = ">=7.16.1" }, + { name = "psycopg", extras = ["binary"], specifier = ">=3.3.4" }, + { name = "pydantic", specifier = ">=2.13.4" }, + { name = "requests", specifier = ">=2.34.2" }, { name = "sentence-transformers", marker = "extra == 'local-embeddings'", specifier = ">=5.1.1" }, - { name = "sqlglot", specifier = ">=26" }, + { name = "sqlglot", specifier = ">=30" }, { name = "torch", marker = "extra == 'local-embeddings'", specifier = ">=2.2.0", index = "https://download.pytorch.org/whl/cpu" }, - { name = "uvicorn", extras = ["standard"], specifier = ">=0.32.0" }, + { name = "uvicorn", extras = ["standard"], specifier = ">=0.48.0" }, ] provides-extras = ["local-embeddings"] @@ -515,7 +523,7 @@ dev = [ [[package]] name = "ktx-sl" -version = "0.4.1" +version = "0.9.0" source = { editable = "python/ktx-sl" } dependencies = [ { name = "pydantic" }, @@ -549,7 +557,7 @@ requires-dist = [ { name = "pytest-cov", marker = "extra == 'dev'" }, { name = "pyyaml", specifier = ">=6" }, { name = "ruff", marker = "extra == 'dev'" }, - { name = "sqlglot", specifier = ">=26" }, + { name = "sqlglot", specifier = ">=30" }, ] provides-extras = ["dev", "tpch"] @@ -593,14 +601,14 @@ wheels = [ [[package]] name = "markdown-it-py" -version = "4.0.0" +version = "4.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mdurl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, ] [[package]] @@ -693,90 +701,90 @@ wheels = [ [[package]] name = "numpy" -version = "2.4.4" +version = "2.4.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/9f/b8cef5bffa569759033adda9481211426f12f53299629b410340795c2514/numpy-2.4.4.tar.gz", hash = "sha256:2d390634c5182175533585cc89f3608a4682ccb173cc9bb940b2881c8d6f8fa0", size = 20731587, upload-time = "2026-03-29T13:22:01.298Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/14/1d/d0a583ce4fefcc3308806a749a536c201ed6b5ad6e1322e227ee4848979d/numpy-2.4.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:08f2e31ed5e6f04b118e49821397f12767934cfdd12a1ce86a058f91e004ee50", size = 16684933, upload-time = "2026-03-29T13:19:22.47Z" }, - { url = "https://files.pythonhosted.org/packages/c1/62/2b7a48fbb745d344742c0277f01286dead15f3f68e4f359fbfcf7b48f70f/numpy-2.4.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e823b8b6edc81e747526f70f71a9c0a07ac4e7ad13020aa736bb7c9d67196115", size = 14694532, upload-time = "2026-03-29T13:19:25.581Z" }, - { url = "https://files.pythonhosted.org/packages/e5/87/499737bfba066b4a3bebff24a8f1c5b2dee410b209bc6668c9be692580f0/numpy-2.4.4-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:4a19d9dba1a76618dd86b164d608566f393f8ec6ac7c44f0cc879011c45e65af", size = 5199661, upload-time = "2026-03-29T13:19:28.31Z" }, - { url = "https://files.pythonhosted.org/packages/cd/da/464d551604320d1491bc345efed99b4b7034143a85787aab78d5691d5a0e/numpy-2.4.4-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:d2a8490669bfe99a233298348acc2d824d496dee0e66e31b66a6022c2ad74a5c", size = 6547539, upload-time = "2026-03-29T13:19:30.97Z" }, - { url = "https://files.pythonhosted.org/packages/7d/90/8d23e3b0dafd024bf31bdec225b3bb5c2dbfa6912f8a53b8659f21216cbf/numpy-2.4.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45dbed2ab436a9e826e302fcdcbe9133f9b0006e5af7168afb8963a6520da103", size = 15668806, upload-time = "2026-03-29T13:19:33.887Z" }, - { url = "https://files.pythonhosted.org/packages/d1/73/a9d864e42a01896bb5974475438f16086be9ba1f0d19d0bb7a07427c4a8b/numpy-2.4.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c901b15172510173f5cb310eae652908340f8dede90fff9e3bf6c0d8dfd92f83", size = 16632682, upload-time = "2026-03-29T13:19:37.336Z" }, - { url = "https://files.pythonhosted.org/packages/34/fb/14570d65c3bde4e202a031210475ae9cde9b7686a2e7dc97ee67d2833b35/numpy-2.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:99d838547ace2c4aace6c4f76e879ddfe02bb58a80c1549928477862b7a6d6ed", size = 17019810, upload-time = "2026-03-29T13:19:40.963Z" }, - { url = "https://files.pythonhosted.org/packages/8a/77/2ba9d87081fd41f6d640c83f26fb7351e536b7ce6dd9061b6af5904e8e46/numpy-2.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0aec54fd785890ecca25a6003fd9a5aed47ad607bbac5cd64f836ad8666f4959", size = 18357394, upload-time = "2026-03-29T13:19:44.859Z" }, - { url = "https://files.pythonhosted.org/packages/a2/23/52666c9a41708b0853fa3b1a12c90da38c507a3074883823126d4e9d5b30/numpy-2.4.4-cp313-cp313-win32.whl", hash = "sha256:07077278157d02f65c43b1b26a3886bce886f95d20aabd11f87932750dfb14ed", size = 5959556, upload-time = "2026-03-29T13:19:47.661Z" }, - { url = "https://files.pythonhosted.org/packages/57/fb/48649b4971cde70d817cf97a2a2fdc0b4d8308569f1dd2f2611959d2e0cf/numpy-2.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:5c70f1cc1c4efbe316a572e2d8b9b9cc44e89b95f79ca3331553fbb63716e2bf", size = 12317311, upload-time = "2026-03-29T13:19:50.67Z" }, - { url = "https://files.pythonhosted.org/packages/ba/d8/11490cddd564eb4de97b4579ef6bfe6a736cc07e94c1598590ae25415e01/numpy-2.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:ef4059d6e5152fa1a39f888e344c73fdc926e1b2dd58c771d67b0acfbf2aa67d", size = 10222060, upload-time = "2026-03-29T13:19:54.229Z" }, - { url = "https://files.pythonhosted.org/packages/99/5d/dab4339177a905aad3e2221c915b35202f1ec30d750dd2e5e9d9a72b804b/numpy-2.4.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4bbc7f303d125971f60ec0aaad5e12c62d0d2c925f0ab1273debd0e4ba37aba5", size = 14822302, upload-time = "2026-03-29T13:19:57.585Z" }, - { url = "https://files.pythonhosted.org/packages/eb/e4/0564a65e7d3d97562ed6f9b0fd0fb0a6f559ee444092f105938b50043876/numpy-2.4.4-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:4d6d57903571f86180eb98f8f0c839fa9ebbfb031356d87f1361be91e433f5b7", size = 5327407, upload-time = "2026-03-29T13:20:00.601Z" }, - { url = "https://files.pythonhosted.org/packages/29/8d/35a3a6ce5ad371afa58b4700f1c820f8f279948cca32524e0a695b0ded83/numpy-2.4.4-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:4636de7fd195197b7535f231b5de9e4b36d2c440b6e566d2e4e4746e6af0ca93", size = 6647631, upload-time = "2026-03-29T13:20:02.855Z" }, - { url = "https://files.pythonhosted.org/packages/f4/da/477731acbd5a58a946c736edfdabb2ac5b34c3d08d1ba1a7b437fa0884df/numpy-2.4.4-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ad2e2ef14e0b04e544ea2fa0a36463f847f113d314aa02e5b402fdf910ef309e", size = 15727691, upload-time = "2026-03-29T13:20:06.004Z" }, - { url = "https://files.pythonhosted.org/packages/e6/db/338535d9b152beabeb511579598418ba0212ce77cf9718edd70262cc4370/numpy-2.4.4-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a285b3b96f951841799528cd1f4f01cd70e7e0204b4abebac9463eecfcf2a40", size = 16681241, upload-time = "2026-03-29T13:20:09.417Z" }, - { url = "https://files.pythonhosted.org/packages/e2/a9/ad248e8f58beb7a0219b413c9c7d8151c5d285f7f946c3e26695bdbbe2df/numpy-2.4.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f8474c4241bc18b750be2abea9d7a9ec84f46ef861dbacf86a4f6e043401f79e", size = 17085767, upload-time = "2026-03-29T13:20:13.126Z" }, - { url = "https://files.pythonhosted.org/packages/b5/1a/3b88ccd3694681356f70da841630e4725a7264d6a885c8d442a697e1146b/numpy-2.4.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4e874c976154687c1f71715b034739b45c7711bec81db01914770373d125e392", size = 18403169, upload-time = "2026-03-29T13:20:17.096Z" }, - { url = "https://files.pythonhosted.org/packages/c2/c9/fcfd5d0639222c6eac7f304829b04892ef51c96a75d479214d77e3ce6e33/numpy-2.4.4-cp313-cp313t-win32.whl", hash = "sha256:9c585a1790d5436a5374bac930dad6ed244c046ed91b2b2a3634eb2971d21008", size = 6083477, upload-time = "2026-03-29T13:20:20.195Z" }, - { url = "https://files.pythonhosted.org/packages/d5/e3/3938a61d1c538aaec8ed6fd6323f57b0c2d2d2219512434c5c878db76553/numpy-2.4.4-cp313-cp313t-win_amd64.whl", hash = "sha256:93e15038125dc1e5345d9b5b68aa7f996ec33b98118d18c6ca0d0b7d6198b7e8", size = 12457487, upload-time = "2026-03-29T13:20:22.946Z" }, - { url = "https://files.pythonhosted.org/packages/97/6a/7e345032cc60501721ef94e0e30b60f6b0bd601f9174ebd36389a2b86d40/numpy-2.4.4-cp313-cp313t-win_arm64.whl", hash = "sha256:0dfd3f9d3adbe2920b68b5cd3d51444e13a10792ec7154cd0a2f6e74d4ab3233", size = 10292002, upload-time = "2026-03-29T13:20:25.909Z" }, - { url = "https://files.pythonhosted.org/packages/6e/06/c54062f85f673dd5c04cbe2f14c3acb8c8b95e3384869bb8cc9bff8cb9df/numpy-2.4.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:f169b9a863d34f5d11b8698ead99febeaa17a13ca044961aa8e2662a6c7766a0", size = 16684353, upload-time = "2026-03-29T13:20:29.504Z" }, - { url = "https://files.pythonhosted.org/packages/4c/39/8a320264a84404c74cc7e79715de85d6130fa07a0898f67fb5cd5bd79908/numpy-2.4.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2483e4584a1cb3092da4470b38866634bafb223cbcd551ee047633fd2584599a", size = 14704914, upload-time = "2026-03-29T13:20:33.547Z" }, - { url = "https://files.pythonhosted.org/packages/91/fb/287076b2614e1d1044235f50f03748f31fa287e3dbe6abeb35cdfa351eca/numpy-2.4.4-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:2d19e6e2095506d1736b7d80595e0f252d76b89f5e715c35e06e937679ea7d7a", size = 5210005, upload-time = "2026-03-29T13:20:36.45Z" }, - { url = "https://files.pythonhosted.org/packages/63/eb/fcc338595309910de6ecabfcef2419a9ce24399680bfb149421fa2df1280/numpy-2.4.4-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:6a246d5914aa1c820c9443ddcee9c02bec3e203b0c080349533fae17727dfd1b", size = 6544974, upload-time = "2026-03-29T13:20:39.014Z" }, - { url = "https://files.pythonhosted.org/packages/44/5d/e7e9044032a716cdfaa3fba27a8e874bf1c5f1912a1ddd4ed071bf8a14a6/numpy-2.4.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:989824e9faf85f96ec9c7761cd8d29c531ad857bfa1daa930cba85baaecf1a9a", size = 15684591, upload-time = "2026-03-29T13:20:42.146Z" }, - { url = "https://files.pythonhosted.org/packages/98/7c/21252050676612625449b4807d6b695b9ce8a7c9e1c197ee6216c8a65c7c/numpy-2.4.4-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:27a8d92cd10f1382a67d7cf4db7ce18341b66438bdd9f691d7b0e48d104c2a9d", size = 16637700, upload-time = "2026-03-29T13:20:46.204Z" }, - { url = "https://files.pythonhosted.org/packages/b1/29/56d2bbef9465db24ef25393383d761a1af4f446a1df9b8cded4fe3a5a5d7/numpy-2.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e44319a2953c738205bf3354537979eaa3998ed673395b964c1176083dd46252", size = 17035781, upload-time = "2026-03-29T13:20:50.242Z" }, - { url = "https://files.pythonhosted.org/packages/e3/2b/a35a6d7589d21f44cea7d0a98de5ddcbb3d421b2622a5c96b1edf18707c3/numpy-2.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e892aff75639bbef0d2a2cfd55535510df26ff92f63c92cd84ef8d4ba5a5557f", size = 18362959, upload-time = "2026-03-29T13:20:54.019Z" }, - { url = "https://files.pythonhosted.org/packages/64/c9/d52ec581f2390e0f5f85cbfd80fb83d965fc15e9f0e1aec2195faa142cde/numpy-2.4.4-cp314-cp314-win32.whl", hash = "sha256:1378871da56ca8943c2ba674530924bb8ca40cd228358a3b5f302ad60cf875fc", size = 6008768, upload-time = "2026-03-29T13:20:56.912Z" }, - { url = "https://files.pythonhosted.org/packages/fa/22/4cc31a62a6c7b74a8730e31a4274c5dc80e005751e277a2ce38e675e4923/numpy-2.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:715d1c092715954784bc79e1174fc2a90093dc4dc84ea15eb14dad8abdcdeb74", size = 12449181, upload-time = "2026-03-29T13:20:59.548Z" }, - { url = "https://files.pythonhosted.org/packages/70/2e/14cda6f4d8e396c612d1bf97f22958e92148801d7e4f110cabebdc0eef4b/numpy-2.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:2c194dd721e54ecad9ad387c1d35e63dce5c4450c6dc7dd5611283dda239aabb", size = 10496035, upload-time = "2026-03-29T13:21:02.524Z" }, - { url = "https://files.pythonhosted.org/packages/b1/e8/8fed8c8d848d7ecea092dc3469643f9d10bc3a134a815a3b033da1d2039b/numpy-2.4.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2aa0613a5177c264ff5921051a5719d20095ea586ca88cc802c5c218d1c67d3e", size = 14824958, upload-time = "2026-03-29T13:21:05.671Z" }, - { url = "https://files.pythonhosted.org/packages/05/1a/d8007a5138c179c2bf33ef44503e83d70434d2642877ee8fbb230e7c0548/numpy-2.4.4-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:42c16925aa5a02362f986765f9ebabf20de75cdefdca827d14315c568dcab113", size = 5330020, upload-time = "2026-03-29T13:21:08.635Z" }, - { url = "https://files.pythonhosted.org/packages/99/64/ffb99ac6ae93faf117bcbd5c7ba48a7f45364a33e8e458545d3633615dda/numpy-2.4.4-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:874f200b2a981c647340f841730fc3a2b54c9d940566a3c4149099591e2c4c3d", size = 6650758, upload-time = "2026-03-29T13:21:10.949Z" }, - { url = "https://files.pythonhosted.org/packages/6e/6e/795cc078b78a384052e73b2f6281ff7a700e9bf53bcce2ee579d4f6dd879/numpy-2.4.4-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c9b39d38a9bd2ae1becd7eac1303d031c5c110ad31f2b319c6e7d98b135c934d", size = 15729948, upload-time = "2026-03-29T13:21:14.047Z" }, - { url = "https://files.pythonhosted.org/packages/5f/86/2acbda8cc2af5f3d7bfc791192863b9e3e19674da7b5e533fded124d1299/numpy-2.4.4-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b268594bccac7d7cf5844c7732e3f20c50921d94e36d7ec9b79e9857694b1b2f", size = 16679325, upload-time = "2026-03-29T13:21:17.561Z" }, - { url = "https://files.pythonhosted.org/packages/bc/59/cafd83018f4aa55e0ac6fa92aa066c0a1877b77a615ceff1711c260ffae8/numpy-2.4.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ac6b31e35612a26483e20750126d30d0941f949426974cace8e6b5c58a3657b0", size = 17084883, upload-time = "2026-03-29T13:21:21.106Z" }, - { url = "https://files.pythonhosted.org/packages/f0/85/a42548db84e65ece46ab2caea3d3f78b416a47af387fcbb47ec28e660dc2/numpy-2.4.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8e3ed142f2728df44263aaf5fb1f5b0b99f4070c553a0d7f033be65338329150", size = 18403474, upload-time = "2026-03-29T13:21:24.828Z" }, - { url = "https://files.pythonhosted.org/packages/ed/ad/483d9e262f4b831000062e5d8a45e342166ec8aaa1195264982bca267e62/numpy-2.4.4-cp314-cp314t-win32.whl", hash = "sha256:dddbbd259598d7240b18c9d87c56a9d2fb3b02fe266f49a7c101532e78c1d871", size = 6155500, upload-time = "2026-03-29T13:21:28.205Z" }, - { url = "https://files.pythonhosted.org/packages/c7/03/2fc4e14c7bd4ff2964b74ba90ecb8552540b6315f201df70f137faa5c589/numpy-2.4.4-cp314-cp314t-win_amd64.whl", hash = "sha256:a7164afb23be6e37ad90b2f10426149fd75aee07ca55653d2aa41e66c4ef697e", size = 12637755, upload-time = "2026-03-29T13:21:31.107Z" }, - { url = "https://files.pythonhosted.org/packages/58/78/548fb8e07b1a341746bfbecb32f2c268470f45fa028aacdbd10d9bc73aab/numpy-2.4.4-cp314-cp314t-win_arm64.whl", hash = "sha256:ba203255017337d39f89bdd58417f03c4426f12beed0440cfd933cb15f8669c7", size = 10566643, upload-time = "2026-03-29T13:21:34.339Z" }, + { url = "https://files.pythonhosted.org/packages/fb/82/bdab26d7438c6791ca31b7c024ca37c1eab8b726ba236129005cd4a06e45/numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0", size = 16684648, upload-time = "2026-05-18T23:34:29.41Z" }, + { url = "https://files.pythonhosted.org/packages/1b/30/a80189bcc7f5e4258b3fbc3968d909d1756f54d023299ecc39ad6fdb9ef8/numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb", size = 14693902, upload-time = "2026-05-18T23:34:33.013Z" }, + { url = "https://files.pythonhosted.org/packages/97/12/70b5d0d7c15e1ebb8a6a84a8caa1d19e181d84fb58bb6d70aca29099dec1/numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f", size = 5198992, upload-time = "2026-05-18T23:34:36.132Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8c/ebd2a8f8a83541f8d38cc5667e8c2b69cecfd30da6e45693e8158857d44b/numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3", size = 6546944, upload-time = "2026-05-18T23:34:38.484Z" }, + { url = "https://files.pythonhosted.org/packages/bb/c5/7b863a97a91671a0338f4253bd3b5a3d3852f0692dae91711c9f4a10e787/numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b", size = 15669392, upload-time = "2026-05-18T23:34:41.257Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9d/3584b9984ca4c047aea75214ce1a4c4c73d849bd71b604264b7f5653f8a8/numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089", size = 16633220, upload-time = "2026-05-18T23:34:45.075Z" }, + { url = "https://files.pythonhosted.org/packages/05/ae/7c67fba23bd98caec7c99261f3a16072ade14813486b0282cb29846de832/numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a", size = 17020800, upload-time = "2026-05-18T23:34:49.065Z" }, + { url = "https://files.pythonhosted.org/packages/d9/5d/3b6725cb31d983c5e66916f5d36f6d7e5521129e4c4404d64f918292a5b6/numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605", size = 18357600, upload-time = "2026-05-18T23:34:52.709Z" }, + { url = "https://files.pythonhosted.org/packages/f7/da/2ccc6c2fe8898dee01d90c75c5f5f914a23daf99e3e0f59516a08760c8b5/numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91", size = 5961134, upload-time = "2026-05-18T23:34:55.618Z" }, + { url = "https://files.pythonhosted.org/packages/b5/cd/9cc4dc876fb065d5c220aae4d5e14826b2715331bb7618ce1fb07a679d99/numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359", size = 12318598, upload-time = "2026-05-18T23:34:58.928Z" }, + { url = "https://files.pythonhosted.org/packages/39/1e/c0bcba1f8694116485fe28fd1be698c278fcda4141c5b0e53a2aed8b12a8/numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778", size = 10222272, upload-time = "2026-05-18T23:35:02.167Z" }, + { url = "https://files.pythonhosted.org/packages/63/6d/cc5619247c8f4204e507f5883528372e4ac4bb189e579fb859a12e480b1f/numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1", size = 14821197, upload-time = "2026-05-18T23:35:05.468Z" }, + { url = "https://files.pythonhosted.org/packages/00/58/f1c39161c87d9e9bed660f1ed4bafc0e403d5ec9650b6dd77aead07d489b/numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe", size = 5326287, upload-time = "2026-05-18T23:35:08.693Z" }, + { url = "https://files.pythonhosted.org/packages/af/57/3917ab0fd97f271a8694513581b8a36c655f111c446852c302f04ccdb6fc/numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997", size = 6646763, upload-time = "2026-05-18T23:35:11.459Z" }, + { url = "https://files.pythonhosted.org/packages/eb/0f/037e64c494b67581ae18193d770adef354c41f3f2c8ebf865602d949bf8f/numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20", size = 15728070, upload-time = "2026-05-18T23:35:14.79Z" }, + { url = "https://files.pythonhosted.org/packages/21/a6/5d2bae9c9542eb4df16dc9c46dc79c186e9bad53805dfa5399a6023c6db0/numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d", size = 16681752, upload-time = "2026-05-18T23:35:18.836Z" }, + { url = "https://files.pythonhosted.org/packages/92/14/23d1dfb410ae362cd59ce53e936b1513d545eb40db3949ced632e19a459e/numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67", size = 17086024, upload-time = "2026-05-18T23:35:22.52Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6e/23595a2c642cdf3bc567877064bdd7f91c8b0038a4453cf2daf7248eafe9/numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd", size = 18403398, upload-time = "2026-05-18T23:35:26.398Z" }, + { url = "https://files.pythonhosted.org/packages/8a/90/0ac3bc947217e66dec77e7cbc6a1979d1af70b6461b82f620d3bccd5e4c8/numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab", size = 6084971, upload-time = "2026-05-18T23:35:29.387Z" }, + { url = "https://files.pythonhosted.org/packages/77/71/5673e351671a1d2bd6063b91b44f70c0affea7d1516fa7a6572941ba4aa1/numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75", size = 12458532, upload-time = "2026-05-18T23:35:32.175Z" }, + { url = "https://files.pythonhosted.org/packages/3f/88/19d3503c5046e688f049274b27a3ef3d771152fa80d3ba3d01a3dff61abe/numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd", size = 10291881, upload-time = "2026-05-18T23:35:35.465Z" }, + { url = "https://files.pythonhosted.org/packages/f8/91/3ab2044d05fd16d343c5ac2e69b127f1b2854040dd20b193257c78028bd3/numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079", size = 16683458, upload-time = "2026-05-18T23:35:38.353Z" }, + { url = "https://files.pythonhosted.org/packages/8e/62/764ce66fa4147ae6d73071a3abf804ffe606f174618697c571acdf26a7c9/numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7", size = 14704559, upload-time = "2026-05-18T23:35:42.14Z" }, + { url = "https://files.pythonhosted.org/packages/60/61/23f27c172f022e04025b7dc2367f4d63c1a398120607ec896228649a6f48/numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5", size = 5209716, upload-time = "2026-05-18T23:35:45.377Z" }, + { url = "https://files.pythonhosted.org/packages/03/71/21cf70dc6ea3e3acb95fc53a265b2fc248b981f0194ceb5b475271b8809d/numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096", size = 6543947, upload-time = "2026-05-18T23:35:47.926Z" }, + { url = "https://files.pythonhosted.org/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" }, + { url = "https://files.pythonhosted.org/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b4/298628d98c72b57e57f7165ae6a481a1deaf6f3c28262a6e4c739c275930/numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1", size = 6010196, upload-time = "2026-05-18T23:36:05.92Z" }, + { url = "https://files.pythonhosted.org/packages/df/ac/46de6dda46478f7942f839e094970be2d4a861e005c4b3bf07c92e291a09/numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261", size = 12450334, upload-time = "2026-05-18T23:36:09.107Z" }, + { url = "https://files.pythonhosted.org/packages/78/92/b8b798ac784102c0da830d2257d59358e3d3d90d1e2b3f2575dad976c5cf/numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6", size = 10495678, upload-time = "2026-05-18T23:36:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/30/34/ec28d1aa8115971537c01469ab2011ee96827930f0a124de1000cc2a7ed7/numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a", size = 14823672, upload-time = "2026-05-18T23:36:16.473Z" }, + { url = "https://files.pythonhosted.org/packages/16/bd/f6d1fede4e54e8042a7ff97bb495510f3c220f94bcd9e8b228e87c92cc0d/numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e", size = 5328731, upload-time = "2026-05-18T23:36:19.767Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f0/e105b9e2fd728a9910103884decd6951d9dd73896b914a98d9a231de02ee/numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e", size = 6649805, upload-time = "2026-05-18T23:36:22.266Z" }, + { url = "https://files.pythonhosted.org/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" }, + { url = "https://files.pythonhosted.org/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" }, + { url = "https://files.pythonhosted.org/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" }, + { url = "https://files.pythonhosted.org/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" }, + { url = "https://files.pythonhosted.org/packages/9c/50/0753655aa844c99cd9e018aacf76f130f1bd81d881bb74bc0aef5d73a8ba/numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063", size = 6156982, upload-time = "2026-05-18T23:36:40.817Z" }, + { url = "https://files.pythonhosted.org/packages/b2/d4/7c67becf668f973cb490cec3e98dfd799d866f9c989a54d355672cfa0db6/numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627", size = 12638908, upload-time = "2026-05-18T23:36:43.996Z" }, + { url = "https://files.pythonhosted.org/packages/43/bb/e1c71a4295b1b1d1393d50dbb4f2a36283c6859d9d3892e84f00ec5a91d5/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", size = 10565867, upload-time = "2026-05-18T23:36:47.114Z" }, ] [[package]] name = "orjson" -version = "3.11.8" +version = "3.11.9" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/1b/2024d06792d0779f9dbc51531b61c24f76c75b9f4ce05e6f3377a1814cea/orjson-3.11.8.tar.gz", hash = "sha256:96163d9cdc5a202703e9ad1b9ae757d5f0ca62f4fa0cc93d1f27b0e180cc404e", size = 5603832, upload-time = "2026-03-31T16:16:27.878Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/0c/964746fcafbd16f8ff53219ad9f6b412b34f345c75f384ad434ceaadb538/orjson-3.11.9.tar.gz", hash = "sha256:4fef17e1f8722c11587a6ef18e35902450221da0028e65dbaaa543619e68e48f", size = 5599163, upload-time = "2026-05-06T15:11:08.309Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/66/7f/95fba509bb2305fab0073558f1e8c3a2ec4b2afe58ed9fcb7d3b8beafe94/orjson-3.11.8-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:3f23426851d98478c8970da5991f84784a76682213cd50eb73a1da56b95239dc", size = 229180, upload-time = "2026-03-31T16:15:36.426Z" }, - { url = "https://files.pythonhosted.org/packages/f6/9d/b237215c743ca073697d759b5503abd2cb8a0d7b9c9e21f524bcf176ab66/orjson-3.11.8-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ebaed4cef74a045b83e23537b52ef19a367c7e3f536751e355a2a394f8648559", size = 128754, upload-time = "2026-03-31T16:15:38.049Z" }, - { url = "https://files.pythonhosted.org/packages/42/3d/27d65b6d11e63f133781425f132807aef793ed25075fec686fc8e46dd528/orjson-3.11.8-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97c8f5d3b62380b70c36ffacb2a356b7c6becec86099b177f73851ba095ef623", size = 131877, upload-time = "2026-03-31T16:15:39.484Z" }, - { url = "https://files.pythonhosted.org/packages/dd/cc/faee30cd8f00421999e40ef0eba7332e3a625ce91a58200a2f52c7fef235/orjson-3.11.8-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:436c4922968a619fb7fef1ccd4b8b3a76c13b67d607073914d675026e911a65c", size = 130361, upload-time = "2026-03-31T16:15:41.274Z" }, - { url = "https://files.pythonhosted.org/packages/5c/bb/a6c55896197f97b6d4b4e7c7fd77e7235517c34f5d6ad5aadd43c54c6d7c/orjson-3.11.8-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1ab359aff0436d80bfe8a23b46b5fea69f1e18aaf1760a709b4787f1318b317f", size = 135521, upload-time = "2026-03-31T16:15:42.758Z" }, - { url = "https://files.pythonhosted.org/packages/9c/7c/ca3a3525aa32ff636ebb1778e77e3587b016ab2edb1b618b36ba96f8f2c0/orjson-3.11.8-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f89b6d0b3a8d81e1929d3ab3d92bbc225688bd80a770c49432543928fe09ac55", size = 146862, upload-time = "2026-03-31T16:15:44.341Z" }, - { url = "https://files.pythonhosted.org/packages/3c/0c/18a9d7f18b5edd37344d1fd5be17e94dc652c67826ab749c6e5948a78112/orjson-3.11.8-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29c009e7a2ca9ad0ed1376ce20dd692146a5d9fe4310848904b6b4fee5c5c137", size = 132847, upload-time = "2026-03-31T16:15:46.368Z" }, - { url = "https://files.pythonhosted.org/packages/23/91/7e722f352ad67ca573cee44de2a58fb810d0f4eb4e33276c6a557979fd8a/orjson-3.11.8-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:705b895b781b3e395c067129d8551655642dfe9437273211d5404e87ac752b53", size = 133637, upload-time = "2026-03-31T16:15:48.123Z" }, - { url = "https://files.pythonhosted.org/packages/af/04/32845ce13ac5bd1046ddb02ac9432ba856cc35f6d74dde95864fe0ad5523/orjson-3.11.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:88006eda83858a9fdf73985ce3804e885c2befb2f506c9a3723cdeb5a2880e3e", size = 141906, upload-time = "2026-03-31T16:15:49.626Z" }, - { url = "https://files.pythonhosted.org/packages/02/5e/c551387ddf2d7106d9039369862245c85738b828844d13b99ccb8d61fd06/orjson-3.11.8-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:55120759e61309af7fcf9e961c6f6af3dde5921cdb3ee863ef63fd9db126cae6", size = 423722, upload-time = "2026-03-31T16:15:51.176Z" }, - { url = "https://files.pythonhosted.org/packages/00/a3/ecfe62434096f8a794d4976728cb59bcfc4a643977f21c2040545d37eb4c/orjson-3.11.8-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:98bdc6cb889d19bed01de46e67574a2eab61f5cc6b768ed50e8ac68e9d6ffab6", size = 147801, upload-time = "2026-03-31T16:15:52.939Z" }, - { url = "https://files.pythonhosted.org/packages/18/6d/0dce10b9f6643fdc59d99333871a38fa5a769d8e2fc34a18e5d2bfdee900/orjson-3.11.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:708c95f925a43ab9f34625e45dcdadf09ec8a6e7b664a938f2f8d5650f6c090b", size = 136460, upload-time = "2026-03-31T16:15:54.431Z" }, - { url = "https://files.pythonhosted.org/packages/01/d6/6dde4f31842d87099238f1f07b459d24edc1a774d20687187443ab044191/orjson-3.11.8-cp313-cp313-win32.whl", hash = "sha256:01c4e5a6695dc09098f2e6468a251bc4671c50922d4d745aff1a0a33a0cf5b8d", size = 131956, upload-time = "2026-03-31T16:15:56.081Z" }, - { url = "https://files.pythonhosted.org/packages/c1/f9/4e494a56e013db957fb77186b818b916d4695b8fa2aa612364974160e91b/orjson-3.11.8-cp313-cp313-win_amd64.whl", hash = "sha256:c154a35dd1330707450bb4d4e7dd1f17fa6f42267a40c1e8a1daa5e13719b4b8", size = 127410, upload-time = "2026-03-31T16:15:57.54Z" }, - { url = "https://files.pythonhosted.org/packages/57/7f/803203d00d6edb6e9e7eef421d4e1adbb5ea973e40b3533f3cfd9aeb374e/orjson-3.11.8-cp313-cp313-win_arm64.whl", hash = "sha256:4861bde57f4d253ab041e374f44023460e60e71efaa121f3c5f0ed457c3a701e", size = 127338, upload-time = "2026-03-31T16:15:59.106Z" }, - { url = "https://files.pythonhosted.org/packages/6d/35/b01910c3d6b85dc882442afe5060cbf719c7d1fc85749294beda23d17873/orjson-3.11.8-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:ec795530a73c269a55130498842aaa762e4a939f6ce481a7e986eeaa790e9da4", size = 229171, upload-time = "2026-03-31T16:16:00.651Z" }, - { url = "https://files.pythonhosted.org/packages/c2/56/c9ec97bd11240abef39b9e5d99a15462809c45f677420fd148a6c5e6295e/orjson-3.11.8-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:c492a0e011c0f9066e9ceaa896fbc5b068c54d365fea5f3444b697ee01bc8625", size = 128746, upload-time = "2026-03-31T16:16:02.673Z" }, - { url = "https://files.pythonhosted.org/packages/3b/e4/66d4f30a90de45e2f0cbd9623588e8ae71eef7679dbe2ae954ed6d66a41f/orjson-3.11.8-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:883206d55b1bd5f5679ad5e6ddd3d1a5e3cac5190482927fdb8c78fb699193b5", size = 131867, upload-time = "2026-03-31T16:16:04.342Z" }, - { url = "https://files.pythonhosted.org/packages/19/30/2a645fc9286b928675e43fa2a3a16fb7b6764aa78cc719dc82141e00f30b/orjson-3.11.8-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5774c1fdcc98b2259800b683b19599c133baeb11d60033e2095fd9d4667b82db", size = 124664, upload-time = "2026-03-31T16:16:05.837Z" }, - { url = "https://files.pythonhosted.org/packages/db/44/77b9a86d84a28d52ba3316d77737f6514e17118119ade3f91b639e859029/orjson-3.11.8-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ac7381c83dd3d4a6347e6635950aa448f54e7b8406a27c7ecb4a37e9f1ae08b", size = 129701, upload-time = "2026-03-31T16:16:07.407Z" }, - { url = "https://files.pythonhosted.org/packages/b3/ea/eff3d9bfe47e9bc6969c9181c58d9f71237f923f9c86a2d2f490cd898c82/orjson-3.11.8-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:14439063aebcb92401c11afc68ee4e407258d2752e62d748b6942dad20d2a70d", size = 141202, upload-time = "2026-03-31T16:16:09.48Z" }, - { url = "https://files.pythonhosted.org/packages/52/c8/90d4b4c60c84d62068d0cf9e4d8f0a4e05e76971d133ac0c60d818d4db20/orjson-3.11.8-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa72e71977bff96567b0f500fc5bfd2fdf915f34052c782a4c6ebbdaa97aa858", size = 127194, upload-time = "2026-03-31T16:16:11.02Z" }, - { url = "https://files.pythonhosted.org/packages/8d/c7/ea9e08d1f0ba981adffb629811148b44774d935171e7b3d780ae43c4c254/orjson-3.11.8-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7679bc2f01bb0d219758f1a5f87bb7c8a81c0a186824a393b366876b4948e14f", size = 133639, upload-time = "2026-03-31T16:16:13.434Z" }, - { url = "https://files.pythonhosted.org/packages/6c/8c/ddbbfd6ba59453c8fc7fe1d0e5983895864e264c37481b2a791db635f046/orjson-3.11.8-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:14f7b8fcb35ef403b42fa5ecfa4ed032332a91f3dc7368fbce4184d59e1eae0d", size = 141914, upload-time = "2026-03-31T16:16:14.955Z" }, - { url = "https://files.pythonhosted.org/packages/4e/31/dbfbefec9df060d34ef4962cd0afcb6fa7a9ec65884cb78f04a7859526c3/orjson-3.11.8-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:c2bdf7b2facc80b5e34f48a2d557727d5c5c57a8a450de122ae81fa26a81c1bc", size = 423800, upload-time = "2026-03-31T16:16:16.594Z" }, - { url = "https://files.pythonhosted.org/packages/87/cf/f74e9ae9803d4ab46b163494adba636c6d7ea955af5cc23b8aaa94cfd528/orjson-3.11.8-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ccd7ba1b0605813a0715171d39ec4c314cb97a9c85893c2c5c0c3a3729df38bf", size = 147837, upload-time = "2026-03-31T16:16:18.585Z" }, - { url = "https://files.pythonhosted.org/packages/64/e6/9214f017b5db85e84e68602792f742e5dc5249e963503d1b356bee611e01/orjson-3.11.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cdbc8c9c02463fef4d3c53a9ba3336d05496ec8e1f1c53326a1e4acc11f5c600", size = 136441, upload-time = "2026-03-31T16:16:20.151Z" }, - { url = "https://files.pythonhosted.org/packages/24/dd/3590348818f58f837a75fb969b04cdf187ae197e14d60b5e5a794a38b79d/orjson-3.11.8-cp314-cp314-win32.whl", hash = "sha256:0b57f67710a8cd459e4e54eb96d5f77f3624eba0c661ba19a525807e42eccade", size = 131983, upload-time = "2026-03-31T16:16:21.823Z" }, - { url = "https://files.pythonhosted.org/packages/3f/0f/b6cb692116e05d058f31ceee819c70f097fa9167c82f67fabe7516289abc/orjson-3.11.8-cp314-cp314-win_amd64.whl", hash = "sha256:735e2262363dcbe05c35e3a8869898022af78f89dde9e256924dc02e99fe69ca", size = 127396, upload-time = "2026-03-31T16:16:23.685Z" }, - { url = "https://files.pythonhosted.org/packages/c0/d1/facb5b5051fabb0ef9d26c6544d87ef19a939a9a001198655d0d891062dd/orjson-3.11.8-cp314-cp314-win_arm64.whl", hash = "sha256:6ccdea2c213cf9f3d9490cbd5d427693c870753df41e6cb375bd79bcbafc8817", size = 127330, upload-time = "2026-03-31T16:16:25.496Z" }, + { url = "https://files.pythonhosted.org/packages/32/33/93fcc25907235c344ae73122f8a4e01d2d393ef062b4af7d2e2487a32c37/orjson-3.11.9-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:4bab1b2d6141fe7b32ae71dac905666ece4f94936efbfb13d55bb7739a3a6021", size = 228458, upload-time = "2026-05-06T15:10:20.079Z" }, + { url = "https://files.pythonhosted.org/packages/8f/27/b1e6dadb3c080313c03fdd8067b85e6a0460c7d8d6a1c3984ef77b904e4d/orjson-3.11.9-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:844417969855fc7a41be124aafe83dc424592a7f77cd4501900c67307122b92c", size = 128368, upload-time = "2026-05-06T15:10:21.549Z" }, + { url = "https://files.pythonhosted.org/packages/21/0f/c9ede0bf052f6b4051e64a7d4fa91b725cccf8321a6a786e86eb03519f00/orjson-3.11.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffe02797b5e9f3a9d8292ddcd289b474ad13e81ad83cd1891a240811f1d2cb81", size = 132070, upload-time = "2026-05-06T15:10:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/fd/26/d398e28048dc18205bbe812f2c88cb9b40313db2470778e25964796458fe/orjson-3.11.9-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e4eed3b200023042814d2fc8a5d2e880f13b52e1ed2485e83da4f3962f7dc1a", size = 127892, upload-time = "2026-05-06T15:10:24.714Z" }, + { url = "https://files.pythonhosted.org/packages/66/60/52b0054c4c700d5aa7fc5b7ca96917400d8f061307778578e67a10e25852/orjson-3.11.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aff7da9952a5ad1cef8e68017724d96c7b9a66e99e91d6252e1b133d67a7b10", size = 135217, upload-time = "2026-05-06T15:10:26.084Z" }, + { url = "https://files.pythonhosted.org/packages/d5/97/1e3dc2b2a28b7b2528f403d2fc1d79ec5f39af3bc143ab65d3ec26426385/orjson-3.11.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d4e98d6f3b8afed8bc8cd9718ec0cdf46661826beefb53fe8eafb37f2bf0362", size = 145980, upload-time = "2026-05-06T15:10:28.062Z" }, + { url = "https://files.pythonhosted.org/packages/fc/39/31fbfe7850f2de32dee7e7e5c09f26d403ab01e440ac96001c6b01ad3c99/orjson-3.11.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a81d52442a7c99b3662333235b3adf96a1715864658b35bb797212be7bddb97", size = 132738, upload-time = "2026-05-06T15:10:29.727Z" }, + { url = "https://files.pythonhosted.org/packages/a1/08/dca0082dd2a194acb93e5457e73455388e2e2ca464a2672449a9ddbb679d/orjson-3.11.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e39364e726a8fff737309aff059ff67d8a8c8d5b677be7bb49a8b3e84b7e218", size = 134033, upload-time = "2026-05-06T15:10:31.152Z" }, + { url = "https://files.pythonhosted.org/packages/11/d4/5bdb0626801230139987385554c5d4c42255218ac906525bf4347f22cd95/orjson-3.11.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4fd66214623f1b17501df9f0543bef0b833979ab5b6ded1e1d123222866aa8c9", size = 141492, upload-time = "2026-05-06T15:10:32.641Z" }, + { url = "https://files.pythonhosted.org/packages/fa/88/a21fb53b3ede6703aede6dce4710ed4111e5b201cfa6bbff5e544f9d47d7/orjson-3.11.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8ecc30f10465fa1e0ce13fd01d9e22c316e5053a719a8d915d4545a09a5ff677", size = 415087, upload-time = "2026-05-06T15:10:34.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/57/1b30daf70f0d8180e9a73cefbfbdd99e4bf19eb020466502b01fba7e0e50/orjson-3.11.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:97db4c94a7db398a5bd636273324f0b3fd58b350bbbac8bb380ceb825a9b40f4", size = 148031, upload-time = "2026-05-06T15:10:36.358Z" }, + { url = "https://files.pythonhosted.org/packages/04/83/45fbb6d962e260807f99441db9613cee868ceda4baceda59b3720a563f97/orjson-3.11.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f78cf8fec5bd627f4082b8dfeac7871b43d7f3274904492a43dab39f18a19a0", size = 136915, upload-time = "2026-05-06T15:10:38.013Z" }, + { url = "https://files.pythonhosted.org/packages/5f/cc/2d10025f9056d376e4127ec05a5808b218d46f035fdc08178a5411b34250/orjson-3.11.9-cp313-cp313-win32.whl", hash = "sha256:d4087e5c0209a0a8efe4de3303c234b9c44d1174161dcd851e8eea07c7560b32", size = 131613, upload-time = "2026-05-06T15:10:39.569Z" }, + { url = "https://files.pythonhosted.org/packages/67/bd/2775ff28bfe883b9aa1ff348300542eb2ef1ee18d8ae0e3a49846817a865/orjson-3.11.9-cp313-cp313-win_amd64.whl", hash = "sha256:051b102c93b4f634e89f3866b07b9a9a98915ada541f4ec30f177067b2694979", size = 127086, upload-time = "2026-05-06T15:10:41.262Z" }, + { url = "https://files.pythonhosted.org/packages/91/2b/d26799e580939e32a7da9a39531bc9e58e15ca32ffaa6a8cb3e9bb0d22cd/orjson-3.11.9-cp313-cp313-win_arm64.whl", hash = "sha256:cce9127885941bd28f080cecf1f1d288336b7e0d812c345b08be88b572796254", size = 126696, upload-time = "2026-05-06T15:10:42.651Z" }, + { url = "https://files.pythonhosted.org/packages/8e/eb/5da01e356015aee6ecfa1187ced87aef51364e306f5e695dd52719bf0e78/orjson-3.11.9-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b6ef1979adc4bc243523f1a2ba91418030a8e29b0a99cbe7e0e2d6807d4dce6e", size = 228465, upload-time = "2026-05-06T15:10:44.097Z" }, + { url = "https://files.pythonhosted.org/packages/64/62/3e0e0c14c957133bcd855395c62b55ed4e3b0af23ffea11b032cb1dcbdb1/orjson-3.11.9-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:f36b7f32c7c0db4a719f1fc5824db4a9c6f8bd1a354debb91faf26ebf3a4c71e", size = 128364, upload-time = "2026-05-06T15:10:45.839Z" }, + { url = "https://files.pythonhosted.org/packages/5a/5a/07d8aa117211a8ed7630bda80c8c0b14d04e0f8dcf99bcf49656e4a710eb/orjson-3.11.9-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08f4d8ebb44925c794e535b2bebc507cebf32209df81de22ae285fb0d8d66de0", size = 132063, upload-time = "2026-05-06T15:10:47.267Z" }, + { url = "https://files.pythonhosted.org/packages/d6/ec/4acaf21483e18aa945be74a474c74b434f284b549f275a0a39b9f98956e9/orjson-3.11.9-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6cc7923789694fd58f001cbcac7e47abc13af4d560ebbfcf3b41a8b1a0748124", size = 122356, upload-time = "2026-05-06T15:10:48.765Z" }, + { url = "https://files.pythonhosted.org/packages/13/d8/5f0555e7638801323b7a75850f92e7dfa891bc84fe27a1ba4449170d1200/orjson-3.11.9-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea5c46eb2d3af39e806b986f4b09d5c2706a1f5afde3cbf7544ce6616127173c", size = 129592, upload-time = "2026-05-06T15:10:50.13Z" }, + { url = "https://files.pythonhosted.org/packages/b6/30/ed9860412a3603ceb3c5955bfd72d28b9d0e7ba6ed81add14f83d7114236/orjson-3.11.9-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f5d89a2ed90731df3be64bab0aa44f78bff39fdc9d71c291f4a8023aa46425b7", size = 140491, upload-time = "2026-05-06T15:10:51.582Z" }, + { url = "https://files.pythonhosted.org/packages/d0/17/adc514dea7ac7c505527febf884934b815d34f0c7b8693c1a8b39c5c4a57/orjson-3.11.9-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25e4aed0312d292c09f61af25bba34e0b2c88546041472b09088c39a4d828af1", size = 127309, upload-time = "2026-05-06T15:10:53.329Z" }, + { url = "https://files.pythonhosted.org/packages/76/3e/c0b690253f0b82d86e99949af13533363acfb5432ecb5d53dd5b3bce9c34/orjson-3.11.9-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaea64f3f467d22e70eeed68bdccb3bc4f83f650446c4a03c59f2cba28a108db", size = 134030, upload-time = "2026-05-06T15:10:54.988Z" }, + { url = "https://files.pythonhosted.org/packages/c1/7a/bc82a0bb25e9faaf92dc4d9ef002732efc09737706af83e346788641d4a7/orjson-3.11.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a028425d1b440c5d92a6be1e1a020739dfe67ea87d96c6dbe828c1b30041728b", size = 141482, upload-time = "2026-05-06T15:10:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/01/55/e69188b939f77d5d32a9833745ace31ea5ccae3ab613a1ec185d3cd2c4fb/orjson-3.11.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5b192c6cf397e4455b11523c5cf2b18ed084c1bbd61b6c0926344d2129481972", size = 415178, upload-time = "2026-05-06T15:10:58.446Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1a/b8a5a7ac527e80b9cb11d51e3f6689b709279183264b9ec5c7bc680bb8b5/orjson-3.11.9-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea407d4ccf5891d667d045fecae97a7a1e5e87b3b97f97ae1803c2e741130be0", size = 148089, upload-time = "2026-05-06T15:11:00.441Z" }, + { url = "https://files.pythonhosted.org/packages/97/4e/00503f64204bf859b37213a63927028f30fb6268cd8677fb0a5ad48155e1/orjson-3.11.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f63aaf97afd9f6dec5b1a68e1b8da12bfccb4cb9a9a65c3e0b6c847849e7586", size = 136921, upload-time = "2026-05-06T15:11:02.176Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ba/a23b82a0a8d0ed7bed4e5f5035aae751cad4ff6a1e8d2ecd14d8860f5929/orjson-3.11.9-cp314-cp314-win32.whl", hash = "sha256:e30ab17845bb9fa54ccf67fa4f9f5282652d54faa6d17452f47d0f369d038673", size = 131638, upload-time = "2026-05-06T15:11:03.696Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c3/0c6798456bade745c75c452342dabacce5798196483e77e643be1f53877d/orjson-3.11.9-cp314-cp314-win_amd64.whl", hash = "sha256:32ef5f4283a3be81913947d19608eacb7c6608026851123790cd9cc8982af34b", size = 127078, upload-time = "2026-05-06T15:11:05.123Z" }, + { url = "https://files.pythonhosted.org/packages/16/21/5a3f1e8913103b703a436a5664238e5b965ec392b555fe68943ea3691e6b/orjson-3.11.9-cp314-cp314-win_arm64.whl", hash = "sha256:eebdbdeef0094e4f5aefa20dcd4eb2368ab5e7a3b4edea27f1e7b2892e009cf9", size = 126687, upload-time = "2026-05-06T15:11:06.602Z" }, ] [[package]] @@ -790,55 +798,55 @@ wheels = [ [[package]] name = "pandas" -version = "3.0.2" +version = "3.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "python-dateutil" }, { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/da/99/b342345300f13440fe9fe385c3c481e2d9a595ee3bab4d3219247ac94e9a/pandas-3.0.2.tar.gz", hash = "sha256:f4753e73e34c8d83221ba58f232433fca2748be8b18dbca02d242ed153945043", size = 4645855, upload-time = "2026-03-31T06:48:30.816Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/ca/3e639a1ea6fcd0617ca4e8ca45f62a74de33a56ae6cd552735470b22c8d3/pandas-3.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b5918ba197c951dec132b0c5929a00c0bf05d5942f590d3c10a807f6e15a57d3", size = 10321105, upload-time = "2026-03-31T06:46:57.327Z" }, - { url = "https://files.pythonhosted.org/packages/0b/77/dbc82ff2fb0e63c6564356682bf201edff0ba16c98630d21a1fb312a8182/pandas-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d606a041c89c0a474a4702d532ab7e73a14fe35c8d427b972a625c8e46373668", size = 9864088, upload-time = "2026-03-31T06:46:59.935Z" }, - { url = "https://files.pythonhosted.org/packages/5c/2b/341f1b04bbca2e17e13cd3f08c215b70ef2c60c5356ef1e8c6857449edc7/pandas-3.0.2-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:710246ba0616e86891b58ab95f2495143bb2bc83ab6b06747c74216f583a6ac9", size = 10369066, upload-time = "2026-03-31T06:47:02.792Z" }, - { url = "https://files.pythonhosted.org/packages/12/c5/cbb1ffefb20a93d3f0e1fdcda699fb84976210d411b008f97f48bf6ce27e/pandas-3.0.2-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5d3cfe227c725b1f3dff4278b43d8c784656a42a9325b63af6b1492a8232209e", size = 10876780, upload-time = "2026-03-31T06:47:06.205Z" }, - { url = "https://files.pythonhosted.org/packages/98/fe/2249ae5e0a69bd0ddf17353d0a5d26611d70970111f5b3600cdc8be883e7/pandas-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c3b723df9087a9a9a840e263ebd9f88b64a12075d1bf2ea401a5a42f254f084d", size = 11375181, upload-time = "2026-03-31T06:47:09.383Z" }, - { url = "https://files.pythonhosted.org/packages/de/64/77a38b09e70b6464883b8d7584ab543e748e42c1b5d337a2ee088e0df741/pandas-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a3096110bf9eac0070b7208465f2740e2d8a670d5cb6530b5bb884eca495fd39", size = 11928899, upload-time = "2026-03-31T06:47:12.686Z" }, - { url = "https://files.pythonhosted.org/packages/5e/52/42855bf626868413f761addd574acc6195880ae247a5346477a4361c3acb/pandas-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:07a10f5c36512eead51bc578eb3354ad17578b22c013d89a796ab5eee90cd991", size = 9746574, upload-time = "2026-03-31T06:47:15.64Z" }, - { url = "https://files.pythonhosted.org/packages/88/39/21304ae06a25e8bf9fc820d69b29b2c495b2ae580d1e143146c309941760/pandas-3.0.2-cp313-cp313-win_arm64.whl", hash = "sha256:5fdbfa05931071aba28b408e59226186b01eb5e92bea2ab78b65863ca3228d84", size = 9047156, upload-time = "2026-03-31T06:47:18.595Z" }, - { url = "https://files.pythonhosted.org/packages/72/20/7defa8b27d4f330a903bb68eea33be07d839c5ea6bdda54174efcec0e1d2/pandas-3.0.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:dbc20dea3b9e27d0e66d74c42b2d0c1bed9c2ffe92adea33633e3bedeb5ac235", size = 10756238, upload-time = "2026-03-31T06:47:22.012Z" }, - { url = "https://files.pythonhosted.org/packages/e9/95/49433c14862c636afc0e9b2db83ff16b3ad92959364e52b2955e44c8e94c/pandas-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b75c347eff42497452116ce05ef461822d97ce5b9ff8df6edacb8076092c855d", size = 10408520, upload-time = "2026-03-31T06:47:25.197Z" }, - { url = "https://files.pythonhosted.org/packages/3b/f8/462ad2b5881d6b8ec8e5f7ed2ea1893faa02290d13870a1600fe72ad8efc/pandas-3.0.2-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1478075142e83a5571782ad007fb201ed074bdeac7ebcc8890c71442e96adf7", size = 10324154, upload-time = "2026-03-31T06:47:28.097Z" }, - { url = "https://files.pythonhosted.org/packages/0a/65/d1e69b649cbcddda23ad6e4c40ef935340f6f652a006e5cbc3555ac8adb3/pandas-3.0.2-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5880314e69e763d4c8b27937090de570f1fb8d027059a7ada3f7f8e98bdcb677", size = 10714449, upload-time = "2026-03-31T06:47:30.85Z" }, - { url = "https://files.pythonhosted.org/packages/47/a4/85b59bc65b8190ea3689882db6cdf32a5003c0ccd5a586c30fdcc3ffc4fc/pandas-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b5329e26898896f06035241a626d7c335daa479b9bbc82be7c2742d048e41172", size = 11338475, upload-time = "2026-03-31T06:47:34.026Z" }, - { url = "https://files.pythonhosted.org/packages/1e/c4/bc6966c6e38e5d9478b935272d124d80a589511ed1612a5d21d36f664c68/pandas-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:81526c4afd31971f8b62671442a4b2b51e0aa9acc3819c9f0f12a28b6fcf85f1", size = 11786568, upload-time = "2026-03-31T06:47:36.941Z" }, - { url = "https://files.pythonhosted.org/packages/e8/74/09298ca9740beed1d3504e073d67e128aa07e5ca5ca2824b0c674c0b8676/pandas-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:7cadd7e9a44ec13b621aec60f9150e744cfc7a3dd32924a7e2f45edff31823b0", size = 10488652, upload-time = "2026-03-31T06:47:40.612Z" }, - { url = "https://files.pythonhosted.org/packages/bb/40/c6ea527147c73b24fc15c891c3fcffe9c019793119c5742b8784a062c7db/pandas-3.0.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:db0dbfd2a6cdf3770aa60464d50333d8f3d9165b2f2671bcc299b72de5a6677b", size = 10326084, upload-time = "2026-03-31T06:47:43.834Z" }, - { url = "https://files.pythonhosted.org/packages/95/25/bdb9326c3b5455f8d4d3549fce7abcf967259de146fe2cf7a82368141948/pandas-3.0.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0555c5882688a39317179ab4a0ed41d3ebc8812ab14c69364bbee8fb7a3f6288", size = 9914146, upload-time = "2026-03-31T06:47:46.67Z" }, - { url = "https://files.pythonhosted.org/packages/8d/77/3a227ff3337aa376c60d288e1d61c5d097131d0ac71f954d90a8f369e422/pandas-3.0.2-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:01f31a546acd5574ef77fe199bc90b55527c225c20ccda6601cf6b0fd5ed597c", size = 10444081, upload-time = "2026-03-31T06:47:49.681Z" }, - { url = "https://files.pythonhosted.org/packages/15/88/3cdd54fa279341afa10acf8d2b503556b1375245dccc9315659f795dd2e9/pandas-3.0.2-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:deeca1b5a931fdf0c2212c8a659ade6d3b1edc21f0914ce71ef24456ca7a6535", size = 10897535, upload-time = "2026-03-31T06:47:53.033Z" }, - { url = "https://files.pythonhosted.org/packages/06/9d/98cc7a7624f7932e40f434299260e2917b090a579d75937cb8a57b9d2de3/pandas-3.0.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0f48afd9bb13300ffb5a3316973324c787054ba6665cda0da3fbd67f451995db", size = 11446992, upload-time = "2026-03-31T06:47:56.193Z" }, - { url = "https://files.pythonhosted.org/packages/9a/cd/19ff605cc3760e80602e6826ddef2824d8e7050ed80f2e11c4b079741dc3/pandas-3.0.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6c4d8458b97a35717b62469a4ea0e85abd5ed8687277f5ccfc67f8a5126f8c53", size = 11968257, upload-time = "2026-03-31T06:47:59.137Z" }, - { url = "https://files.pythonhosted.org/packages/db/60/aba6a38de456e7341285102bede27514795c1eaa353bc0e7638b6b785356/pandas-3.0.2-cp314-cp314-win_amd64.whl", hash = "sha256:b35d14bb5d8285d9494fe93815a9e9307c0876e10f1e8e89ac5b88f728ec8dcf", size = 9865893, upload-time = "2026-03-31T06:48:02.038Z" }, - { url = "https://files.pythonhosted.org/packages/08/71/e5ec979dd2e8a093dacb8864598c0ff59a0cee0bbcdc0bfec16a51684d4f/pandas-3.0.2-cp314-cp314-win_arm64.whl", hash = "sha256:63d141b56ef686f7f0d714cfb8de4e320475b86bf4b620aa0b7da89af8cbdbbb", size = 9188644, upload-time = "2026-03-31T06:48:05.045Z" }, - { url = "https://files.pythonhosted.org/packages/f1/6c/7b45d85db19cae1eb524f2418ceaa9d85965dcf7b764ed151386b7c540f0/pandas-3.0.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:140f0cffb1fa2524e874dde5b477d9defe10780d8e9e220d259b2c0874c89d9d", size = 10776246, upload-time = "2026-03-31T06:48:07.789Z" }, - { url = "https://files.pythonhosted.org/packages/a8/3e/7b00648b086c106e81766f25322b48aa8dfa95b55e621dbdf2fdd413a117/pandas-3.0.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ae37e833ff4fed0ba352f6bdd8b73ba3ab3256a85e54edfd1ab51ae40cca0af8", size = 10424801, upload-time = "2026-03-31T06:48:10.897Z" }, - { url = "https://files.pythonhosted.org/packages/da/6e/558dd09a71b53b4008e7fc8a98ec6d447e9bfb63cdaeea10e5eb9b2dabe8/pandas-3.0.2-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d888a5c678a419a5bb41a2a93818e8ed9fd3172246555c0b37b7cc27027effd", size = 10345643, upload-time = "2026-03-31T06:48:13.7Z" }, - { url = "https://files.pythonhosted.org/packages/be/e3/921c93b4d9a280409451dc8d07b062b503bbec0531d2627e73a756e99a82/pandas-3.0.2-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b444dc64c079e84df91baa8bf613d58405645461cabca929d9178f2cd392398d", size = 10743641, upload-time = "2026-03-31T06:48:16.659Z" }, - { url = "https://files.pythonhosted.org/packages/56/ca/fd17286f24fa3b4d067965d8d5d7e14fe557dd4f979a0b068ac0deaf8228/pandas-3.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4544c7a54920de8eeacaa1466a6b7268ecfbc9bc64ab4dbb89c6bbe94d5e0660", size = 11361993, upload-time = "2026-03-31T06:48:19.475Z" }, - { url = "https://files.pythonhosted.org/packages/e4/a5/2f6ed612056819de445a433ca1f2821ac3dab7f150d569a59e9cc105de1d/pandas-3.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:734be7551687c00fbd760dc0522ed974f82ad230d4a10f54bf51b80d44a08702", size = 11815274, upload-time = "2026-03-31T06:48:22.695Z" }, - { url = "https://files.pythonhosted.org/packages/00/2f/b622683e99ec3ce00b0854bac9e80868592c5b051733f2cf3a868e5fea26/pandas-3.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:57a07209bebcbcf768d2d13c9b78b852f9a15978dac41b9e6421a81ad4cdd276", size = 10888530, upload-time = "2026-03-31T06:48:25.806Z" }, - { url = "https://files.pythonhosted.org/packages/cb/2b/f8434233fab2bd66a02ec014febe4e5adced20e2693e0e90a07d118ed30e/pandas-3.0.2-cp314-cp314t-win_arm64.whl", hash = "sha256:5371b72c2d4d415d08765f32d689217a43227484e81b2305b52076e328f6f482", size = 9455341, upload-time = "2026-03-31T06:48:28.418Z" }, + { url = "https://files.pythonhosted.org/packages/c5/90/62d8302883c44308c477e222c3daf7c813a34c8e96985882fbd53d964352/pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa", size = 10331071, upload-time = "2026-05-11T18:52:58.838Z" }, + { url = "https://files.pythonhosted.org/packages/7f/ae/6a6493c783a101f165e4356953ba3c74d6f77f0042fa7d753da9dfbb640c/pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7", size = 9875690, upload-time = "2026-05-11T18:53:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/62/7c/5df8e9f56c69a2769fbe9382a5ef8f2658c007e376434e1e2cbb57ad895f/pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8", size = 10381634, upload-time = "2026-05-11T18:53:04.393Z" }, + { url = "https://files.pythonhosted.org/packages/99/68/1237369725aa617bb358263d535803e3053fdbc593513ec5ed9c9896b5b6/pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a", size = 10891243, upload-time = "2026-05-11T18:53:07.643Z" }, + { url = "https://files.pythonhosted.org/packages/25/93/77d108e8af7222b4a503ebde0e30215b1c2e4f8e53a526431890f22d5586/pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb", size = 11388659, upload-time = "2026-05-11T18:53:10.634Z" }, + { url = "https://files.pythonhosted.org/packages/d0/bd/eff5b4399f332ac386c853f6cd2bd3fa2ca0061b9f36ecd9c4d7c4265649/pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2", size = 11942880, upload-time = "2026-05-11T18:53:13.536Z" }, + { url = "https://files.pythonhosted.org/packages/2c/20/559ace4200982c3887d0b86bfd0d856a2143ef8ddab63cc07934951a964c/pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44", size = 9757091, upload-time = "2026-05-11T18:53:16.306Z" }, + { url = "https://files.pythonhosted.org/packages/3a/66/69055a09fe200f29f922a3eeec4804611900b95f52d932ece3393c3c0c19/pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e", size = 9057282, upload-time = "2026-05-11T18:53:18.768Z" }, + { url = "https://files.pythonhosted.org/packages/57/0e/efe801b0e6811e8e650cd21b7f2608e30f08a7067e2bf6e8752b0d56ee3c/pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d", size = 10767016, upload-time = "2026-05-11T18:53:21.227Z" }, + { url = "https://files.pythonhosted.org/packages/ea/dc/eb55135a1d5f0f0519f28da1f609a206d2cad1f9c35c32d51e38dd7261ae/pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066", size = 10420210, upload-time = "2026-05-11T18:53:23.982Z" }, + { url = "https://files.pythonhosted.org/packages/c6/3e/b1d5d955ce33ffecb407465a60bc32769d74fcf68224b7ae67ae11d4dea4/pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd", size = 10336126, upload-time = "2026-05-11T18:53:26.731Z" }, + { url = "https://files.pythonhosted.org/packages/f5/76/a01261711ab60a22d71b862f0de20e4c504bf80457270ad8cb42110f6abc/pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085", size = 10728051, upload-time = "2026-05-11T18:53:29.125Z" }, + { url = "https://files.pythonhosted.org/packages/e9/21/ea191195e587b18cf682e97f433f81b2d0fbe341380e80a3e0d6e4403c8e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870", size = 11350796, upload-time = "2026-05-11T18:53:32.056Z" }, + { url = "https://files.pythonhosted.org/packages/64/69/f0eaaf54939f0e8c6768fd06be9af2cef9b36048b96dfb9e1b2c685a807e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f", size = 11799741, upload-time = "2026-05-11T18:53:34.985Z" }, + { url = "https://files.pythonhosted.org/packages/45/a4/865e0e510cae5fc2194de4db28be638952de942571ba9125934fd9c01d47/pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13", size = 10499958, upload-time = "2026-05-11T18:53:37.857Z" }, + { url = "https://files.pythonhosted.org/packages/86/54/effdcc3c0ff7a08037889200e148ebe94c16c4f653be078c7b3675955df1/pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac", size = 10336065, upload-time = "2026-05-11T18:53:41.099Z" }, + { url = "https://files.pythonhosted.org/packages/68/10/bf2d6738d72748b961a3751ab89522d58c54efc36a8e1a12161216cd45cf/pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f", size = 9926101, upload-time = "2026-05-11T18:53:43.515Z" }, + { url = "https://files.pythonhosted.org/packages/ae/e9/e35cf11c8a136e757b956f5f0efdcaa50aecde85ea055f1898dfc68262f3/pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb", size = 10457553, upload-time = "2026-05-11T18:53:46.394Z" }, + { url = "https://files.pythonhosted.org/packages/58/3b/1cdec6772bdbaf7b25dab360c59f03cadf05492dd724c6540af905389b07/pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a", size = 10914065, upload-time = "2026-05-11T18:53:49.134Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c2/1ef644445fcd72e3627bceec77e3560636f87ddce4ed841afe76b83b5bf9/pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360", size = 11459188, upload-time = "2026-05-11T18:53:52.527Z" }, + { url = "https://files.pythonhosted.org/packages/7e/49/4d8d4f42cbc9c4adc7a1870f269c02cbd6cd40d059622c06fb298addcbad/pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76", size = 11982966, upload-time = "2026-05-11T18:53:55.043Z" }, + { url = "https://files.pythonhosted.org/packages/38/55/792619469bab9882d8bbd5865d45a72f6478762d04a9af4bf0d08c503e95/pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5", size = 9876755, upload-time = "2026-05-11T18:53:58.067Z" }, + { url = "https://files.pythonhosted.org/packages/2a/af/33c469653b0ba03b50c3a98192d4c07f0c75c66b263ceb097fce0ee97d31/pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977", size = 9198658, upload-time = "2026-05-11T18:54:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/a2/fa/b8c257bd76b8bd060c3a9151c1fca05e9b9c5e3af5d0f549c0356f6d143d/pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04", size = 10787242, upload-time = "2026-05-11T18:54:03.564Z" }, + { url = "https://files.pythonhosted.org/packages/54/eb/f19206ffb0bf1919002969aa448b4702c6594845156a6f8050674855aac3/pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6", size = 10436369, upload-time = "2026-05-11T18:54:06.311Z" }, + { url = "https://files.pythonhosted.org/packages/fd/24/c7c39fb4fe22b71a0c2d78bf0c585c600092d85f94f086d2b3b2f6ca27e2/pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c", size = 10358306, upload-time = "2026-05-11T18:54:09.085Z" }, + { url = "https://files.pythonhosted.org/packages/16/ec/dd2a9eb7fa1204df88c0864164e35b228ac581062ac612ba0a67fd812e4c/pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028", size = 10758394, upload-time = "2026-05-11T18:54:11.956Z" }, + { url = "https://files.pythonhosted.org/packages/95/6e/00c61ea8e85b4f6d8d35e11852a1a4998fc7fafc91c6a602d1cc9c972d64/pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d", size = 11375717, upload-time = "2026-05-11T18:54:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/31/89/8fc1c268969fac43688d65fd92e67df24bd128d53cb4d2eee534cd307399/pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a", size = 11828897, upload-time = "2026-05-11T18:54:17.146Z" }, + { url = "https://files.pythonhosted.org/packages/56/3b/e7d20dea247a3e6dc0bd8a6953854afbedc03951def4e7371e05e7263e25/pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1", size = 10900855, upload-time = "2026-05-11T18:54:19.72Z" }, + { url = "https://files.pythonhosted.org/packages/0f/54/68a0978d1ef8502b8492099beaa6e7a0c1b32e3b5d4f677f5810cb08711c/pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1", size = 9466464, upload-time = "2026-05-11T18:54:22.754Z" }, ] [[package]] name = "platformdirs" -version = "4.9.6" +version = "4.10.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9f/4a/0883b8e3802965322523f0b200ecf33d31f10991d0401162f4b23c698b42/platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a", size = 29400, upload-time = "2026-04-09T00:04:10.812Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/47/e4501f49c178ae1d9f4a75073fda4204f52647993f075a9db4d14930e0c5/platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7", size = 31224, upload-time = "2026-05-28T03:32:53.587Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348, upload-time = "2026-04-09T00:04:09.463Z" }, + { url = "https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" }, ] [[package]] @@ -852,7 +860,7 @@ wheels = [ [[package]] name = "posthog" -version = "7.15.3" +version = "7.16.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backoff" }, @@ -860,9 +868,9 @@ dependencies = [ { name = "requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c5/ad/0eedae8cc9d2878d5b52c8607bd21f76101cfe4d875e5ff77fec9da3a83c/posthog-7.15.3.tar.gz", hash = "sha256:809dcaf08ca2d8bc0ea8228c28419181b74a79dfd1c0687a3d459a7bbe2e2953", size = 217645, upload-time = "2026-05-21T15:35:04.914Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b4/4f/a954175c862a3565d02c3f627874d85f18313472a0c4b08f45d84aaf3315/posthog-7.16.1.tar.gz", hash = "sha256:3619d3c619ad01f36c6d465e084950882417c63021eb3cfacacb23f900ec52d4", size = 226343, upload-time = "2026-05-27T18:46:20.129Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ad/b4/8dc673bed0f296c1acbb1107aef1c56db576731e894fe765206be5a91774/posthog-7.15.3-py3-none-any.whl", hash = "sha256:fd59fe4f5be637e4a2706b1457301d8308853ff23659036ecfcf6ac0a2d45eee", size = 254591, upload-time = "2026-05-21T15:35:02.846Z" }, + { url = "https://files.pythonhosted.org/packages/3e/28/0f840699a1d0db3c1e5483c6208f0804a51f21ccfa34e6aa356161606adc/posthog-7.16.1-py3-none-any.whl", hash = "sha256:fd5aa4510033f3b039fda2fbfce45f493d140d4782f681e69639793dda317d67", size = 264231, upload-time = "2026-05-27T18:46:17.933Z" }, ] [[package]] @@ -883,14 +891,14 @@ wheels = [ [[package]] name = "psycopg" -version = "3.3.3" +version = "3.3.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "tzdata", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d3/b6/379d0a960f8f435ec78720462fd94c4863e7a31237cf81bf76d0af5883bf/psycopg-3.3.3.tar.gz", hash = "sha256:5e9a47458b3c1583326513b2556a2a9473a1001a56c9efe9e587245b43148dd9", size = 165624, upload-time = "2026-02-18T16:52:16.546Z" } +sdist = { url = "https://files.pythonhosted.org/packages/db/2f/cb91e5502ec9de1de6f1b76cfbf69531932725361168bb06963620c77e2e/psycopg-3.3.4.tar.gz", hash = "sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc", size = 165799, upload-time = "2026-05-01T23:31:55.179Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/5b/181e2e3becb7672b502f0ed7f16ed7352aca7c109cfb94cf3878a9186db9/psycopg-3.3.3-py3-none-any.whl", hash = "sha256:f96525a72bcfade6584ab17e89de415ff360748c766f0106959144dcbb38c698", size = 212768, upload-time = "2026-02-18T16:46:27.365Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/7b3dee031daae7743609ce3c746565d4a3ed7c2c186479eb48e34e838c64/psycopg-3.3.4-py3-none-any.whl", hash = "sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a", size = 213001, upload-time = "2026-05-01T23:20:50.816Z" }, ] [package.optional-dependencies] @@ -900,36 +908,36 @@ binary = [ [[package]] name = "psycopg-binary" -version = "3.3.3" +version = "3.3.4" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/03/0a/cac9fdf1df16a269ba0e5f0f06cac61f826c94cadb39df028cdfe19d3a33/psycopg_binary-3.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:05f32239aec25c5fb15f7948cffdc2dc0dac098e48b80a140e4ba32b572a2e7d", size = 4590414, upload-time = "2026-02-18T16:50:01.441Z" }, - { url = "https://files.pythonhosted.org/packages/9c/c0/d8f8508fbf440edbc0099b1abff33003cd80c9e66eb3a1e78834e3fb4fb9/psycopg_binary-3.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7c84f9d214f2d1de2fafebc17fa68ac3f6561a59e291553dfc45ad299f4898c1", size = 4669021, upload-time = "2026-02-18T16:50:08.803Z" }, - { url = "https://files.pythonhosted.org/packages/04/05/097016b77e343b4568feddf12c72171fc513acef9a4214d21b9478569068/psycopg_binary-3.3.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e77957d2ba17cada11be09a5066d93026cdb61ada7c8893101d7fe1c6e1f3925", size = 5467453, upload-time = "2026-02-18T16:50:14.985Z" }, - { url = "https://files.pythonhosted.org/packages/91/23/73244e5feb55b5ca109cede6e97f32ef45189f0fdac4c80d75c99862729d/psycopg_binary-3.3.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:42961609ac07c232a427da7c87a468d3c82fee6762c220f38e37cfdacb2b178d", size = 5151135, upload-time = "2026-02-18T16:50:24.82Z" }, - { url = "https://files.pythonhosted.org/packages/11/49/5309473b9803b207682095201d8708bbc7842ddf3f192488a69204e36455/psycopg_binary-3.3.3-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ae07a3114313dd91fce686cab2f4c44af094398519af0e0f854bc707e1aeedf1", size = 6737315, upload-time = "2026-02-18T16:50:35.106Z" }, - { url = "https://files.pythonhosted.org/packages/d4/5d/03abe74ef34d460b33c4d9662bf6ec1dd38888324323c1a1752133c10377/psycopg_binary-3.3.3-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d257c58d7b36a621dcce1d01476ad8b60f12d80eb1406aee4cf796f88b2ae482", size = 4979783, upload-time = "2026-02-18T16:50:42.067Z" }, - { url = "https://files.pythonhosted.org/packages/f0/6c/3fbf8e604e15f2f3752900434046c00c90bb8764305a1b81112bff30ba24/psycopg_binary-3.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:07c7211f9327d522c9c47560cae00a4ecf6687f4e02d779d035dd3177b41cb12", size = 4509023, upload-time = "2026-02-18T16:50:50.116Z" }, - { url = "https://files.pythonhosted.org/packages/9c/6b/1a06b43b7c7af756c80b67eac8bfaa51d77e68635a8a8d246e4f0bb7604a/psycopg_binary-3.3.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:8e7e9eca9b363dbedeceeadd8be97149d2499081f3c52d141d7cd1f395a91f83", size = 4185874, upload-time = "2026-02-18T16:50:55.97Z" }, - { url = "https://files.pythonhosted.org/packages/2b/d3/bf49e3dcaadba510170c8d111e5e69e5ae3f981c1554c5bb71c75ce354bb/psycopg_binary-3.3.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:cb85b1d5702877c16f28d7b92ba030c1f49ebcc9b87d03d8c10bf45a2f1c7508", size = 3925668, upload-time = "2026-02-18T16:51:03.299Z" }, - { url = "https://files.pythonhosted.org/packages/f8/92/0aac830ed6a944fe334404e1687a074e4215630725753f0e3e9a9a595b62/psycopg_binary-3.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4d4606c84d04b80f9138d72f1e28c6c02dc5ae0c7b8f3f8aaf89c681ce1cd1b1", size = 4234973, upload-time = "2026-02-18T16:51:09.097Z" }, - { url = "https://files.pythonhosted.org/packages/2e/96/102244653ee5a143ece5afe33f00f52fe64e389dfce8dbc87580c6d70d3d/psycopg_binary-3.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:74eae563166ebf74e8d950ff359be037b85723d99ca83f57d9b244a871d6c13b", size = 3551342, upload-time = "2026-02-18T16:51:13.892Z" }, - { url = "https://files.pythonhosted.org/packages/a2/71/7a57e5b12275fe7e7d84d54113f0226080423a869118419c9106c083a21c/psycopg_binary-3.3.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:497852c5eaf1f0c2d88ab74a64a8097c099deac0c71de1cbcf18659a8a04a4b2", size = 4607368, upload-time = "2026-02-18T16:51:19.295Z" }, - { url = "https://files.pythonhosted.org/packages/c7/04/cb834f120f2b2c10d4003515ef9ca9d688115b9431735e3936ae48549af8/psycopg_binary-3.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:258d1ea53464d29768bf25930f43291949f4c7becc706f6e220c515a63a24edd", size = 4687047, upload-time = "2026-02-18T16:51:23.84Z" }, - { url = "https://files.pythonhosted.org/packages/40/e9/47a69692d3da9704468041aa5ed3ad6fc7f6bb1a5ae788d261a26bbca6c7/psycopg_binary-3.3.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:111c59897a452196116db12e7f608da472fbff000693a21040e35fc978b23430", size = 5487096, upload-time = "2026-02-18T16:51:29.645Z" }, - { url = "https://files.pythonhosted.org/packages/0b/b6/0e0dd6a2f802864a4ae3dbadf4ec620f05e3904c7842b326aafc43e5f464/psycopg_binary-3.3.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:17bb6600e2455993946385249a3c3d0af52cd70c1c1cdbf712e9d696d0b0bf1b", size = 5168720, upload-time = "2026-02-18T16:51:36.499Z" }, - { url = "https://files.pythonhosted.org/packages/6f/0d/977af38ac19a6b55d22dff508bd743fd7c1901e1b73657e7937c7cccb0a3/psycopg_binary-3.3.3-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:642050398583d61c9856210568eb09a8e4f2fe8224bf3be21b67a370e677eead", size = 6762076, upload-time = "2026-02-18T16:51:43.167Z" }, - { url = "https://files.pythonhosted.org/packages/34/40/912a39d48322cf86895c0eaf2d5b95cb899402443faefd4b09abbba6b6e1/psycopg_binary-3.3.3-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:533efe6dc3a7cba5e2a84e38970786bb966306863e45f3db152007e9f48638a6", size = 4997623, upload-time = "2026-02-18T16:51:47.707Z" }, - { url = "https://files.pythonhosted.org/packages/98/0c/c14d0e259c65dc7be854d926993f151077887391d5a081118907a9d89603/psycopg_binary-3.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:5958dbf28b77ce2033482f6cb9ef04d43f5d8f4b7636e6963d5626f000efb23e", size = 4532096, upload-time = "2026-02-18T16:51:51.421Z" }, - { url = "https://files.pythonhosted.org/packages/39/21/8b7c50a194cfca6ea0fd4d1f276158307785775426e90700ab2eba5cd623/psycopg_binary-3.3.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:a6af77b6626ce92b5817bf294b4d45ec1a6161dba80fc2d82cdffdd6814fd023", size = 4208884, upload-time = "2026-02-18T16:51:57.336Z" }, - { url = "https://files.pythonhosted.org/packages/c7/2c/a4981bf42cf30ebba0424971d7ce70a222ae9b82594c42fc3f2105d7b525/psycopg_binary-3.3.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:47f06fcbe8542b4d96d7392c476a74ada521c5aebdb41c3c0155f6595fc14c8d", size = 3944542, upload-time = "2026-02-18T16:52:04.266Z" }, - { url = "https://files.pythonhosted.org/packages/60/e9/b7c29b56aa0b85a4e0c4d89db691c1ceef08f46a356369144430c155a2f5/psycopg_binary-3.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e7800e6c6b5dc4b0ca7cc7370f770f53ac83886b76afda0848065a674231e856", size = 4254339, upload-time = "2026-02-18T16:52:10.444Z" }, - { url = "https://files.pythonhosted.org/packages/98/5a/291d89f44d3820fffb7a04ebc8f3ef5dda4f542f44a5daea0c55a84abf45/psycopg_binary-3.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:165f22ab5a9513a3d7425ffb7fcc7955ed8ccaeef6d37e369d6cc1dff1582383", size = 3652796, upload-time = "2026-02-18T16:52:14.02Z" }, + { url = "https://files.pythonhosted.org/packages/09/43/13e9c406fbbf354580476e248a16b64802a376873ebe6339e30bb655572d/psycopg_binary-3.3.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7", size = 4590377, upload-time = "2026-05-01T23:29:18.782Z" }, + { url = "https://files.pythonhosted.org/packages/22/be/2923cd7c3683e7afdecf4f10796a18de02f5c5ddc0969aa2ad0a8cdd3bbd/psycopg_binary-3.3.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:75a9067e236f9b9ae3535b66fe99bddb33d39c0de10112e49b9ab11eee53dc31", size = 4669023, upload-time = "2026-05-01T23:29:25.884Z" }, + { url = "https://files.pythonhosted.org/packages/96/a0/2c913d6fe13d6a8bd13597d36739bf47af063ad9399e402cfecab16f3c1e/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b56b603ebcea8aa10b46228b8410ba7f13e7c2ee54389d4d9be0927fd8ce2a70", size = 5467423, upload-time = "2026-05-01T23:29:33.416Z" }, + { url = "https://files.pythonhosted.org/packages/e7/38/205d10bc1ad0df4a21c5c51659126bd3ea0ef98fcad1e852f78c249bb9c3/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3", size = 5151137, upload-time = "2026-05-01T23:29:42.013Z" }, + { url = "https://files.pythonhosted.org/packages/36/fc/f0381ddcd45eff3bb70dbca6823a996048d7f507b2ec3fc92c6fabc0fe87/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26df2717e59c0473e4465a97dfb1b7afebaa479277870fd5784d1436470db47c", size = 6736671, upload-time = "2026-05-01T23:29:51.626Z" }, + { url = "https://files.pythonhosted.org/packages/95/40/fa545ae152c24327651e5624e4902121e808270be36c10b12e9939be09bc/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dc1f79fd16bb1f3f4421417a514607539f17804d95c7ed617265369d1981cae", size = 4979601, upload-time = "2026-05-01T23:29:56.961Z" }, + { url = "https://files.pythonhosted.org/packages/86/e4/2f8a47ee97f90cd2b933d0463081d35631ff419de2b8c984a5f369857de0/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:136f199a407b5348b9b857c504aff60c77622a28482e7195839ce1b51238c4cc", size = 4510513, upload-time = "2026-05-01T23:30:07.243Z" }, + { url = "https://files.pythonhosted.org/packages/0e/0e/94e842ff4a7f98ed162580ca2e8b8864b28c1e0350f2443f8ee47f821167/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b6f5a29e9c775b9f12a1a717aa7a2c80f9e1db6f27ba44a5b59c80ac61d2ffcf", size = 4187243, upload-time = "2026-05-01T23:30:15.352Z" }, + { url = "https://files.pythonhosted.org/packages/d0/83/fc6c174b672e29b7de996ea77b6cbddf46c891751c3355f6974292baa6b4/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ee17a2cf4943cde261adfad1bbc5bf38d6b3776d7afff74c7cabcbeaeb08c260", size = 3927347, upload-time = "2026-05-01T23:30:21.186Z" }, + { url = "https://files.pythonhosted.org/packages/e9/65/768364d4a97a15b1a7f47ba52688c1686f22941d8332a8398cefc468e25f/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c4ab71be17bdca30cb34c34c4e1496e2f5d6f20c199c12bad226070b22ef9bf", size = 4236393, upload-time = "2026-05-01T23:30:26.211Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3b/218efbc9e645becd80cdf651acda05f85cfe546b7a9c0458c7cbc8fe1f74/psycopg_binary-3.3.4-cp313-cp313-win_amd64.whl", hash = "sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38", size = 3564592, upload-time = "2026-05-01T23:30:31.764Z" }, + { url = "https://files.pythonhosted.org/packages/48/a6/828c9185701dab71b234c2a76c38a08b098ebfec5020716b4e93807492b5/psycopg_binary-3.3.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:28b7398fdd19db3232c884fb24550bdfe951221f510e195e233299e4c9b78f97", size = 4607292, upload-time = "2026-05-01T23:30:38.962Z" }, + { url = "https://files.pythonhosted.org/packages/92/58/5b40dbc9d839045c9dae956960e4fb6d20bcabe6c59a2aa34fc3a371913f/psycopg_binary-3.3.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1fbaa292a3c8bb61b45df1ad3da1908ccee7cb889db9425e3557d9e34e2a4829", size = 4687023, upload-time = "2026-05-01T23:30:47.227Z" }, + { url = "https://files.pythonhosted.org/packages/85/a9/793f0ac107a9003b48441d0d1f9f616d96e0f37458dd8dc12528ceff55fb/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94596f9e7633ee3f6440711d43bb70aa31cc0a46a900ab8b4201a366ace5c9e7", size = 5486985, upload-time = "2026-05-01T23:30:55.517Z" }, + { url = "https://files.pythonhosted.org/packages/8f/26/42e8533497e2592334f68ec529cf5f840f7fa4e99575a4bb61aa184dbfbf/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8c0056529e68dbe9184cd4019a1f3d8f3a4ead2f6fc7a5afcf27d3314edd1277", size = 5168745, upload-time = "2026-05-01T23:31:01.904Z" }, + { url = "https://files.pythonhosted.org/packages/15/af/b7151776cc08d5935d45c833ec818a9beb417cf7c08239af1aafbdae78ee/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c09aad7051326e7603c14e50636db9c01f78272dc54b3accff03d46370461e6", size = 6761486, upload-time = "2026-05-01T23:31:14.511Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ed/c92533b9124712d592cbf1cd6c76da933a2e0acea81dfe1fbe7e735f0cff/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:514404ed543efd620c85602b747df2a23cf1241b4067199e1a66f2d2757aaa41", size = 4997427, upload-time = "2026-05-01T23:31:20.901Z" }, + { url = "https://files.pythonhosted.org/packages/a2/23/ccadfd0de416aa188356daa199453af24087b042e296088706d190ae0295/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:46893c26858be12cc49ca4226ed6a60b4bfccadd946b3bebb783a60b38788228", size = 4533549, upload-time = "2026-05-01T23:31:26.204Z" }, + { url = "https://files.pythonhosted.org/packages/fd/a0/c8f43cee36386f7bc891ab41a9d31ea07cf9826038e732da79f26b1e5f34/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:df1d567fc430f6df15c9fcf67d87685fc49bdb325adc0db5af1adfb2f44eb5c9", size = 4210256, upload-time = "2026-05-01T23:31:33.884Z" }, + { url = "https://files.pythonhosted.org/packages/4e/2c/c1547871be3790676e8868b38655496422f94f0978dfb66b74bdba2f1676/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:6b9016b1714da4dd5ecaaa75b82098aa5a0b87854ce9b092e21c27c4ae23e014", size = 3946204, upload-time = "2026-05-01T23:31:39.626Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b1/f6670f00fa7ea601584623f6c11602ab92117d83eaff885e0210f6de7418/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:47c656a8a7ba6eb0cff1801a4caaa9c8bdc12d03080e273aff1c8ac39971a77e", size = 4255811, upload-time = "2026-05-01T23:31:44.986Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" }, ] [[package]] name = "pydantic" -version = "2.13.3" +version = "2.13.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types" }, @@ -937,65 +945,65 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d9/e4/40d09941a2cebcb20609b86a559817d5b9291c49dd6f8c87e5feffbe703a/pydantic-2.13.3.tar.gz", hash = "sha256:af09e9d1d09f4e7fe37145c1f577e1d61ceb9a41924bf0094a36506285d0a84d", size = 844068, upload-time = "2026-04-20T14:46:43.632Z" } +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/0a/fd7d723f8f8153418fb40cf9c940e82004fce7e987026b08a68a36dd3fe7/pydantic-2.13.3-py3-none-any.whl", hash = "sha256:6db14ac8dfc9a1e57f87ea2c0de670c251240f43cb0c30a5130e9720dc612927", size = 471981, upload-time = "2026-04-20T14:46:41.402Z" }, + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, ] [[package]] name = "pydantic-core" -version = "2.46.3" +version = "2.46.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2a/ef/f7abb56c49382a246fd2ce9c799691e3c3e7175ec74b14d99e798bcddb1a/pydantic_core-2.46.3.tar.gz", hash = "sha256:41c178f65b8c29807239d47e6050262eb6bf84eb695e41101e62e38df4a5bc2c", size = 471412, upload-time = "2026-04-20T14:40:56.672Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/3c/9b5e8eb9821936d065439c3b0fb1490ffa64163bfe7e1595985a47896073/pydantic_core-2.46.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:12bc98de041458b80c86c56b24df1d23832f3e166cbaff011f25d187f5c62c37", size = 2102109, upload-time = "2026-04-20T14:41:24.219Z" }, - { url = "https://files.pythonhosted.org/packages/91/97/1c41d1f5a19f241d8069f1e249853bcce378cdb76eec8ab636d7bc426280/pydantic_core-2.46.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:85348b8f89d2c3508b65b16c3c33a4da22b8215138d8b996912bb1532868885f", size = 1951820, upload-time = "2026-04-20T14:42:14.236Z" }, - { url = "https://files.pythonhosted.org/packages/30/b4/d03a7ae14571bc2b6b3c7b122441154720619afe9a336fa3a95434df5e2f/pydantic_core-2.46.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1105677a6df914b1fb71a81b96c8cce7726857e1717d86001f29be06a25ee6f8", size = 1977785, upload-time = "2026-04-20T14:42:31.648Z" }, - { url = "https://files.pythonhosted.org/packages/ae/0c/4086f808834b59e3c8f1aa26df8f4b6d998cdcf354a143d18ef41529d1fe/pydantic_core-2.46.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:87082cd65669a33adeba5470769e9704c7cf026cc30afb9cc77fd865578ebaad", size = 2062761, upload-time = "2026-04-20T14:40:37.093Z" }, - { url = "https://files.pythonhosted.org/packages/fa/71/a649be5a5064c2df0db06e0a512c2281134ed2fcc981f52a657936a7527c/pydantic_core-2.46.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60e5f66e12c4f5212d08522963380eaaeac5ebd795826cfd19b2dfb0c7a52b9c", size = 2232989, upload-time = "2026-04-20T14:42:59.254Z" }, - { url = "https://files.pythonhosted.org/packages/a2/84/7756e75763e810b3a710f4724441d1ecc5883b94aacb07ca71c5fb5cfb69/pydantic_core-2.46.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b6cdf19bf84128d5e7c37e8a73a0c5c10d51103a650ac585d42dd6ae233f2b7f", size = 2303975, upload-time = "2026-04-20T14:41:32.287Z" }, - { url = "https://files.pythonhosted.org/packages/6c/35/68a762e0c1e31f35fa0dac733cbd9f5b118042853698de9509c8e5bf128b/pydantic_core-2.46.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:031bb17f4885a43773c8c763089499f242aee2ea85cf17154168775dccdecf35", size = 2095325, upload-time = "2026-04-20T14:42:47.685Z" }, - { url = "https://files.pythonhosted.org/packages/77/bf/1bf8c9a8e91836c926eae5e3e51dce009bf495a60ca56060689d3df3f340/pydantic_core-2.46.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:bcf2a8b2982a6673693eae7348ef3d8cf3979c1d63b54fca7c397a635cc68687", size = 2133368, upload-time = "2026-04-20T14:41:22.766Z" }, - { url = "https://files.pythonhosted.org/packages/e5/50/87d818d6bab915984995157ceb2380f5aac4e563dddbed6b56f0ed057aba/pydantic_core-2.46.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28e8cf2f52d72ced402a137145923a762cbb5081e48b34312f7a0c8f55928ec3", size = 2173908, upload-time = "2026-04-20T14:42:52.044Z" }, - { url = "https://files.pythonhosted.org/packages/91/88/a311fb306d0bd6185db41fa14ae888fb81d0baf648a761ae760d30819d33/pydantic_core-2.46.3-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:17eaface65d9fc5abb940003020309c1bf7a211f5f608d7870297c367e6f9022", size = 2186422, upload-time = "2026-04-20T14:43:29.55Z" }, - { url = "https://files.pythonhosted.org/packages/8f/79/28fd0d81508525ab2054fef7c77a638c8b5b0afcbbaeee493cf7c3fef7e1/pydantic_core-2.46.3-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:93fd339f23408a07e98950a89644f92c54d8729719a40b30c0a30bb9ebc55d23", size = 2332709, upload-time = "2026-04-20T14:42:16.134Z" }, - { url = "https://files.pythonhosted.org/packages/b3/21/795bf5fe5c0f379308b8ef19c50dedab2e7711dbc8d0c2acf08f1c7daa05/pydantic_core-2.46.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:23cbdb3aaa74dfe0837975dbf69b469753bbde8eacace524519ffdb6b6e89eb7", size = 2372428, upload-time = "2026-04-20T14:41:10.974Z" }, - { url = "https://files.pythonhosted.org/packages/45/b3/ed14c659cbe7605e3ef063077680a64680aec81eb1a04763a05190d49b7f/pydantic_core-2.46.3-cp313-cp313-win32.whl", hash = "sha256:610eda2e3838f401105e6326ca304f5da1e15393ae25dacae5c5c63f2c275b13", size = 1965601, upload-time = "2026-04-20T14:41:42.128Z" }, - { url = "https://files.pythonhosted.org/packages/ef/bb/adb70d9a762ddd002d723fbf1bd492244d37da41e3af7b74ad212609027e/pydantic_core-2.46.3-cp313-cp313-win_amd64.whl", hash = "sha256:68cc7866ed863db34351294187f9b729964c371ba33e31c26f478471c52e1ed0", size = 2071517, upload-time = "2026-04-20T14:43:36.096Z" }, - { url = "https://files.pythonhosted.org/packages/52/eb/66faefabebfe68bd7788339c9c9127231e680b11906368c67ce112fdb47f/pydantic_core-2.46.3-cp313-cp313-win_arm64.whl", hash = "sha256:f64b5537ac62b231572879cd08ec05600308636a5d63bcbdb15063a466977bec", size = 2035802, upload-time = "2026-04-20T14:43:38.507Z" }, - { url = "https://files.pythonhosted.org/packages/7f/db/a7bcb4940183fda36022cd18ba8dd12f2dff40740ec7b58ce7457befa416/pydantic_core-2.46.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:afa3aa644f74e290cdede48a7b0bee37d1c35e71b05105f6b340d484af536d9b", size = 2097614, upload-time = "2026-04-20T14:44:38.374Z" }, - { url = "https://files.pythonhosted.org/packages/24/35/e4066358a22e3e99519db370494c7528f5a2aa1367370e80e27e20283543/pydantic_core-2.46.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ced3310e51aa425f7f77da8bbbb5212616655bedbe82c70944320bc1dbe5e018", size = 1951896, upload-time = "2026-04-20T14:40:53.996Z" }, - { url = "https://files.pythonhosted.org/packages/87/92/37cf4049d1636996e4b888c05a501f40a43ff218983a551d57f9d5e14f0d/pydantic_core-2.46.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e29908922ce9da1a30b4da490bd1d3d82c01dcfdf864d2a74aacee674d0bfa34", size = 1979314, upload-time = "2026-04-20T14:41:49.446Z" }, - { url = "https://files.pythonhosted.org/packages/d8/36/9ff4d676dfbdfb2d591cf43f3d90ded01e15b1404fd101180ed2d62a2fd3/pydantic_core-2.46.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0c9ff69140423eea8ed2d5477df3ba037f671f5e897d206d921bc9fdc39613e7", size = 2056133, upload-time = "2026-04-20T14:42:23.574Z" }, - { url = "https://files.pythonhosted.org/packages/bc/f0/405b442a4d7ba855b06eec8b2bf9c617d43b8432d099dfdc7bf999293495/pydantic_core-2.46.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b675ab0a0d5b1c8fdb81195dc5bcefea3f3c240871cdd7ff9a2de8aa50772eb2", size = 2228726, upload-time = "2026-04-20T14:44:22.816Z" }, - { url = "https://files.pythonhosted.org/packages/e7/f8/65cd92dd5a0bd89ba277a98ecbfaf6fc36bbd3300973c7a4b826d6ab1391/pydantic_core-2.46.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0087084960f209a9a4af50ecd1fb063d9ad3658c07bb81a7a53f452dacbfb2ba", size = 2301214, upload-time = "2026-04-20T14:44:48.792Z" }, - { url = "https://files.pythonhosted.org/packages/fd/86/ef96a4c6e79e7a2d0410826a68fbc0eccc0fd44aa733be199d5fcac3bb87/pydantic_core-2.46.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed42e6cc8e1b0e2b9b96e2276bad70ae625d10d6d524aed0c93de974ae029f9f", size = 2099927, upload-time = "2026-04-20T14:41:40.196Z" }, - { url = "https://files.pythonhosted.org/packages/6d/53/269caf30e0096e0a8a8f929d1982a27b3879872cca2d917d17c2f9fdf4fe/pydantic_core-2.46.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:f1771ce258afb3e4201e67d154edbbae712a76a6081079fe247c2f53c6322c22", size = 2128789, upload-time = "2026-04-20T14:41:15.868Z" }, - { url = "https://files.pythonhosted.org/packages/00/b0/1a6d9b6a587e118482910c244a1c5acf4d192604174132efd12bf0ac486f/pydantic_core-2.46.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a7610b6a5242a6c736d8ad47fd5fff87fcfe8f833b281b1c409c3d6835d9227f", size = 2173815, upload-time = "2026-04-20T14:44:25.152Z" }, - { url = "https://files.pythonhosted.org/packages/87/56/e7e00d4041a7e62b5a40815590114db3b535bf3ca0bf4dca9f16cef25246/pydantic_core-2.46.3-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:ff5e7783bcc5476e1db448bf268f11cb257b1c276d3e89f00b5727be86dd0127", size = 2181608, upload-time = "2026-04-20T14:41:28.933Z" }, - { url = "https://files.pythonhosted.org/packages/e8/22/4bd23c3d41f7c185d60808a1de83c76cf5aeabf792f6c636a55c3b1ec7f9/pydantic_core-2.46.3-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:9d2e32edcc143bc01e95300671915d9ca052d4f745aa0a49c48d4803f8a85f2c", size = 2326968, upload-time = "2026-04-20T14:42:03.962Z" }, - { url = "https://files.pythonhosted.org/packages/24/ac/66cd45129e3915e5ade3b292cb3bc7fd537f58f8f8dbdaba6170f7cabb74/pydantic_core-2.46.3-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:6e42d83d1c6b87fa56b521479cff237e626a292f3b31b6345c15a99121b454c1", size = 2369842, upload-time = "2026-04-20T14:41:35.52Z" }, - { url = "https://files.pythonhosted.org/packages/a2/51/dd4248abb84113615473aa20d5545b7c4cd73c8644003b5259686f93996c/pydantic_core-2.46.3-cp314-cp314-win32.whl", hash = "sha256:07bc6d2a28c3adb4f7c6ae46aa4f2d2929af127f587ed44057af50bf1ce0f505", size = 1959661, upload-time = "2026-04-20T14:41:00.042Z" }, - { url = "https://files.pythonhosted.org/packages/20/eb/59980e5f1ae54a3b86372bd9f0fa373ea2d402e8cdcd3459334430f91e91/pydantic_core-2.46.3-cp314-cp314-win_amd64.whl", hash = "sha256:8940562319bc621da30714617e6a7eaa6b98c84e8c685bcdc02d7ed5e7c7c44e", size = 2071686, upload-time = "2026-04-20T14:43:16.471Z" }, - { url = "https://files.pythonhosted.org/packages/8c/db/1cf77e5247047dfee34bc01fa9bca134854f528c8eb053e144298893d370/pydantic_core-2.46.3-cp314-cp314-win_arm64.whl", hash = "sha256:5dcbbcf4d22210ced8f837c96db941bdb078f419543472aca5d9a0bb7cddc7df", size = 2026907, upload-time = "2026-04-20T14:43:31.732Z" }, - { url = "https://files.pythonhosted.org/packages/57/c0/b3df9f6a543276eadba0a48487b082ca1f201745329d97dbfa287034a230/pydantic_core-2.46.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:d0fe3dce1e836e418f912c1ad91c73357d03e556a4d286f441bf34fed2dbeecf", size = 2095047, upload-time = "2026-04-20T14:42:37.982Z" }, - { url = "https://files.pythonhosted.org/packages/66/57/886a938073b97556c168fd99e1a7305bb363cd30a6d2c76086bf0587b32a/pydantic_core-2.46.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9ce92e58abc722dac1bf835a6798a60b294e48eb0e625ec9fd994b932ac5feee", size = 1934329, upload-time = "2026-04-20T14:43:49.655Z" }, - { url = "https://files.pythonhosted.org/packages/0b/7c/b42eaa5c34b13b07ecb51da21761297a9b8eb43044c864a035999998f328/pydantic_core-2.46.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a03e6467f0f5ab796a486146d1b887b2dc5e5f9b3288898c1b1c3ad974e53e4a", size = 1974847, upload-time = "2026-04-20T14:42:10.737Z" }, - { url = "https://files.pythonhosted.org/packages/e6/9b/92b42db6543e7de4f99ae977101a2967b63122d4b6cf7773812da2d7d5b5/pydantic_core-2.46.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2798b6ba041b9d70acfb9071a2ea13c8456dd1e6a5555798e41ba7b0790e329c", size = 2041742, upload-time = "2026-04-20T14:40:44.262Z" }, - { url = "https://files.pythonhosted.org/packages/0f/19/46fbe1efabb5aa2834b43b9454e70f9a83ad9c338c1291e48bdc4fecf167/pydantic_core-2.46.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9be3e221bdc6d69abf294dcf7aff6af19c31a5cdcc8f0aa3b14be29df4bd03b1", size = 2236235, upload-time = "2026-04-20T14:41:27.307Z" }, - { url = "https://files.pythonhosted.org/packages/77/da/b3f95bc009ad60ec53120f5d16c6faa8cabdbe8a20d83849a1f2b8728148/pydantic_core-2.46.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f13936129ce841f2a5ddf6f126fea3c43cd128807b5a59588c37cf10178c2e64", size = 2282633, upload-time = "2026-04-20T14:44:33.271Z" }, - { url = "https://files.pythonhosted.org/packages/cc/6e/401336117722e28f32fb8220df676769d28ebdf08f2f4469646d404c43a3/pydantic_core-2.46.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28b5f2ef03416facccb1c6ef744c69793175fd27e44ef15669201601cf423acb", size = 2109679, upload-time = "2026-04-20T14:44:41.065Z" }, - { url = "https://files.pythonhosted.org/packages/fc/53/b289f9bc8756a32fe718c46f55afaeaf8d489ee18d1a1e7be1db73f42cc4/pydantic_core-2.46.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:830d1247d77ad23852314f069e9d7ddafeec5f684baf9d7e7065ed46a049c4e6", size = 2108342, upload-time = "2026-04-20T14:42:50.144Z" }, - { url = "https://files.pythonhosted.org/packages/10/5b/8292fc7c1f9111f1b2b7c1b0dcf1179edcd014fc3ea4517499f50b829d71/pydantic_core-2.46.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0793c90c1a3c74966e7975eaef3ed30ebdff3260a0f815a62a22adc17e4c01c", size = 2157208, upload-time = "2026-04-20T14:42:08.133Z" }, - { url = "https://files.pythonhosted.org/packages/2b/9e/f80044e9ec07580f057a89fc131f78dda7a58751ddf52bbe05eaf31db50f/pydantic_core-2.46.3-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:d2d0aead851b66f5245ec0c4fb2612ef457f8bbafefdf65a2bf9d6bac6140f47", size = 2167237, upload-time = "2026-04-20T14:42:25.412Z" }, - { url = "https://files.pythonhosted.org/packages/f8/84/6781a1b037f3b96be9227edbd1101f6d3946746056231bf4ac48cdff1a8d/pydantic_core-2.46.3-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:2f40e4246676beb31c5ce77c38a55ca4e465c6b38d11ea1bd935420568e0b1ab", size = 2312540, upload-time = "2026-04-20T14:40:40.313Z" }, - { url = "https://files.pythonhosted.org/packages/3e/db/19c0839feeb728e7df03255581f198dfdf1c2aeb1e174a8420b63c5252e5/pydantic_core-2.46.3-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:cf489cf8986c543939aeee17a09c04d6ffb43bfef8ca16fcbcc5cfdcbed24dba", size = 2369556, upload-time = "2026-04-20T14:41:09.427Z" }, - { url = "https://files.pythonhosted.org/packages/e0/15/3228774cb7cd45f5f721ddf1b2242747f4eb834d0c491f0c02d606f09fed/pydantic_core-2.46.3-cp314-cp314t-win32.whl", hash = "sha256:ffe0883b56cfc05798bf994164d2b2ff03efe2d22022a2bb080f3b626176dd56", size = 1949756, upload-time = "2026-04-20T14:41:25.717Z" }, - { url = "https://files.pythonhosted.org/packages/b8/2a/c79cf53fd91e5a87e30d481809f52f9a60dd221e39de66455cf04deaad37/pydantic_core-2.46.3-cp314-cp314t-win_amd64.whl", hash = "sha256:706d9d0ce9cf4593d07270d8e9f53b161f90c57d315aeec4fb4fd7a8b10240d8", size = 2051305, upload-time = "2026-04-20T14:43:18.627Z" }, - { url = "https://files.pythonhosted.org/packages/0b/db/d8182a7f1d9343a032265aae186eb063fe26ca4c40f256b21e8da4498e89/pydantic_core-2.46.3-cp314-cp314t-win_arm64.whl", hash = "sha256:77706aeb41df6a76568434701e0917da10692da28cb69d5fb6919ce5fdb07374", size = 2026310, upload-time = "2026-04-20T14:41:01.778Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, ] [[package]] @@ -1051,15 +1059,15 @@ wheels = [ [[package]] name = "python-discovery" -version = "1.2.2" +version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, { name = "platformdirs" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/de/ef/3bae0e537cfe91e8431efcba4434463d2c5a65f5a89edd47c6cf2f03c55f/python_discovery-1.2.2.tar.gz", hash = "sha256:876e9c57139eb757cb5878cbdd9ae5379e5d96266c99ef731119e04fffe533bb", size = 58872, upload-time = "2026-04-07T17:28:49.249Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/12/38c1a0b1e64806780c9563e3fc9f6e472251839662587cfbe9bfaf2ae10a/python_discovery-1.4.0.tar.gz", hash = "sha256:eb8bc7daad3c226c147e45bb4e970a1feb1bf4048ee178e6db59e197b8010ce3", size = 68455, upload-time = "2026-05-28T01:15:37.639Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d8/db/795879cc3ddfe338599bddea6388cc5100b088db0a4caf6e6c1af1c27e04/python_discovery-1.2.2-py3-none-any.whl", hash = "sha256:e1ae95d9af875e78f15e19aed0c6137ab1bb49c200f21f5061786490c9585c7a", size = 31894, upload-time = "2026-04-07T17:28:48.09Z" }, + { url = "https://files.pythonhosted.org/packages/c8/8d/3d316429f65029532bb1e28ff77b797d86b5ac3915bb44ca4e19aa283d43/python_discovery-1.4.0-py3-none-any.whl", hash = "sha256:26ed78d703e234879a66244c7d4114563fb13ec5cd30a2d1357e5fb4850782da", size = 33217, upload-time = "2026-05-28T01:15:36.573Z" }, ] [[package]] @@ -1109,79 +1117,79 @@ wheels = [ [[package]] name = "regex" -version = "2026.4.4" +version = "2026.5.9" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cb/0e/3a246dbf05666918bd3664d9d787f84a9108f6f43cc953a077e4a7dfdb7e/regex-2026.4.4.tar.gz", hash = "sha256:e08270659717f6973523ce3afbafa53515c4dc5dcad637dc215b6fd50f689423", size = 416000, upload-time = "2026-04-03T20:56:28.155Z" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/0e/49aee608ad09480e7fd276898c99ec6192985fa331abe4eb3a986094490b/regex-2026.5.9.tar.gz", hash = "sha256:a8234aa23ec39894bfe4a3f1b85616a7032481964a13ac6fc9f10de4f6fca270", size = 416074, upload-time = "2026-05-09T23:15:19.37Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/83/c4373bc5f31f2cf4b66f9b7c31005bd87fe66f0dce17701f7db4ee79ee29/regex-2026.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:62f5519042c101762509b1d717b45a69c0139d60414b3c604b81328c01bd1943", size = 490273, upload-time = "2026-04-03T20:54:11.202Z" }, - { url = "https://files.pythonhosted.org/packages/46/f8/fe62afbcc3cf4ad4ac9adeaafd98aa747869ae12d3e8e2ac293d0593c435/regex-2026.4.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3790ba9fb5dd76715a7afe34dbe603ba03f8820764b1dc929dd08106214ed031", size = 291954, upload-time = "2026-04-03T20:54:13.412Z" }, - { url = "https://files.pythonhosted.org/packages/5a/92/4712b9fe6a33d232eeb1c189484b80c6c4b8422b90e766e1195d6e758207/regex-2026.4.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8fae3c6e795d7678963f2170152b0d892cf6aee9ee8afc8c45e6be38d5107fe7", size = 289487, upload-time = "2026-04-03T20:54:15.824Z" }, - { url = "https://files.pythonhosted.org/packages/88/2c/f83b93f85e01168f1070f045a42d4c937b69fdb8dd7ae82d307253f7e36e/regex-2026.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:298c3ec2d53225b3bf91142eb9691025bab610e0c0c51592dde149db679b3d17", size = 796646, upload-time = "2026-04-03T20:54:18.229Z" }, - { url = "https://files.pythonhosted.org/packages/df/55/61a2e17bf0c4dc57e11caf8dd11771280d8aaa361785f9e3bc40d653f4a7/regex-2026.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e9638791082eaf5b3ac112c587518ee78e083a11c4b28012d8fe2a0f536dfb17", size = 865904, upload-time = "2026-04-03T20:54:20.019Z" }, - { url = "https://files.pythonhosted.org/packages/45/32/1ac8ed1b5a346b5993a3d256abe0a0f03b0b73c8cc88d928537368ac65b6/regex-2026.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ae3e764bd4c5ff55035dc82a8d49acceb42a5298edf6eb2fc4d328ee5dd7afae", size = 912304, upload-time = "2026-04-03T20:54:22.403Z" }, - { url = "https://files.pythonhosted.org/packages/26/47/2ee5c613ab546f0eddebf9905d23e07beb933416b1246c2d8791d01979b4/regex-2026.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffa81f81b80047ba89a3c69ae6a0f78d06f4a42ce5126b0eb2a0a10ad44e0b2e", size = 801126, upload-time = "2026-04-03T20:54:24.308Z" }, - { url = "https://files.pythonhosted.org/packages/75/cd/41dacd129ca9fd20bd7d02f83e0fad83e034ac8a084ec369c90f55ef37e2/regex-2026.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f56ebf9d70305307a707911b88469213630aba821e77de7d603f9d2f0730687d", size = 776772, upload-time = "2026-04-03T20:54:26.319Z" }, - { url = "https://files.pythonhosted.org/packages/89/6d/5af0b588174cb5f46041fa7dd64d3fd5cd2fe51f18766703d1edc387f324/regex-2026.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:773d1dfd652bbffb09336abf890bfd64785c7463716bf766d0eb3bc19c8b7f27", size = 785228, upload-time = "2026-04-03T20:54:28.387Z" }, - { url = "https://files.pythonhosted.org/packages/b7/3b/f5a72b7045bd59575fc33bf1345f156fcfd5a8484aea6ad84b12c5a82114/regex-2026.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d51d20befd5275d092cdffba57ded05f3c436317ee56466c8928ac32d960edaf", size = 860032, upload-time = "2026-04-03T20:54:30.641Z" }, - { url = "https://files.pythonhosted.org/packages/39/a4/72a317003d6fcd7a573584a85f59f525dfe8f67e355ca74eb6b53d66a5e2/regex-2026.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:0a51cdb3c1e9161154f976cb2bef9894bc063ac82f31b733087ffb8e880137d0", size = 765714, upload-time = "2026-04-03T20:54:32.789Z" }, - { url = "https://files.pythonhosted.org/packages/25/1e/5672e16f34dbbcb2560cc7e6a2fbb26dfa8b270711e730101da4423d3973/regex-2026.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ae5266a82596114e41fb5302140e9630204c1b5f325c770bec654b95dd54b0aa", size = 852078, upload-time = "2026-04-03T20:54:34.546Z" }, - { url = "https://files.pythonhosted.org/packages/f7/0d/c813f0af7c6cc7ed7b9558bac2e5120b60ad0fa48f813e4d4bd55446f214/regex-2026.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c882cd92ec68585e9c1cf36c447ec846c0d94edd706fe59e0c198e65822fd23b", size = 789181, upload-time = "2026-04-03T20:54:36.642Z" }, - { url = "https://files.pythonhosted.org/packages/ea/6d/a344608d1adbd2a95090ddd906cec09a11be0e6517e878d02a5123e0917f/regex-2026.4.4-cp313-cp313-win32.whl", hash = "sha256:05568c4fbf3cb4fa9e28e3af198c40d3237cf6041608a9022285fe567ec3ad62", size = 266690, upload-time = "2026-04-03T20:54:38.343Z" }, - { url = "https://files.pythonhosted.org/packages/31/07/54049f89b46235ca6f45cd6c88668a7050e77d4a15555e47dd40fde75263/regex-2026.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:3384df51ed52db0bea967e21458ab0a414f67cdddfd94401688274e55147bb81", size = 277733, upload-time = "2026-04-03T20:54:40.11Z" }, - { url = "https://files.pythonhosted.org/packages/0e/21/61366a8e20f4d43fb597708cac7f0e2baadb491ecc9549b4980b2be27d16/regex-2026.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:acd38177bd2c8e69a411d6521760806042e244d0ef94e2dd03ecdaa8a3c99427", size = 270565, upload-time = "2026-04-03T20:54:41.883Z" }, - { url = "https://files.pythonhosted.org/packages/f1/1e/3a2b9672433bef02f5d39aa1143ca2c08f311c1d041c464a42be9ae648dc/regex-2026.4.4-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:f94a11a9d05afcfcfa640e096319720a19cc0c9f7768e1a61fceee6a3afc6c7c", size = 494126, upload-time = "2026-04-03T20:54:43.602Z" }, - { url = "https://files.pythonhosted.org/packages/4e/4b/c132a4f4fe18ad3340d89fcb56235132b69559136036b845be3c073142ed/regex-2026.4.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:36bcb9d6d1307ab629edc553775baada2aefa5c50ccc0215fbfd2afcfff43141", size = 293882, upload-time = "2026-04-03T20:54:45.41Z" }, - { url = "https://files.pythonhosted.org/packages/f4/5f/eaa38092ce7a023656280f2341dbbd4ad5f05d780a70abba7bb4f4bea54c/regex-2026.4.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261c015b3e2ed0919157046d768774ecde57f03d8fa4ba78d29793447f70e717", size = 292334, upload-time = "2026-04-03T20:54:47.051Z" }, - { url = "https://files.pythonhosted.org/packages/5f/f6/dd38146af1392dac33db7074ab331cec23cced3759167735c42c5460a243/regex-2026.4.4-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c228cf65b4a54583763645dcd73819b3b381ca8b4bb1b349dee1c135f4112c07", size = 811691, upload-time = "2026-04-03T20:54:49.074Z" }, - { url = "https://files.pythonhosted.org/packages/7a/f0/dc54c2e69f5eeec50601054998ec3690d5344277e782bd717e49867c1d29/regex-2026.4.4-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dd2630faeb6876fb0c287f664d93ddce4d50cd46c6e88e60378c05c9047e08ca", size = 871227, upload-time = "2026-04-03T20:54:51.035Z" }, - { url = "https://files.pythonhosted.org/packages/a1/af/cb16bd5dc61621e27df919a4449bbb7e5a1034c34d307e0a706e9cc0f3e3/regex-2026.4.4-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a50ab11b7779b849472337191f3a043e27e17f71555f98d0092fa6d73364520", size = 917435, upload-time = "2026-04-03T20:54:52.994Z" }, - { url = "https://files.pythonhosted.org/packages/5c/71/8b260897f22996b666edd9402861668f45a2ca259f665ac029e6104a2d7d/regex-2026.4.4-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0734f63afe785138549fbe822a8cfeaccd1bae814c5057cc0ed5b9f2de4fc883", size = 816358, upload-time = "2026-04-03T20:54:54.884Z" }, - { url = "https://files.pythonhosted.org/packages/1c/60/775f7f72a510ef238254906c2f3d737fc80b16ca85f07d20e318d2eea894/regex-2026.4.4-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c4ee50606cb1967db7e523224e05f32089101945f859928e65657a2cbb3d278b", size = 785549, upload-time = "2026-04-03T20:54:57.01Z" }, - { url = "https://files.pythonhosted.org/packages/58/42/34d289b3627c03cf381e44da534a0021664188fa49ba41513da0b4ec6776/regex-2026.4.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6c1818f37be3ca02dcb76d63f2c7aaba4b0dc171b579796c6fbe00148dfec6b1", size = 801364, upload-time = "2026-04-03T20:54:58.981Z" }, - { url = "https://files.pythonhosted.org/packages/fc/20/f6ecf319b382a8f1ab529e898b222c3f30600fcede7834733c26279e7465/regex-2026.4.4-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:f5bfc2741d150d0be3e4a0401a5c22b06e60acb9aa4daa46d9e79a6dcd0f135b", size = 866221, upload-time = "2026-04-03T20:55:00.88Z" }, - { url = "https://files.pythonhosted.org/packages/92/6a/9f16d3609d549bd96d7a0b2aee1625d7512ba6a03efc01652149ef88e74d/regex-2026.4.4-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:504ffa8a03609a087cad81277a629b6ce884b51a24bd388a7980ad61748618ff", size = 772530, upload-time = "2026-04-03T20:55:03.213Z" }, - { url = "https://files.pythonhosted.org/packages/fa/f6/aa9768bc96a4c361ac96419fbaf2dcdc33970bb813df3ba9b09d5d7b6d96/regex-2026.4.4-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:70aadc6ff12e4b444586e57fc30771f86253f9f0045b29016b9605b4be5f7dfb", size = 856989, upload-time = "2026-04-03T20:55:05.087Z" }, - { url = "https://files.pythonhosted.org/packages/4d/b4/c671db3556be2473ae3e4bb7a297c518d281452871501221251ea4ecba57/regex-2026.4.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f4f83781191007b6ef43b03debc35435f10cad9b96e16d147efe84a1d48bdde4", size = 803241, upload-time = "2026-04-03T20:55:07.162Z" }, - { url = "https://files.pythonhosted.org/packages/2a/5c/83e3b1d89fa4f6e5a1bc97b4abd4a9a97b3c1ac7854164f694f5f0ba98a0/regex-2026.4.4-cp313-cp313t-win32.whl", hash = "sha256:e014a797de43d1847df957c0a2a8e861d1c17547ee08467d1db2c370b7568baa", size = 269921, upload-time = "2026-04-03T20:55:09.62Z" }, - { url = "https://files.pythonhosted.org/packages/28/07/077c387121f42cdb4d92b1301133c0d93b5709d096d1669ab847dda9fe2e/regex-2026.4.4-cp313-cp313t-win_amd64.whl", hash = "sha256:b15b88b0d52b179712632832c1d6e58e5774f93717849a41096880442da41ab0", size = 281240, upload-time = "2026-04-03T20:55:11.521Z" }, - { url = "https://files.pythonhosted.org/packages/9d/22/ead4a4abc7c59a4d882662aa292ca02c8b617f30b6e163bc1728879e9353/regex-2026.4.4-cp313-cp313t-win_arm64.whl", hash = "sha256:586b89cdadf7d67bf86ae3342a4dcd2b8d70a832d90c18a0ae955105caf34dbe", size = 272440, upload-time = "2026-04-03T20:55:13.365Z" }, - { url = "https://files.pythonhosted.org/packages/f0/f5/ed97c2dc47b5fbd4b73c0d7d75f9ebc8eca139f2bbef476bba35f28c0a77/regex-2026.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:2da82d643fa698e5e5210e54af90181603d5853cf469f5eedf9bfc8f59b4b8c7", size = 490343, upload-time = "2026-04-03T20:55:15.241Z" }, - { url = "https://files.pythonhosted.org/packages/80/e9/de4828a7385ec166d673a5790ad06ac48cdaa98bc0960108dd4b9cc1aef7/regex-2026.4.4-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:54a1189ad9d9357760557c91103d5e421f0a2dabe68a5cdf9103d0dcf4e00752", size = 291909, upload-time = "2026-04-03T20:55:17.558Z" }, - { url = "https://files.pythonhosted.org/packages/b4/d6/5cfbfc97f3201a4d24b596a77957e092030dcc4205894bc035cedcfce62f/regex-2026.4.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:76d67d5afb1fe402d10a6403bae668d000441e2ab115191a804287d53b772951", size = 289692, upload-time = "2026-04-03T20:55:20.561Z" }, - { url = "https://files.pythonhosted.org/packages/8e/ac/f2212d9fd56fe897e36d0110ba30ba2d247bd6410c5bd98499c7e5a1e1f2/regex-2026.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e7cd3e4ee8d80447a83bbc9ab0c8459781fa77087f856c3e740d7763be0df27f", size = 796979, upload-time = "2026-04-03T20:55:22.56Z" }, - { url = "https://files.pythonhosted.org/packages/c9/e3/a016c12675fbac988a60c7e1c16e67823ff0bc016beb27bd7a001dbdabc6/regex-2026.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e19e18c568d2866d8b6a6dfad823db86193503f90823a8f66689315ba28fbe8", size = 866744, upload-time = "2026-04-03T20:55:24.646Z" }, - { url = "https://files.pythonhosted.org/packages/af/a4/0b90ca4cf17adc3cb43de80ec71018c37c88ad64987e8d0d481a95ca60b5/regex-2026.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7698a6f38730fd1385d390d1ed07bb13dce39aa616aca6a6d89bea178464b9a4", size = 911613, upload-time = "2026-04-03T20:55:27.033Z" }, - { url = "https://files.pythonhosted.org/packages/8e/3b/2b3dac0b82d41ab43aa87c6ecde63d71189d03fe8854b8ca455a315edac3/regex-2026.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:173a66f3651cdb761018078e2d9487f4cf971232c990035ec0eb1cdc6bf929a9", size = 800551, upload-time = "2026-04-03T20:55:29.532Z" }, - { url = "https://files.pythonhosted.org/packages/25/fe/5365eb7aa0e753c4b5957815c321519ecab033c279c60e1b1ae2367fa810/regex-2026.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa7922bbb2cc84fa062d37723f199d4c0cd200245ce269c05db82d904db66b83", size = 776911, upload-time = "2026-04-03T20:55:31.526Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b3/7fb0072156bba065e3b778a7bc7b0a6328212be5dd6a86fd207e0c4f2dab/regex-2026.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:59f67cd0a0acaf0e564c20bbd7f767286f23e91e2572c5703bf3e56ea7557edb", size = 785751, upload-time = "2026-04-03T20:55:33.797Z" }, - { url = "https://files.pythonhosted.org/packages/02/1a/9f83677eb699273e56e858f7bd95acdbee376d42f59e8bfca2fd80d79df3/regex-2026.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:475e50f3f73f73614f7cba5524d6de49dee269df00272a1b85e3d19f6d498465", size = 860484, upload-time = "2026-04-03T20:55:35.745Z" }, - { url = "https://files.pythonhosted.org/packages/3b/7a/93937507b61cfcff8b4c5857f1b452852b09f741daa9acae15c971d8554e/regex-2026.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a1c0c7d67b64d85ac2e1879923bad2f08a08f3004055f2f406ef73c850114bd4", size = 765939, upload-time = "2026-04-03T20:55:37.972Z" }, - { url = "https://files.pythonhosted.org/packages/86/ea/81a7f968a351c6552b1670ead861e2a385be730ee28402233020c67f9e0f/regex-2026.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:1371c2ccbb744d66ee63631cc9ca12aa233d5749972626b68fe1a649dd98e566", size = 851417, upload-time = "2026-04-03T20:55:39.92Z" }, - { url = "https://files.pythonhosted.org/packages/4c/7e/323c18ce4b5b8f44517a36342961a0306e931e499febbd876bb149d900f0/regex-2026.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:59968142787042db793348a3f5b918cf24ced1f23247328530e063f89c128a95", size = 789056, upload-time = "2026-04-03T20:55:42.303Z" }, - { url = "https://files.pythonhosted.org/packages/c0/af/e7510f9b11b1913b0cd44eddb784b2d650b2af6515bfce4cffcc5bfd1d38/regex-2026.4.4-cp314-cp314-win32.whl", hash = "sha256:59efe72d37fd5a91e373e5146f187f921f365f4abc1249a5ab446a60f30dd5f8", size = 272130, upload-time = "2026-04-03T20:55:44.995Z" }, - { url = "https://files.pythonhosted.org/packages/9a/51/57dae534c915e2d3a21490e88836fa2ae79dde3b66255ecc0c0a155d2c10/regex-2026.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:e0aab3ff447845049d676827d2ff714aab4f73f340e155b7de7458cf53baa5a4", size = 280992, upload-time = "2026-04-03T20:55:47.316Z" }, - { url = "https://files.pythonhosted.org/packages/0a/5e/abaf9f4c3792e34edb1434f06717fae2b07888d85cb5cec29f9204931bf8/regex-2026.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:a7a5bb6aa0cf62208bb4fa079b0c756734f8ad0e333b425732e8609bd51ee22f", size = 273563, upload-time = "2026-04-03T20:55:49.273Z" }, - { url = "https://files.pythonhosted.org/packages/ff/06/35da85f9f217b9538b99cbb170738993bcc3b23784322decb77619f11502/regex-2026.4.4-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:97850d0638391bdc7d35dc1c1039974dcb921eaafa8cc935ae4d7f272b1d60b3", size = 494191, upload-time = "2026-04-03T20:55:51.258Z" }, - { url = "https://files.pythonhosted.org/packages/54/5b/1bc35f479eef8285c4baf88d8c002023efdeebb7b44a8735b36195486ae7/regex-2026.4.4-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:ee7337f88f2a580679f7bbfe69dc86c043954f9f9c541012f49abc554a962f2e", size = 293877, upload-time = "2026-04-03T20:55:53.214Z" }, - { url = "https://files.pythonhosted.org/packages/39/5b/f53b9ad17480b3ddd14c90da04bfb55ac6894b129e5dea87bcaf7d00e336/regex-2026.4.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7429f4e6192c11d659900c0648ba8776243bf396ab95558b8c51a345afeddde6", size = 292410, upload-time = "2026-04-03T20:55:55.736Z" }, - { url = "https://files.pythonhosted.org/packages/bb/56/52377f59f60a7c51aa4161eecf0b6032c20b461805aca051250da435ffc9/regex-2026.4.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc4f10fbd5dd13dcf4265b4cc07d69ca70280742870c97ae10093e3d66000359", size = 811831, upload-time = "2026-04-03T20:55:57.802Z" }, - { url = "https://files.pythonhosted.org/packages/dd/63/8026310bf066f702a9c361f83a8c9658f3fe4edb349f9c1e5d5273b7c40c/regex-2026.4.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a152560af4f9742b96f3827090f866eeec5becd4765c8e0d3473d9d280e76a5a", size = 871199, upload-time = "2026-04-03T20:56:00.333Z" }, - { url = "https://files.pythonhosted.org/packages/20/9f/a514bbb00a466dbb506d43f187a04047f7be1505f10a9a15615ead5080ee/regex-2026.4.4-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54170b3e95339f415d54651f97df3bff7434a663912f9358237941bbf9143f55", size = 917649, upload-time = "2026-04-03T20:56:02.445Z" }, - { url = "https://files.pythonhosted.org/packages/cb/6b/8399f68dd41a2030218839b9b18360d79b86d22b9fab5ef477c7f23ca67c/regex-2026.4.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:07f190d65f5a72dcb9cf7106bfc3d21e7a49dd2879eda2207b683f32165e4d99", size = 816388, upload-time = "2026-04-03T20:56:04.595Z" }, - { url = "https://files.pythonhosted.org/packages/1e/9c/103963f47c24339a483b05edd568594c2be486188f688c0170fd504b2948/regex-2026.4.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9a2741ce5a29d3c84b0b94261ba630ab459a1b847a0d6beca7d62d188175c790", size = 785746, upload-time = "2026-04-03T20:56:07.13Z" }, - { url = "https://files.pythonhosted.org/packages/fa/ee/7f6054c0dec0cee3463c304405e4ff42e27cff05bf36fcb34be549ab17bd/regex-2026.4.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b26c30df3a28fd9793113dac7385a4deb7294a06c0f760dd2b008bd49a9139bc", size = 801483, upload-time = "2026-04-03T20:56:09.365Z" }, - { url = "https://files.pythonhosted.org/packages/30/c2/51d3d941cf6070dc00c3338ecf138615fc3cce0421c3df6abe97a08af61a/regex-2026.4.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:421439d1bee44b19f4583ccf42670ca464ffb90e9fdc38d37f39d1ddd1e44f1f", size = 866331, upload-time = "2026-04-03T20:56:12.039Z" }, - { url = "https://files.pythonhosted.org/packages/16/e8/76d50dcc122ac33927d939f350eebcfe3dbcbda96913e03433fc36de5e63/regex-2026.4.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b40379b53ecbc747fd9bdf4a0ea14eb8188ca1bd0f54f78893a39024b28f4863", size = 772673, upload-time = "2026-04-03T20:56:14.558Z" }, - { url = "https://files.pythonhosted.org/packages/a5/6e/5f6bf75e20ea6873d05ba4ec78378c375cbe08cdec571c83fbb01606e563/regex-2026.4.4-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:08c55c13d2eef54f73eeadc33146fb0baaa49e7335eb1aff6ae1324bf0ddbe4a", size = 857146, upload-time = "2026-04-03T20:56:16.663Z" }, - { url = "https://files.pythonhosted.org/packages/0b/33/3c76d9962949e487ebba353a18e89399f292287204ac8f2f4cfc3a51c233/regex-2026.4.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9776b85f510062f5a75ef112afe5f494ef1635607bf1cc220c1391e9ac2f5e81", size = 803463, upload-time = "2026-04-03T20:56:18.923Z" }, - { url = "https://files.pythonhosted.org/packages/19/eb/ef32dcd2cb69b69bc0c3e55205bce94a7def48d495358946bc42186dcccc/regex-2026.4.4-cp314-cp314t-win32.whl", hash = "sha256:385edaebde5db5be103577afc8699fea73a0e36a734ba24870be7ffa61119d74", size = 275709, upload-time = "2026-04-03T20:56:20.996Z" }, - { url = "https://files.pythonhosted.org/packages/a0/86/c291bf740945acbf35ed7dbebf8e2eea2f3f78041f6bd7cdab80cb274dc0/regex-2026.4.4-cp314-cp314t-win_amd64.whl", hash = "sha256:5d354b18839328927832e2fa5f7c95b7a3ccc39e7a681529e1685898e6436d45", size = 285622, upload-time = "2026-04-03T20:56:23.641Z" }, - { url = "https://files.pythonhosted.org/packages/d5/e7/ec846d560ae6a597115153c02ca6138a7877a1748b2072d9521c10a93e58/regex-2026.4.4-cp314-cp314t-win_arm64.whl", hash = "sha256:af0384cb01a33600c49505c27c6c57ab0b27bf84a74e28524c92ca897ebdac9d", size = 275773, upload-time = "2026-04-03T20:56:26.07Z" }, + { url = "https://files.pythonhosted.org/packages/aa/da/797e91ecec6f84135da778ddce78c20e0af5d2a15c26f87a81bc3eadb6db/regex-2026.5.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d626b84406444b165fc0ba981604edea39f0588ff1f92baa23fe50799ea9afdb", size = 490303, upload-time = "2026-05-09T23:13:04.382Z" }, + { url = "https://files.pythonhosted.org/packages/44/da/bf30abaaa737b58f4a4b8c4a03659e02fd92092c822e0197ed9e0daab917/regex-2026.5.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d7bdc0ab8f3dd7e1b4f9ab88634e13374669db86bb3c72e8292f07ae313f539f", size = 292019, upload-time = "2026-05-09T23:13:06.022Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e7/d0eaf5713828417b9e5648cf81fa9bacd4961f6ab98c380c2034f8716e35/regex-2026.5.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a8820737949116ffff55fe18f9fc644530063ba6ebfcb8314239416e78f1347c", size = 289468, upload-time = "2026-05-09T23:13:08.214Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9b/b3fdd62b003baa1a9b593cd8c8699c9651c2e80cc21a5c715707983c42d7/regex-2026.5.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa0fbdbac82cb3e4450d0ccde7d7a35607f4cb2dd9fba4b8b69bfaf8c9fa6aed", size = 796749, upload-time = "2026-05-09T23:13:10.573Z" }, + { url = "https://files.pythonhosted.org/packages/d4/30/66ab84588765f5b4b271a9ca09ef7ce2b87caa95176ec3d2ad65d7bc4902/regex-2026.5.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:57e8915c7986aa33d25e4d3629cef711cd2863f2961b10409f0c04cb8b7d9020", size = 865445, upload-time = "2026-05-09T23:13:12.523Z" }, + { url = "https://files.pythonhosted.org/packages/1a/89/f05169e8588aac365f35ffc7f3bc3184f095ef4cfded7cfaa3c7fd5dbd89/regex-2026.5.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:508f56a89ba9cb26e4168cbc37dbd60a28d82430a9e18ad1d25fe0883c314ca2", size = 912322, upload-time = "2026-05-09T23:13:14.281Z" }, + { url = "https://files.pythonhosted.org/packages/30/e1/c93444052cf41581f3c884ab3fb5823daf0992f11cd4388d4275ca610558/regex-2026.5.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b6d189041f15691cfa2b6c4290448ec221244d225b3f5fe9e7771b34ffcdf6e2", size = 801269, upload-time = "2026-05-09T23:13:16.569Z" }, + { url = "https://files.pythonhosted.org/packages/50/fe/0cf96b882f540e62e8b9956599798203d599c44cf4c77917ca27400ff69b/regex-2026.5.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e82db382b44d0111b22601c509c89f64434816c9e0eef9d1989cda8cc6ff1c04", size = 777085, upload-time = "2026-05-09T23:13:18.675Z" }, + { url = "https://files.pythonhosted.org/packages/23/5c/d78d4924e7fc875557b9e9b768423925fdfaac5549d06da7810019a9bd26/regex-2026.5.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2acfb48634f64996b57f90f39afa692ff362162722581921fe92239a59960f3c", size = 785153, upload-time = "2026-05-09T23:13:20.525Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e0/5214774090e7b4524dcea3e3c4aa74141d43043f8beb49c1599db1c8b53a/regex-2026.5.9-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d29eebfc9525db68cad3c97eedd7f754fa265aa5cd0cf4f863b2421e1b48fc9f", size = 860164, upload-time = "2026-05-09T23:13:22.263Z" }, + { url = "https://files.pythonhosted.org/packages/6e/e1/4a57a83350319b1271f0d7a249b8672513ed928b237a741631270de6caea/regex-2026.5.9-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:debb893095e944091c16e641a6e33c1b0f4cb61ab945ec5afbf53ce7068834d8", size = 765731, upload-time = "2026-05-09T23:13:24.277Z" }, + { url = "https://files.pythonhosted.org/packages/12/f4/499e74a20c156fc75836ee04a72a38d1a063978f600937f9760467beb1b0/regex-2026.5.9-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d659eee77986549c9ea45b861c7567e44d6287c3dc9a4565478853f7b9fe2ff6", size = 852062, upload-time = "2026-05-09T23:13:26.125Z" }, + { url = "https://files.pythonhosted.org/packages/5b/92/7eebc0d0a01e78629695f342ba17e0deaff8fb45e79cc0d7b98287da6e3e/regex-2026.5.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2efa205e6d98b24d1f3ab395c11aa15cdf10935bca283d0285e0499c284fba21", size = 789577, upload-time = "2026-05-09T23:13:27.814Z" }, + { url = "https://files.pythonhosted.org/packages/05/a4/018e71f7d2ad48c1ebe6d3ae0026f9b7cb4802fd15c7cc02fdf724355102/regex-2026.5.9-cp313-cp313-win32.whl", hash = "sha256:f3844f134e834076677dd369976e9f5068679fcb8e50102fdf6b7ac96a3ec127", size = 266691, upload-time = "2026-05-09T23:13:29.549Z" }, + { url = "https://files.pythonhosted.org/packages/e6/1d/861a93719fb9ee7dbfc3761b3797b7a3e112a5d42c6129459d2d741be9b5/regex-2026.5.9-cp313-cp313-win_amd64.whl", hash = "sha256:3527bb4942d2c14552155406cdedd906567456821848aed1cb4933a391bf5eca", size = 277747, upload-time = "2026-05-09T23:13:31.859Z" }, + { url = "https://files.pythonhosted.org/packages/d9/c6/0a2436ae4da1ba76e51cb98943c6838a9a721faa40ebe2dce07694ae34e3/regex-2026.5.9-cp313-cp313-win_arm64.whl", hash = "sha256:56a33f191f17d8c417f99945ebdc1e691d3af9605d86ec68c7e54a57e3e17af6", size = 270500, upload-time = "2026-05-09T23:13:33.525Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e9/d21346f7b60ed58789371358ed66b09d00f832e1bd7c06e55d9da5679882/regex-2026.5.9-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:01f28d868834624c934b8d2e0aa1c8341337e37831f4a012f18a5afcba4cbaf3", size = 494172, upload-time = "2026-05-09T23:13:35.935Z" }, + { url = "https://files.pythonhosted.org/packages/c4/43/fd1177a2032037c681baecdb3422ee4e1424aec4e4f470ef47793d325274/regex-2026.5.9-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:48036f6374aaa79eb3b754ec29c61d1c6b1606749d705a13f8854fa2539671f6", size = 293952, upload-time = "2026-05-09T23:13:38.307Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7d/9fbf919768368d3f8a4f6c692cf2aa61e482b2b81ec6a298ace4cbf02480/regex-2026.5.9-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b96350aa424e79d4fd6b567b344dcbe2b2d6bfc48dfe7717587e1fa6d43da6ff", size = 292314, upload-time = "2026-05-09T23:13:40.353Z" }, + { url = "https://files.pythonhosted.org/packages/e2/6c/e41bfeecb589716843e7c4df09ba46ff2a42961457afece19059d85caeef/regex-2026.5.9-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f3af7a4903c5c04a11a196a5aa75cdd7dd3f8508132f9fb3259d9f5908e3b88", size = 811681, upload-time = "2026-05-09T23:13:42.543Z" }, + { url = "https://files.pythonhosted.org/packages/87/83/a5c1c525fba0aa656e88ad0face0b1829788ef4c2fb6b26df58aa1151b84/regex-2026.5.9-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7e87577720152d2caae19fe2baaf1f8d5ca12091e9e229f03915c37d1e4b9178", size = 871135, upload-time = "2026-05-09T23:13:44.326Z" }, + { url = "https://files.pythonhosted.org/packages/18/d4/80882e799e440dd878b0979cbebf8fa4d54624a332c83037c7a701649e3f/regex-2026.5.9-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c8b9b9d294cfea3cd19c718ade7cc93492b2c4991abd9a68d0b3477ae6d8e100", size = 917265, upload-time = "2026-05-09T23:13:47.295Z" }, + { url = "https://files.pythonhosted.org/packages/ae/ff/8db60211e2286e396aad7dc7725356c502bff0901ea05bd6cdc2e1a042b9/regex-2026.5.9-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:728d8bfd28a8845c8b6bc5dc7ce010453d206396786c0765c2740cb65f37791e", size = 816311, upload-time = "2026-05-09T23:13:49.885Z" }, + { url = "https://files.pythonhosted.org/packages/4c/47/742ef579c61730f8d268e5cf1f9ce0e37e2ea041ad0f5644724f2378e463/regex-2026.5.9-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7e30b874d341fac767d7df5a0870540541c2c054b80cfaac116e8d367a8a7ff2", size = 785498, upload-time = "2026-05-09T23:13:52.25Z" }, + { url = "https://files.pythonhosted.org/packages/7f/ab/cb0999802dcb0fb95b1ab005e8d4163d8afdd67efc2cb6b6630ac13f8cb1/regex-2026.5.9-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fd190e88a895a8901325fad284a3f74ea52b1da8525b76cc811fa9b1edf0ce2b", size = 801348, upload-time = "2026-05-09T23:13:54.127Z" }, + { url = "https://files.pythonhosted.org/packages/7d/62/8ca59a24c55bc34d166eefaf3717bd77772f329fdbf984d86581e0a3571c/regex-2026.5.9-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:8e76e8161ad00694cfce6767d5dea860c6391ac5b83e5c3a39661e696f11fc7e", size = 866493, upload-time = "2026-05-09T23:13:56.067Z" }, + { url = "https://files.pythonhosted.org/packages/8d/3d/30f2ae62cef3278bb5bb821f467277a55fb73f01032cf85997e15e8289a8/regex-2026.5.9-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ddda5340e6c01a293027dd46232fa79eaff1b48058ce7a98f572b6445b088041", size = 772811, upload-time = "2026-05-09T23:13:57.867Z" }, + { url = "https://files.pythonhosted.org/packages/d8/ae/7d2089bcd78ad0c0161bc684339df50032acb438a7bd3305e7ddb1193cec/regex-2026.5.9-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:205109e96b3cf5adf8f4cd62bedde9487feb282b9497a3535451e5a24cd706a0", size = 856584, upload-time = "2026-05-09T23:13:59.679Z" }, + { url = "https://files.pythonhosted.org/packages/a9/29/92ff47f75990131ea4f24ba17819e5a9d141e10819807e09addd73409af6/regex-2026.5.9-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dfbe4579b9f08036aa7d101d1835437a20783574ac66327e6b29b4018a138081", size = 803453, upload-time = "2026-05-09T23:14:01.978Z" }, + { url = "https://files.pythonhosted.org/packages/04/99/eff29f1037dcab36702c9ee5d6858cf1ce2336ea8ea2987f64245b99ea5e/regex-2026.5.9-cp313-cp313t-win32.whl", hash = "sha256:ed2c9e8068b614c574d8d30e543d617cf5379b0535d46f97ef00e904745a08b5", size = 269951, upload-time = "2026-05-09T23:14:03.661Z" }, + { url = "https://files.pythonhosted.org/packages/0e/9d/8870b8981d27b22cda77bb26a5ac7ebfa9c7d9e0dea195a834a82380e748/regex-2026.5.9-cp313-cp313t-win_amd64.whl", hash = "sha256:b46b0f094dc1d3b90356c85a0bd2c9bafc4a6a190b9d6f8ddd5a033b6e088ed4", size = 281240, upload-time = "2026-05-09T23:14:05.56Z" }, + { url = "https://files.pythonhosted.org/packages/72/b1/3379415e8f135c13ac551353397cc4fe97b4978f3cac73c5fcbcded548b8/regex-2026.5.9-cp313-cp313t-win_arm64.whl", hash = "sha256:872acc074bd29ffc9913ecdfedf6ea77502312ca44a4aa0d3779089c6069d8de", size = 272383, upload-time = "2026-05-09T23:14:07.843Z" }, + { url = "https://files.pythonhosted.org/packages/13/3e/9c3cd292d8808b3645a2ce517e200179b6d0e903f176300bd8b542e14de5/regex-2026.5.9-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:1bd7587a2948b4085195d5a3374eaf4a425dc3e55784c038175355ecf3bbbf8a", size = 490376, upload-time = "2026-05-09T23:14:09.64Z" }, + { url = "https://files.pythonhosted.org/packages/60/70/d43ee8a2ca0a8b68d167f21658b85520ac0574617c7f320367c5047f7556/regex-2026.5.9-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:dea2e88e1cce4522496cce630e11e67b98b7076620bc4336c3f674bc21a375f4", size = 291964, upload-time = "2026-05-09T23:14:11.424Z" }, + { url = "https://files.pythonhosted.org/packages/21/91/9d50b433828d8e74196904e168a43abf1e6e88b2a15d47ed742456720c37/regex-2026.5.9-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2099f7e7ff7b6aa3192312650a56e91cc091e49d50b04e4f6f8b6e28b3b27f1c", size = 289682, upload-time = "2026-05-09T23:14:13.123Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d2/b835e3cafbb9d977736912436259ff551d60919f7d7b3d37d46659c63564/regex-2026.5.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecd353045824e4477562a2ac718c25799cdaaa41f7aa925a806a8a3e6848a5b9", size = 796996, upload-time = "2026-05-09T23:14:14.923Z" }, + { url = "https://files.pythonhosted.org/packages/2c/a6/9f992d00019166b9de01c546dd4549bc679f2a68df11b877740b0760b7c2/regex-2026.5.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:65c8c8c37377794bd5b2f3ebe51919042bf17aec802e23c833d89782ed0c78af", size = 866089, upload-time = "2026-05-09T23:14:17.757Z" }, + { url = "https://files.pythonhosted.org/packages/e0/08/4d32af657e049b19cb62b02e46e38fe1518797bfb2203ee93a510b21b0dc/regex-2026.5.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5b73ab8afcf66c622db143d1c6fda4e58e4d537ee4f125229ad47b1ab80f34c0", size = 911530, upload-time = "2026-05-09T23:14:20.353Z" }, + { url = "https://files.pythonhosted.org/packages/d9/27/2af43dd1dc201d1fecefda64a45f4ad0995855b92724f795a777b402ee69/regex-2026.5.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0de5cf193997384ed2ca6f1cd4f78055b255d93d82d5a8cd6ba0d11c10b167e4", size = 800643, upload-time = "2026-05-09T23:14:22.265Z" }, + { url = "https://files.pythonhosted.org/packages/a4/dd/23a249047013b5321d4a60c4d2437462086f601b061776a525e5fba2a59f/regex-2026.5.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d641a8c9a61618047796d572a39a79b26167b0411d2c3031937b2fe2d081e2cf", size = 777223, upload-time = "2026-05-09T23:14:24.179Z" }, + { url = "https://files.pythonhosted.org/packages/94/6a/e85ed9538cd19586d0465076a4578a12e093ce776d15f3f8ce92733a8dd6/regex-2026.5.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:24b2355ef5cc9aa5b8f07d17704face1c166fdcc2290fa7bd6e6c925655a8346", size = 785760, upload-time = "2026-05-09T23:14:26.065Z" }, + { url = "https://files.pythonhosted.org/packages/2a/c4/f25473209438638e947c55f9156fd8f236f74169229028cc99116380868e/regex-2026.5.9-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:a24852d3c29ad9e47593593d8a247c44ccc3d0548ef12c822d6ed0810affe676", size = 860891, upload-time = "2026-05-09T23:14:28.17Z" }, + { url = "https://files.pythonhosted.org/packages/f9/f7/f4f86e3c74419c37370e91f150ae0c2ef7d34b2e0e4cdd5da046a02e4022/regex-2026.5.9-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:916714069da19329ef7de197dcbc77bb3104145c7c2c864dbfbe318f46b88b14", size = 765891, upload-time = "2026-05-09T23:14:30.06Z" }, + { url = "https://files.pythonhosted.org/packages/26/70/704d8e13765939146b1cd0ef4e2feb71d7929727d2290f026eed10095955/regex-2026.5.9-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:fa411799ca8da32a8d38d020a88faa5b6f91657d284761352940ecf9f7c3bbdd", size = 851380, upload-time = "2026-05-09T23:14:32.123Z" }, + { url = "https://files.pythonhosted.org/packages/26/29/1a13582a8460038edc38e49f64ceb0dd7c60f5caba77571f4bf6601965d9/regex-2026.5.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1e6da47d679b7010ef27556b6e0f99771b744936db1792a10ceac6547ae1503e", size = 789350, upload-time = "2026-05-09T23:14:34.799Z" }, + { url = "https://files.pythonhosted.org/packages/73/56/3dcafe34fc72e271d62ad9a291801e88a1457bb251c132f15fcc2e5aad1a/regex-2026.5.9-cp314-cp314-win32.whl", hash = "sha256:98bd73080e8756255137e1bd3f3f00295bbc5aa383c0e0f973920e9134d7c4ad", size = 272130, upload-time = "2026-05-09T23:14:36.729Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9c/02eebf0be95efe416c664db7fb8b6b05b7a0b06a7544f2884f2558b0526f/regex-2026.5.9-cp314-cp314-win_amd64.whl", hash = "sha256:ff8d372ac2acdc048d1c19916f27ee61bc5722728458ba6ca5052f2c72d51763", size = 280999, upload-time = "2026-05-09T23:14:39.126Z" }, + { url = "https://files.pythonhosted.org/packages/70/5a/1dd1abee76cb7a846a0bcf42fdc87e5720c3c33c24f3e37814310a513d9f/regex-2026.5.9-cp314-cp314-win_arm64.whl", hash = "sha256:e1d93bf647916292e8edcec150c07ddf3dc50179ccaf770c04a7f9e452155372", size = 273500, upload-time = "2026-05-09T23:14:41.059Z" }, + { url = "https://files.pythonhosted.org/packages/86/c1/c5f619b0057a7965cb78ec559c1d7a45ce8c99a35bea95483d64959a93d9/regex-2026.5.9-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:83d0ee4a57d1c87cb549e195ec300b8f0ec3a82eba66d835e4e2ed8634fe4499", size = 494269, upload-time = "2026-05-09T23:14:42.869Z" }, + { url = "https://files.pythonhosted.org/packages/05/2c/5d01f1aee33de4bbe60c8452945bfc8477ca7c5ae4450f6bfe711036cb36/regex-2026.5.9-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:d3d7eb5c9a7f6df82ed3cfac9beb93882a5cbcb5b8b157b56cb2b3b276574ac1", size = 293954, upload-time = "2026-05-09T23:14:44.822Z" }, + { url = "https://files.pythonhosted.org/packages/7a/fe/e8988b2ae2108c6ef71bd4aa8d87fbe257976dd0810e826cd75f701c68b6/regex-2026.5.9-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:075160bf16658e16d35233300b8453aac25de4cbea808d22348b6979668e924d", size = 292405, upload-time = "2026-05-09T23:14:47.211Z" }, + { url = "https://files.pythonhosted.org/packages/79/34/d2b0937faa7859263f7f0a3c6b103a1296306be6952dc173d0154e9a2f49/regex-2026.5.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45375819235558a4ff1c4971dc32881f022613abdb180128f5cb4768c1765a1c", size = 811855, upload-time = "2026-05-09T23:14:49.21Z" }, + { url = "https://files.pythonhosted.org/packages/80/fe/daf53a47457a8486db66c66c01ceb9c2303eecee3f87197f1e77eb1a736d/regex-2026.5.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ead4b163ac30a29574510cd4b3e2e985ac5290c05fc7095557d6a5f403fc31b5", size = 871189, upload-time = "2026-05-09T23:14:51.555Z" }, + { url = "https://files.pythonhosted.org/packages/1c/75/058fc4470cbfbf57d800aff1a0022b929a3f9fa553ee10a0cdf2070eb31f/regex-2026.5.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8c6e4218fbdfbcd4f6c19efca40930d24a621bf4b48cb76bc6640543bd28ef20", size = 917485, upload-time = "2026-05-09T23:14:53.633Z" }, + { url = "https://files.pythonhosted.org/packages/88/e7/179cfda3a28bc843b5c6cfe7f79f23489c791ed95f151083803660878432/regex-2026.5.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6351571c8a42b505eb555c0dc47d740d0fb66977dc142919eea6f4325b7c56a0", size = 816369, upload-time = "2026-05-09T23:14:56.198Z" }, + { url = "https://files.pythonhosted.org/packages/41/90/6f0cc422071688266d344fca8462d787cba0a2c144acb25721f9a61ec265/regex-2026.5.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:002205cafd2a9e78c6290c7d1df277bf3277b3b7a30e0b4bb0dac2e2e3f7cb2d", size = 785869, upload-time = "2026-05-09T23:14:58.602Z" }, + { url = "https://files.pythonhosted.org/packages/02/67/a31f1760f09c27b251ef39e9beb541f462cf977381d067faa764c2c0e393/regex-2026.5.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8abd33fef90b2a9efac5557d6033ca82d1195ed3a15fea5af15ba7b463c6a63b", size = 801427, upload-time = "2026-05-09T23:15:00.642Z" }, + { url = "https://files.pythonhosted.org/packages/e3/c4/1a80654597b6bc1e1ea0494824c31200e8a956abe290afae9b19a166a148/regex-2026.5.9-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:31037c82eccb44b7ea2e9e221d7c01429430e989a1f4b91ea5a855f6017b509a", size = 866482, upload-time = "2026-05-09T23:15:03.384Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/960724e06482c08466ff5611e242e86f80062949cdf6b4b9cc317b9dd93d/regex-2026.5.9-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5604dfd046dc37eca90250fc3be938b076c8059fa772ac0ed6f499b0f0fb0415", size = 773022, upload-time = "2026-05-09T23:15:05.625Z" }, + { url = "https://files.pythonhosted.org/packages/50/a8/a9979c3e7918280e93159ebcab5ef1a65116dd4f3bd6091be0eae4a126e8/regex-2026.5.9-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:0e1b1b4e496afbb24f4a62aba855ee4f88f25578927697b340702e48c9ee6bc2", size = 856642, upload-time = "2026-05-09T23:15:07.966Z" }, + { url = "https://files.pythonhosted.org/packages/fe/d4/a9b732f2f0072c0ab12227483abb24fffcb9f73f8a2b203df0a6d0434735/regex-2026.5.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:be3372b9df6ddecff6486d37e19095a7b4973137caf5512407a89f4455361f41", size = 803552, upload-time = "2026-05-09T23:15:10.215Z" }, + { url = "https://files.pythonhosted.org/packages/d5/fe/1b3113817447a1d4155e4ac76d2e072f42c0bcba2f43fa8a0e756ea2cd91/regex-2026.5.9-cp314-cp314t-win32.whl", hash = "sha256:3ddd90103f9e5c471c49c7852ecc1fe27c7e45eb99e977aefe7caa4e779f4f58", size = 275746, upload-time = "2026-05-09T23:15:12.609Z" }, + { url = "https://files.pythonhosted.org/packages/92/73/93d42045302636c91f2e5ef588b65b84b01428f28ec77de256b1dfdfbe5c/regex-2026.5.9-cp314-cp314t-win_amd64.whl", hash = "sha256:ca518ed29c46eecba6010b15f1b9a479314d2de409536e71b6a13aa04e3b8a77", size = 285685, upload-time = "2026-05-09T23:15:15.086Z" }, + { url = "https://files.pythonhosted.org/packages/da/80/35b4c33c804a165a7f55289afda3ea9e3eb6d15800341a2d66455c0f1f30/regex-2026.5.9-cp314-cp314t-win_arm64.whl", hash = "sha256:5e41809d2683fcde7d5a8c87a6567ba1fb1ce0de9f31bff578de00a4b2d76daa", size = 275713, upload-time = "2026-05-09T23:15:16.98Z" }, ] [[package]] name = "requests" -version = "2.33.1" +version = "2.34.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, @@ -1189,9 +1197,9 @@ dependencies = [ { name = "idna" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5f/a4/98b9c7c6428a668bf7e42ebb7c79d576a1c3c1e3ae2d47e674b468388871/requests-2.33.1.tar.gz", hash = "sha256:18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517", size = 134120, upload-time = "2026-03-30T16:09:15.531Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/8e/7540e8a2036f79a125c1d2ebadf69ed7901608859186c856fa0388ef4197/requests-2.33.1-py3-none-any.whl", hash = "sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a", size = 64947, upload-time = "2026-03-30T16:09:13.83Z" }, + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] [[package]] @@ -1209,27 +1217,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.15.12" +version = "0.15.14" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/99/43/3291f1cc9106f4c63bdce7a8d0df5047fe8422a75b091c16b5e9355e0b11/ruff-0.15.12.tar.gz", hash = "sha256:ecea26adb26b4232c0c2ca19ccbc0083a68344180bba2a600605538ce51a40a6", size = 4643852, upload-time = "2026-04-24T18:17:14.305Z" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/8a/8bce2894573e9dae6ff4d77fe34ad727d79b9e6238ad288c5638990d90f6/ruff-0.15.14.tar.gz", hash = "sha256:48e866b165be4a9bdbf310f7d3c9a07edef2fe8cd63ffeb4e00bb590506ebf9f", size = 4700910, upload-time = "2026-05-21T14:34:55.177Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/6e/e78ffb61d4686f3d96ba3df2c801161843746dcbcbb17a1e927d4829312b/ruff-0.15.12-py3-none-linux_armv6l.whl", hash = "sha256:f86f176e188e94d6bdbc09f09bfd9dc729059ad93d0e7390b5a73efe19f8861c", size = 10640713, upload-time = "2026-04-24T18:17:22.841Z" }, - { url = "https://files.pythonhosted.org/packages/ae/08/a317bc231fb9e7b93e4ef3089501e51922ff88d6936ce5cf870c4fe55419/ruff-0.15.12-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e3bcd123364c3770b8e1b7baaf343cc99a35f197c5c6e8af79015c666c423a6c", size = 11069267, upload-time = "2026-04-24T18:17:30.105Z" }, - { url = "https://files.pythonhosted.org/packages/aa/a4/f828e9718d3dce1f5f11c39c4f65afd32783c8b2aebb2e3d259e492c47bd/ruff-0.15.12-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fe87510d000220aa1ed530d4448a7c696a0cae1213e5ec30e5874287b66557b5", size = 10397182, upload-time = "2026-04-24T18:17:07.177Z" }, - { url = "https://files.pythonhosted.org/packages/71/e0/3310fc6d1b5e1fdea22bf3b1b807c7e187b581021b0d7d4514cccdb5fb71/ruff-0.15.12-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84a1630093121375a3e2a95b4a6dc7b59e2b4ee76216e32d81aae550a832d002", size = 10758012, upload-time = "2026-04-24T18:16:55.759Z" }, - { url = "https://files.pythonhosted.org/packages/11/c1/a606911aee04c324ddaa883ae418f3569792fd3c4a10c50e0dd0a2311e1e/ruff-0.15.12-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fb129f40f114f089ebe0ca56c0d251cf2061b17651d464bb6478dc01e69f11f5", size = 10447479, upload-time = "2026-04-24T18:16:51.677Z" }, - { url = "https://files.pythonhosted.org/packages/9d/68/4201e8444f0894f21ab4aeeaee68aa4f10b51613514a20d80bd628d57e88/ruff-0.15.12-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0c862b172d695db7598426b8af465e7e9ac00a3ea2a3630ee67eb82e366aaa6", size = 11234040, upload-time = "2026-04-24T18:17:16.529Z" }, - { url = "https://files.pythonhosted.org/packages/34/ff/8a6d6cf4ccc23fd67060874e832c18919d1557a0611ebef03fdb01fff11e/ruff-0.15.12-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2849ea9f3484c3aca43a82f484210370319e7170df4dfe4843395ddf6c57bc33", size = 12087377, upload-time = "2026-04-24T18:17:04.944Z" }, - { url = "https://files.pythonhosted.org/packages/85/f6/c669cf73f5152f623d34e69866a46d5e6185816b19fcd5b6dd8a2d299922/ruff-0.15.12-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9e77c7e51c07fe396826d5969a5b846d9cd4c402535835fb6e21ce8b28fef847", size = 11367784, upload-time = "2026-04-24T18:17:25.409Z" }, - { url = "https://files.pythonhosted.org/packages/e8/39/c61d193b8a1daaa8977f7dea9e8d8ba866e02ea7b65d32f6861693aa4c12/ruff-0.15.12-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83b2f4f2f3b1026b5fb449b467d9264bf22067b600f7b6f41fc5958909f449d0", size = 11344088, upload-time = "2026-04-24T18:17:12.258Z" }, - { url = "https://files.pythonhosted.org/packages/c2/8d/49afab3645e31e12c590acb6d3b5b69d7aab5b81926dbaf7461f9441f37a/ruff-0.15.12-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:9ba3b8f1afd7e2e43d8943e55f249e13f9682fde09711644a6e7290eb4f3e339", size = 11271770, upload-time = "2026-04-24T18:17:02.457Z" }, - { url = "https://files.pythonhosted.org/packages/46/06/33f41fe94403e2b755481cdfb9b7ef3e4e0ed031c4581124658d935d52b4/ruff-0.15.12-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e852ba9fdc890655e1d78f2df1499efbe0e54126bd405362154a75e2bde159c5", size = 10719355, upload-time = "2026-04-24T18:17:27.648Z" }, - { url = "https://files.pythonhosted.org/packages/0d/59/18aa4e014debbf559670e4048e39260a85c7fcee84acfd761ac01e7b8d35/ruff-0.15.12-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dd8aed930da53780d22fc70bdf84452c843cf64f8cb4eb38984319c24c5cd5fd", size = 10462758, upload-time = "2026-04-24T18:17:32.347Z" }, - { url = "https://files.pythonhosted.org/packages/25/e7/cc9f16fd0f3b5fddcbd7ec3d6ae30c8f3fde1047f32a4093a98d633c6570/ruff-0.15.12-py3-none-musllinux_1_2_i686.whl", hash = "sha256:01da3988d225628b709493d7dc67c3b9b12c0210016b08690ef9bd27970b262b", size = 10953498, upload-time = "2026-04-24T18:17:20.674Z" }, - { url = "https://files.pythonhosted.org/packages/72/7a/a9ba7f98c7a575978698f4230c5e8cc54bbc761af34f560818f933dafa0c/ruff-0.15.12-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:9cae0f92bd5700d1213188b31cd3bdd2b315361296d10b96b8e2337d3d11f53e", size = 11447765, upload-time = "2026-04-24T18:17:09.755Z" }, - { url = "https://files.pythonhosted.org/packages/ea/f9/0ae446942c846b8266059ad8a30702a35afae55f5cdc54c5adf8d7afdc27/ruff-0.15.12-py3-none-win32.whl", hash = "sha256:d0185894e038d7043ba8fd6aee7499ece6462dc0ea9f1e260c7451807c714c20", size = 10657277, upload-time = "2026-04-24T18:17:18.591Z" }, - { url = "https://files.pythonhosted.org/packages/33/f1/9614e03e1cdcbf9437570b5400ced8a720b5db22b28d8e0f1bda429f660d/ruff-0.15.12-py3-none-win_amd64.whl", hash = "sha256:c87a162d61ab3adca47c03f7f717c68672edec7d1b5499e652331780fe74950d", size = 11837758, upload-time = "2026-04-24T18:17:00.113Z" }, - { url = "https://files.pythonhosted.org/packages/c0/98/6beb4b351e472e5f4c4613f7c35a5290b8be2497e183825310c4c3a3984b/ruff-0.15.12-py3-none-win_arm64.whl", hash = "sha256:a538f7a82d061cee7be55542aca1d86d1393d55d81d4fcc314370f4340930d4f", size = 11120821, upload-time = "2026-04-24T18:16:57.979Z" }, + { url = "https://files.pythonhosted.org/packages/b9/c8/74a92c6ff9fcfb4f1f947126d3ebee8389276e161ecc85de5bda7cda51bd/ruff-0.15.14-py3-none-linux_armv6l.whl", hash = "sha256:8dd2db9416e487c8d4b01fa7056bb02c4d05969d4f8d17a08c229c2f4ff3c108", size = 10739177, upload-time = "2026-05-21T14:34:37.332Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/254a35c20acc38a7223c9d2d594af12e794432464f2cdeb52af1dc4a892d/ruff-0.15.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:be4ff55af755bd71a00ab3dc6bd7ffc467bd76e0df6881e286c2e3d23e8fb43b", size = 11144969, upload-time = "2026-05-21T14:34:43.978Z" }, + { url = "https://files.pythonhosted.org/packages/56/9e/d13e40f83b8d0a94430e6778ce1d94a43b38cf2efe63278bdd2b4c65abbf/ruff-0.15.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:48d5909d7d06276ce7dde6d32bfa4b0d4cb2651145cd8ee4b440722cbc77832f", size = 10478207, upload-time = "2026-05-21T14:34:48.378Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f1/b15a7839fa4f332f8acec78e20564f26bb2d866e3d21710b877fd0263000/ruff-0.15.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca8cbfa94c4f90984a67561978602746d4cd27103568f745fa90eee3f0d4107d", size = 10818459, upload-time = "2026-05-21T14:34:22.318Z" }, + { url = "https://files.pythonhosted.org/packages/45/33/53d651177f84f94b400a0e27f8824eeada3dddc9d5ee8aeb048f4352a520/ruff-0.15.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9a6bbc0333f1ab053423bcbf6226477d266ca7cec7738c4c8e3f55647803f3c4", size = 10541800, upload-time = "2026-05-21T14:34:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a6/868f87e0bf9786ed24b5d0d0ad8676b8a94fd1912f42cddf9cfc7857818a/ruff-0.15.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a24a4f7605d7003a6674d4387651effd939dead3fddd0f36561eb77a9a2e542", size = 11342149, upload-time = "2026-05-21T14:34:46.365Z" }, + { url = "https://files.pythonhosted.org/packages/a7/8b/38cd5c19faffdcc05a408d2b78edccc69492ab9720eadb49ea15ef80d768/ruff-0.15.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:049b5326e53ed80978f2fc041a280603f69dd6b0c95464342a2bb4572d9d9e2f", size = 12212563, upload-time = "2026-05-21T14:34:28.579Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4d/a3c5b874a556d5731e3e657aaf04311bb76f0a5c3ec220ed43051be6b64b/ruff-0.15.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4ed42e6696c8dfa5f06728e6441993901f548eb92d73bc472cb5a38d1395fbf", size = 11493299, upload-time = "2026-05-21T14:34:41.836Z" }, + { url = "https://files.pythonhosted.org/packages/1e/c0/56472c251d09858a53e51efbd485b09e1995d8731668b76d52e5dd6ee0f1/ruff-0.15.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:715c543cf450c4888251f91c52f1942a800541d9bddd7ac060aa4e6b77ae7cba", size = 11455931, upload-time = "2026-05-21T14:34:57.276Z" }, + { url = "https://files.pythonhosted.org/packages/2c/4a/e2e7b4d8dbf233d4eace59c75bc3435fa6d8bd3bae82d351d4e4300c0fd1/ruff-0.15.14-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:72ebab6013ec887d439d8b7593737a0a4ffb06d45d209d4e4bf2e92813082d3f", size = 11400794, upload-time = "2026-05-21T14:34:39.773Z" }, + { url = "https://files.pythonhosted.org/packages/97/c7/83c0539fe34c3e09136204d1e75d6052492364e0b3cb05e9465423f567d7/ruff-0.15.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:49072d36abdbe97a8dd7f480afe9c675699c0c495d4c84076e2c1203c4550581", size = 10804759, upload-time = "2026-05-21T14:34:31.045Z" }, + { url = "https://files.pythonhosted.org/packages/86/a6/18f2bfc095a2ab4a78745644e428205532ce6653a5d0fa8501572891534d/ruff-0.15.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:958522aee105068640c2c2ceae08f413ae44d922f52a1374ac13d6a96032fc93", size = 10539517, upload-time = "2026-05-21T14:34:53.064Z" }, + { url = "https://files.pythonhosted.org/packages/54/3a/5a8b3b69c654d4e4bf1d246ac5b49cbcdac6eaab6905925f8915f31e3b80/ruff-0.15.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f3707da619a143a2e8830e2abab8224478d69ace2d28cb6c20543ae97c36bf61", size = 11065169, upload-time = "2026-05-21T14:34:24.484Z" }, + { url = "https://files.pythonhosted.org/packages/ed/c5/8864e4e7925b836ea354b31d57641ec03830564e281a8b6f061f8c3e0ec1/ruff-0.15.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:bb01d645694e3ec0102105d07ef2d53703970407d59c04e59d3ba0b7a1d53553", size = 11560214, upload-time = "2026-05-21T14:34:50.975Z" }, + { url = "https://files.pythonhosted.org/packages/36/38/012bf76752e1f89ed50b77b99532d90f3a3e287bc7918e1fc0948ac866ac/ruff-0.15.14-py3-none-win32.whl", hash = "sha256:6d0c1ad2a0ab718d39b6d8fd2217981ce4d625cd96a720095f798fb47d8b13e6", size = 10805548, upload-time = "2026-05-21T14:34:33.453Z" }, + { url = "https://files.pythonhosted.org/packages/d1/b7/4ea2c170f10ad760fff2a5250beb18897719dc8b52b53a24cddbb9dd3f19/ruff-0.15.14-py3-none-win_amd64.whl", hash = "sha256:802342981e056db3851a7836e5b070f8f15f67d4a685ae2a6160939d364b2902", size = 11939523, upload-time = "2026-05-21T14:34:18.077Z" }, + { url = "https://files.pythonhosted.org/packages/62/d5/bc97ff895ec35cf3925d4bd60f3b39d822f377a446906ec9bcc87405e59b/ruff-0.15.14-py3-none-win_arm64.whl", hash = "sha256:ff47b90a9ef6a40c9e2f3b479c1fb78531adf055b94c1eba0a7ba04b31951826", size = 11208607, upload-time = "2026-05-21T14:34:26.525Z" }, ] [[package]] @@ -1345,22 +1353,22 @@ wheels = [ [[package]] name = "sentence-transformers" -version = "5.4.1" +version = "5.5.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "huggingface-hub" }, { name = "numpy" }, { name = "scikit-learn" }, { name = "scipy" }, - { name = "torch", version = "2.11.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin'" }, - { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin'" }, + { name = "torch", version = "2.12.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin'" }, + { name = "torch", version = "2.12.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin'" }, { name = "tqdm" }, { name = "transformers" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4d/68/7f98c221940ce783b492ad6140384daf2e2918cd7175009d6a362c22b9ee/sentence_transformers-5.4.1.tar.gz", hash = "sha256:436bcb1182a0ff42a8fb2b1c43498a70d0a75b688d182f2cd0d1dd115af61ddc", size = 428910, upload-time = "2026-04-14T13:34:59.006Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cf/d4/7ef93157485e978c016f49da05363c1e4e7237beb5343b64b5631101f0f1/sentence_transformers-5.5.1.tar.gz", hash = "sha256:02b7740dfc60bdbbcb6061625f5d97a5c1a4e2d3baac5f9391b912bb5eae2290", size = 445161, upload-time = "2026-05-20T07:37:44.465Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/d9/3a9b6f2ccdedc9dc00fe37b2fc58f58f8efbff44565cf4bf39d8568bb13a/sentence_transformers-5.4.1-py3-none-any.whl", hash = "sha256:a6d640fc363849b63affb8e140e9d328feabab86f83d58ac3e16b1c28140b790", size = 571311, upload-time = "2026-04-14T13:34:57.731Z" }, + { url = "https://files.pythonhosted.org/packages/bf/03/ee99a6b030e7a2e056547729f8a4709dd93e13d9c6f07590f74c395c4017/sentence_transformers-5.5.1-py3-none-any.whl", hash = "sha256:4fe11d433badc5282d32f7fc08bc714216b7a5aca426f9df77a45a554756deb7", size = 588887, upload-time = "2026-05-20T07:37:43.004Z" }, ] [[package]] @@ -1392,23 +1400,23 @@ wheels = [ [[package]] name = "sqlglot" -version = "30.6.0" +version = "30.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3c/66/6ece15f197874e56c76e1d0269cebf284ba992a80dfadca9d1972fdf7edf/sqlglot-30.6.0.tar.gz", hash = "sha256:246d34d39927422a50a3fa155f37b2f6346fba85f1a755b13c941eb32ef93361", size = 5835307, upload-time = "2026-04-20T20:11:08.164Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/64/89299aefc6ebdf4fc899f5dc14c7fcb7eb9da9290a2b4d615ae7ab884b17/sqlglot-30.8.0.tar.gz", hash = "sha256:1c5f93fb742dd9aaa75eee6bb33a637794a858b9a86375fac23a2dc0f7bc127e", size = 5869750, upload-time = "2026-05-13T09:04:38.923Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/e7/64fe971cbca33a0446b06f4a5ff8e3fa4a1dbd0a039ceabcc3e6cf4087a9/sqlglot-30.6.0-py3-none-any.whl", hash = "sha256:e005fc2f47994f90d7d8df341f1cbe937518497b0b7b1507d4c03c4c9dfd2778", size = 673920, upload-time = "2026-04-20T20:11:05.758Z" }, + { url = "https://files.pythonhosted.org/packages/88/4e/80705091aaf9c95e125d243f0aa871bc9f3670b4c9d963e6bad3b3dce8ff/sqlglot-30.8.0-py3-none-any.whl", hash = "sha256:af903378c331d5b72277a1b41118f07bc3e50cf4478e2d47eed12c96ee6a22a4", size = 687831, upload-time = "2026-05-13T09:04:36.336Z" }, ] [[package]] name = "starlette" -version = "1.0.0" +version = "1.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/81/69/17425771797c36cded50b7fe44e850315d039f28b15901ab44839e70b593/starlette-1.0.0.tar.gz", hash = "sha256:6a4beaf1f81bb472fd19ea9b918b50dc3a77a6f2e190a12954b25e6ed5eea149", size = 2655289, upload-time = "2026-03-22T18:29:46.779Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c5/bf/616a066c2760f6c2b1ae3437cc28149734d069fbb46511712beae118a68c/starlette-1.2.0.tar.gz", hash = "sha256:3c5a6b23fff42492914e93890bb80cbfea72dbf37de268eec06185d62a4ca553", size = 2668923, upload-time = "2026-05-28T11:42:50.568Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/c9/584bc9651441b4ba60cc4d557d8a547b5aff901af35bda3a4ee30c819b82/starlette-1.0.0-py3-none-any.whl", hash = "sha256:d3ec55e0bb321692d275455ddfd3df75fff145d009685eb40dc91fc66b03d38b", size = 72651, upload-time = "2026-03-22T18:29:45.111Z" }, + { url = "https://files.pythonhosted.org/packages/9f/85/492183764d5d01d4514be3730fdb8e228a80605783099551c51627578b5d/starlette-1.2.0-py3-none-any.whl", hash = "sha256:36e0c76ac59157e75dc4b3bdeafba97fb04eaf1878045f15dbef666a6f092ed7", size = 73213, upload-time = "2026-05-28T11:42:48.801Z" }, ] [[package]] @@ -1460,7 +1468,7 @@ wheels = [ [[package]] name = "torch" -version = "2.11.0" +version = "2.12.0" source = { registry = "https://download.pytorch.org/whl/cpu" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'darwin'", @@ -1476,15 +1484,15 @@ dependencies = [ { name = "typing-extensions", marker = "sys_platform == 'darwin'" }, ] wheels = [ - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:442ec9dc78592564fdad69cf0beaa9da2f82ab810ccb4f13903869a90bf3f15d", upload-time = "2026-03-23T15:17:02Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:cc3a195701bba2239c313ee311487f80f8aaebe9e89b9073dddbcf2f93b5a0ba", upload-time = "2026-03-23T15:17:06Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:072a0d6e4865e8b0dc0dbfe6ebed68fae235124222835ef03e5814d414d8c012", upload-time = "2026-03-23T15:17:10Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:23ec7789017da9d95b6d543d790814785e6f30905c5443efa8257d1490d73f79", upload-time = "2026-03-23T15:17:14Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:90dd587a5f61bfe1307148b581e2084fc5bc4a06e2b90a20e9a36b81087ff16b", upload-time = "2026-05-12T16:20:17Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:10ee1448a9f304d3b987eb4656f664ba6e4d7b410ca7a5a7c642199777a2cf88", upload-time = "2026-05-12T16:20:21Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:f7dfae4a519197dfa050e98d8e36378a0fb5899625a875c2b54445005a2e404e", upload-time = "2026-05-12T16:20:26Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:b4556715c8572758625d62b6e0ae3b1f76c440221913a6fb5e100f321fb4fb02", upload-time = "2026-05-12T16:20:31Z" }, ] [[package]] name = "torch" -version = "2.11.0+cpu" +version = "2.12.0+cpu" source = { registry = "https://download.pytorch.org/whl/cpu" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -1504,22 +1512,23 @@ dependencies = [ { name = "typing-extensions", marker = "sys_platform != 'darwin'" }, ] wheels = [ - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp313-cp313-linux_s390x.whl", hash = "sha256:d1eff25ccc454faf21c9666c81bfab8e405e87c12d300708d4559620bc191a36", upload-time = "2026-04-28T00:06:42Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:48b3e21a311445acdd0b27f13830e21d93adef70d4721e051e9f059baeb9b8f9", upload-time = "2026-04-28T00:06:51Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:45025d7752dbc6b4c784c03afaee9c5f19730ce084b2e43fc9a2fe1677d9ff86", upload-time = "2026-04-28T00:07:02Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:ed70d4a4fc9f8b826c02fa1a9800a83820fb2fa6ae607680b53390f9ef394d85", upload-time = "2026-04-28T00:07:12Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp313-cp313t-linux_s390x.whl", hash = "sha256:65d427a196ab0abe359b93c5bffedd76ded02df2b1b1d2d9f11a2609b69f426a", upload-time = "2026-04-28T00:07:19Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:8f13dc7075ae04ca5f876a9f40b4e47522a04c23e30824b4409f42a3f3e57aa4", upload-time = "2026-04-28T00:07:27Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:8713bb8679376ea0ec25742100b6cfb8447e0904c48bddefb9eb0ac1abbfa60a", upload-time = "2026-04-28T00:07:37Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:62ec1f1694c185f601eab74eb7fc0e8e10c64c06ae82f13c3592774c231c4877", upload-time = "2026-04-28T00:07:47Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp314-cp314-linux_s390x.whl", hash = "sha256:c9a14c367f470623b978e273a4e1915995b4ba7a0ae999178b06c273eea3536f", upload-time = "2026-04-28T00:07:54Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:71676f6a9a84bbd385e010198b51fa1c2324fb8f3c512a32d2c81af65f68f4c9", upload-time = "2026-04-28T00:08:02Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:f8481ea9088e4e5b81178a75aabdbb658bde8639bc1a15fd5d8f930abc966735", upload-time = "2026-04-28T00:08:11Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp314-cp314-win_amd64.whl", hash = "sha256:7575af4c9f7f7500ed62b1dafeb069aa0ba35b368a5f09793b3976b3d50f4fe4", upload-time = "2026-04-28T00:08:20Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp314-cp314t-linux_s390x.whl", hash = "sha256:825f1596878280a3a4c861441674888bc2d792e4ab7b045cb35feeab3f4f5dd7", upload-time = "2026-04-28T00:08:27Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c8a0bdfb2fd915b6c2cd27c856f63f729c366a4917772eba6b2b02aa3bce70d5", upload-time = "2026-04-28T00:08:36Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:768f22924a25cad2adeb9c6cbac5159e71067c8d4019b1511960d7435a5ca652", upload-time = "2026-04-28T00:08:47Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp314-cp314t-win_amd64.whl", hash = "sha256:6db45e7b2526d996fbf47c3d08737807a60a4e17996a6d91a97027fe260832c8", upload-time = "2026-04-28T00:08:57Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp313-cp313-linux_s390x.whl", hash = "sha256:5e0da19e1c3bfdc9b92638c552579eac678354485d61fc8921b0461fd6c40449", upload-time = "2026-05-12T23:17:05Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:68b7ddd4db4603a03e106e74c7098c8d8c8943d33c1e5ada009ca4cd885759c3", upload-time = "2026-05-12T23:17:12Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:ada78018bdfa30d1c766596cd32d910dbf5b03424cd859231b6d2a00533de922", upload-time = "2026-05-12T23:17:20Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:59bc266826e683899d49ee0af9829f3eafd0a16e15b5db9dc591c8d955003b66", upload-time = "2026-05-12T23:17:27Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp313-cp313-win_arm64.whl", hash = "sha256:97a5160abf3ca9d59a2cd7b4b4de89d9dfe290d36a1ac720262a55fbcee10b6c", upload-time = "2026-05-12T23:17:31Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp313-cp313t-linux_s390x.whl", hash = "sha256:32b9b7a0974cd6149cb98def0a28a49d92d7c14a384273d5539da9624239e950", upload-time = "2026-05-12T23:17:36Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:93ed8dc52c113580daf6124982b3232629045dccc5cd83a8f5ed478f7bac7340", upload-time = "2026-05-12T23:17:43Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:6a1c86abd4ed15a0736cf2663ad69642ae5d1288c99e30346070e6241018a0a9", upload-time = "2026-05-12T23:17:54Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:768dce4b7b3353795f667d1cb0dd7dfba06f570cd39539576097335e05bb71fe", upload-time = "2026-05-12T23:18:02Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp314-cp314-linux_s390x.whl", hash = "sha256:ee1f329acfd0c2a1ccaa3393bcaf9857ea58759549bb2d67e271a6eab42382b3", upload-time = "2026-05-12T23:18:08Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:797c066367792c92eb97cafba7fd0caa8d7455e6078a4ee880630077378dc372", upload-time = "2026-05-12T23:18:15Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:a8f419ce3f25388d36e67153ec63b3a1b17059c49f5a7759a7e91ac4843660d3", upload-time = "2026-05-12T23:18:22Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp314-cp314-win_amd64.whl", hash = "sha256:1dd196c43e74e7b3b526ff434e7efbdef3f3792a2efbecfc983d7dce501840d2", upload-time = "2026-05-12T23:18:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp314-cp314t-linux_s390x.whl", hash = "sha256:d0d2080cb13c94ebc0c884d237e404490743d0f40192c8a180abf3b6b6f334cf", upload-time = "2026-05-12T23:18:35Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:f7bc15972acad257723775237cdd120024cca844b8bc64701822fa596bcb7e14", upload-time = "2026-05-12T23:18:42Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:4d79f961250d1763487ecbc90af019a80009f9e87cadc5366b3ec4ba5671fea6", upload-time = "2026-05-12T23:18:50Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-cp314-cp314t-win_amd64.whl", hash = "sha256:46b8f4c41ac36bb5d5b47f5437b3de5541b313275e59c1d2aefd3bef32b0f531", upload-time = "2026-05-12T23:18:58Z" }, ] [[package]] @@ -1536,7 +1545,7 @@ wheels = [ [[package]] name = "transformers" -version = "5.6.2" +version = "5.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "huggingface-hub" }, @@ -1549,14 +1558,14 @@ dependencies = [ { name = "tqdm" }, { name = "typer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a4/e9/c6c80a07690142a7d05444271f47b9f3c8aac7dea01d52e1137ee480ad78/transformers-5.6.2.tar.gz", hash = "sha256:e657134c3e5a6bc00a3c35f4e2674bb51adfcd89898495b788a18552bac2b91a", size = 8311867, upload-time = "2026-04-23T18:33:29.332Z" } +sdist = { url = "https://files.pythonhosted.org/packages/51/58/7f843608f2e8421f86bb97060b54649be6239ec612b82bf9d41e65c26c00/transformers-5.9.0.tar.gz", hash = "sha256:25997cb8fa6053533171634b6162d7df54346530ec2aa9b42bb834e63668c842", size = 8642240, upload-time = "2026-05-20T14:50:49.278Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/95/0b0218149b0d6f14df35f5b8f676fa83df4f19ed253c3cc447107ef86eca/transformers-5.6.2-py3-none-any.whl", hash = "sha256:f8d3a1bb96778fed9b8aabfd0dd6e19843e4b0f2bb6b59f32b8a92051b0f348f", size = 10364898, upload-time = "2026-04-23T18:33:26.081Z" }, + { url = "https://files.pythonhosted.org/packages/02/ca/2eaa5359f2ccb8c2e1656bc26305ad0cf438aa392ce4b29ae67a315c186e/transformers-5.9.0-py3-none-any.whl", hash = "sha256:1d19509bcff7028ebc6b277d71caa712e8353778463d38764237d14b42b52788", size = 10787648, upload-time = "2026-05-20T14:50:45.337Z" }, ] [[package]] name = "typer" -version = "0.25.0" +version = "0.25.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, @@ -1564,9 +1573,9 @@ dependencies = [ { name = "rich" }, { name = "shellingham" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7b/27/ede8cec7596e0041ba7e7b80b47d132562f56ff454313a16f6084e555c9f/typer-0.25.0.tar.gz", hash = "sha256:123eaf9f19bb40fd268310e12a542c0c6b4fab9c98d9d23342a01ff95e3ce930", size = 120150, upload-time = "2026-04-26T08:46:14.767Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/51/9aed62104cea109b820bbd6c14245af756112017d309da813ef107d42e7e/typer-0.25.1.tar.gz", hash = "sha256:9616eb8853a09ffeabab1698952f33c6f29ffdbceb4eaeecf571880e8d7664cc", size = 122276, upload-time = "2026-04-30T19:32:16.964Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/72/193d4e586ec5a4db834a36bbeb47641a62f951f114ffd0fe5b1b46e8d56f/typer-0.25.0-py3-none-any.whl", hash = "sha256:ac01b48823d3db9a83c9e164338057eadbb1c9957a2a6b4eeb486669c560b5dc", size = 55993, upload-time = "2026-04-26T08:46:15.889Z" }, + { url = "https://files.pythonhosted.org/packages/3f/f9/2b3ff4e56e5fa7debfaf9eb135d0da96f3e9a1d5b27222223c7296336e5f/typer-0.25.1-py3-none-any.whl", hash = "sha256:75caa44ed46a03fb2dab8808753ffacdbfea88495e74c85a28c5eefcf5f39c89", size = 58409, upload-time = "2026-04-30T19:32:18.271Z" }, ] [[package]] @@ -1610,15 +1619,15 @@ wheels = [ [[package]] name = "uvicorn" -version = "0.46.0" +version = "0.48.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "h11" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1f/93/041fca8274050e40e6791f267d82e0e2e27dd165627bd640d3e0e378d877/uvicorn-0.46.0.tar.gz", hash = "sha256:fb9da0926999cc6cb22dc7cd71a94a632f078e6ae47ff683c5c420750fb7413d", size = 88758, upload-time = "2026-04-23T07:16:00.151Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e6/bf/f6544ba992ddb9a6077343a576f9844f7f8f06ab819aefd00206e9255f18/uvicorn-0.48.0.tar.gz", hash = "sha256:a5504207195d08c2511bf9125ede5ac4a4b71725d519e758d01dcf0bc2d31c37", size = 91074, upload-time = "2026-05-24T12:08:41.925Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/31/a3/5b1562db76a5a488274b2332a97199b32d0442aca0ed193697fd47786316/uvicorn-0.46.0-py3-none-any.whl", hash = "sha256:bbebbcbed972d162afca128605223022bedd345b7bc7855ce66deb31487a9048", size = 70926, upload-time = "2026-04-23T07:15:58.355Z" }, + { url = "https://files.pythonhosted.org/packages/01/be/72532be3da7acc5fdfbccdb95215cd04f995a0886532a5b423f929cda4cc/uvicorn-0.48.0-py3-none-any.whl", hash = "sha256:48097851328b87ec36117d3d575234519eb58c2b22d79666e9bbc6c49a761dad", size = 71410, upload-time = "2026-05-24T12:08:40.258Z" }, ] [package.optional-dependencies] @@ -1660,7 +1669,7 @@ wheels = [ [[package]] name = "virtualenv" -version = "21.2.4" +version = "21.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "distlib" }, @@ -1668,66 +1677,81 @@ dependencies = [ { name = "platformdirs" }, { name = "python-discovery" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0c/98/3a7e644e19cb26133488caff231be390579860bbbb3da35913c49a1d0a46/virtualenv-21.2.4.tar.gz", hash = "sha256:b294ef68192638004d72524ce7ef303e9d0cf5a44c95ce2e54a7500a6381cada", size = 5850742, upload-time = "2026-04-14T22:15:31.438Z" } +sdist = { url = "https://files.pythonhosted.org/packages/95/f0/b47ecf438211a25a97f8f0e4b23c22bc2496ebfea18dd6ec16210f09cc36/virtualenv-21.4.1.tar.gz", hash = "sha256:2ca543c713b72840ceffd94e9bdedfbd09a661defa1f7f69e5429ad4059442e2", size = 7613344, upload-time = "2026-05-28T04:12:49.905Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/27/8d/edd0bd910ff803c308ee9a6b7778621af0d10252219ad9f19ef4d4982a61/virtualenv-21.2.4-py3-none-any.whl", hash = "sha256:29d21e941795206138d0f22f4e45ff7050e5da6c6472299fb7103318763861ac", size = 5831232, upload-time = "2026-04-14T22:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/ff/dc/ac4f3a987a87e1a18556896f257c4e15c95ed157b7975347ec6b313b75ce/virtualenv-21.4.1-py3-none-any.whl", hash = "sha256:caf4ff72d1b4039057f41d8e8466e859513d67c0400d9c6b62c02c9d1ebc3e12", size = 7594078, upload-time = "2026-05-28T04:12:47.686Z" }, ] [[package]] name = "watchfiles" -version = "1.1.1" +version = "1.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c2/c9/8869df9b2a2d6c59d79220a4db37679e74f807c559ffe5265e08b227a210/watchfiles-1.1.1.tar.gz", hash = "sha256:a173cb5c16c4f40ab19cecf48a534c409f7ea983ab8fed0741304a1c0a31b3f2", size = 94440, upload-time = "2025-10-14T15:06:21.08Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/41/5e1a4bb12aac5f1493fa1bdc11154eca3b258ca4eba65d39c473fe19d8e9/watchfiles-1.2.0.tar.gz", hash = "sha256:c995fba777f1ea992f090f9236e9284cf7a5d1a0130dd5a3d82c598cacd76838", size = 108252, upload-time = "2026-05-18T04:32:04.251Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/f4/f750b29225fe77139f7ae5de89d4949f5a99f934c65a1f1c0b248f26f747/watchfiles-1.1.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:130e4876309e8686a5e37dba7d5e9bc77e6ed908266996ca26572437a5271e18", size = 404321, upload-time = "2025-10-14T15:05:02.063Z" }, - { url = "https://files.pythonhosted.org/packages/2b/f9/f07a295cde762644aa4c4bb0f88921d2d141af45e735b965fb2e87858328/watchfiles-1.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5f3bde70f157f84ece3765b42b4a52c6ac1a50334903c6eaf765362f6ccca88a", size = 391783, upload-time = "2025-10-14T15:05:03.052Z" }, - { url = "https://files.pythonhosted.org/packages/bc/11/fc2502457e0bea39a5c958d86d2cb69e407a4d00b85735ca724bfa6e0d1a/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14e0b1fe858430fc0251737ef3824c54027bedb8c37c38114488b8e131cf8219", size = 449279, upload-time = "2025-10-14T15:05:04.004Z" }, - { url = "https://files.pythonhosted.org/packages/e3/1f/d66bc15ea0b728df3ed96a539c777acfcad0eb78555ad9efcaa1274688f0/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f27db948078f3823a6bb3b465180db8ebecf26dd5dae6f6180bd87383b6b4428", size = 459405, upload-time = "2025-10-14T15:05:04.942Z" }, - { url = "https://files.pythonhosted.org/packages/be/90/9f4a65c0aec3ccf032703e6db02d89a157462fbb2cf20dd415128251cac0/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:059098c3a429f62fc98e8ec62b982230ef2c8df68c79e826e37b895bc359a9c0", size = 488976, upload-time = "2025-10-14T15:05:05.905Z" }, - { url = "https://files.pythonhosted.org/packages/37/57/ee347af605d867f712be7029bb94c8c071732a4b44792e3176fa3c612d39/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfb5862016acc9b869bb57284e6cb35fdf8e22fe59f7548858e2f971d045f150", size = 595506, upload-time = "2025-10-14T15:05:06.906Z" }, - { url = "https://files.pythonhosted.org/packages/a8/78/cc5ab0b86c122047f75e8fc471c67a04dee395daf847d3e59381996c8707/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:319b27255aacd9923b8a276bb14d21a5f7ff82564c744235fc5eae58d95422ae", size = 474936, upload-time = "2025-10-14T15:05:07.906Z" }, - { url = "https://files.pythonhosted.org/packages/62/da/def65b170a3815af7bd40a3e7010bf6ab53089ef1b75d05dd5385b87cf08/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c755367e51db90e75b19454b680903631d41f9e3607fbd941d296a020c2d752d", size = 456147, upload-time = "2025-10-14T15:05:09.138Z" }, - { url = "https://files.pythonhosted.org/packages/57/99/da6573ba71166e82d288d4df0839128004c67d2778d3b566c138695f5c0b/watchfiles-1.1.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:c22c776292a23bfc7237a98f791b9ad3144b02116ff10d820829ce62dff46d0b", size = 630007, upload-time = "2025-10-14T15:05:10.117Z" }, - { url = "https://files.pythonhosted.org/packages/a8/51/7439c4dd39511368849eb1e53279cd3454b4a4dbace80bab88feeb83c6b5/watchfiles-1.1.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:3a476189be23c3686bc2f4321dd501cb329c0a0469e77b7b534ee10129ae6374", size = 622280, upload-time = "2025-10-14T15:05:11.146Z" }, - { url = "https://files.pythonhosted.org/packages/95/9c/8ed97d4bba5db6fdcdb2b298d3898f2dd5c20f6b73aee04eabe56c59677e/watchfiles-1.1.1-cp313-cp313-win32.whl", hash = "sha256:bf0a91bfb5574a2f7fc223cf95eeea79abfefa404bf1ea5e339c0c1560ae99a0", size = 272056, upload-time = "2025-10-14T15:05:12.156Z" }, - { url = "https://files.pythonhosted.org/packages/1f/f3/c14e28429f744a260d8ceae18bf58c1d5fa56b50d006a7a9f80e1882cb0d/watchfiles-1.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:52e06553899e11e8074503c8e716d574adeeb7e68913115c4b3653c53f9bae42", size = 288162, upload-time = "2025-10-14T15:05:13.208Z" }, - { url = "https://files.pythonhosted.org/packages/dc/61/fe0e56c40d5cd29523e398d31153218718c5786b5e636d9ae8ae79453d27/watchfiles-1.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:ac3cc5759570cd02662b15fbcd9d917f7ecd47efe0d6b40474eafd246f91ea18", size = 277909, upload-time = "2025-10-14T15:05:14.49Z" }, - { url = "https://files.pythonhosted.org/packages/79/42/e0a7d749626f1e28c7108a99fb9bf524b501bbbeb9b261ceecde644d5a07/watchfiles-1.1.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:563b116874a9a7ce6f96f87cd0b94f7faf92d08d0021e837796f0a14318ef8da", size = 403389, upload-time = "2025-10-14T15:05:15.777Z" }, - { url = "https://files.pythonhosted.org/packages/15/49/08732f90ce0fbbc13913f9f215c689cfc9ced345fb1bcd8829a50007cc8d/watchfiles-1.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3ad9fe1dae4ab4212d8c91e80b832425e24f421703b5a42ef2e4a1e215aff051", size = 389964, upload-time = "2025-10-14T15:05:16.85Z" }, - { url = "https://files.pythonhosted.org/packages/27/0d/7c315d4bd5f2538910491a0393c56bf70d333d51bc5b34bee8e68e8cea19/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce70f96a46b894b36eba678f153f052967a0d06d5b5a19b336ab0dbbd029f73e", size = 448114, upload-time = "2025-10-14T15:05:17.876Z" }, - { url = "https://files.pythonhosted.org/packages/c3/24/9e096de47a4d11bc4df41e9d1e61776393eac4cb6eb11b3e23315b78b2cc/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cb467c999c2eff23a6417e58d75e5828716f42ed8289fe6b77a7e5a91036ca70", size = 460264, upload-time = "2025-10-14T15:05:18.962Z" }, - { url = "https://files.pythonhosted.org/packages/cc/0f/e8dea6375f1d3ba5fcb0b3583e2b493e77379834c74fd5a22d66d85d6540/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:836398932192dae4146c8f6f737d74baeac8b70ce14831a239bdb1ca882fc261", size = 487877, upload-time = "2025-10-14T15:05:20.094Z" }, - { url = "https://files.pythonhosted.org/packages/ac/5b/df24cfc6424a12deb41503b64d42fbea6b8cb357ec62ca84a5a3476f654a/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:743185e7372b7bc7c389e1badcc606931a827112fbbd37f14c537320fca08620", size = 595176, upload-time = "2025-10-14T15:05:21.134Z" }, - { url = "https://files.pythonhosted.org/packages/8f/b5/853b6757f7347de4e9b37e8cc3289283fb983cba1ab4d2d7144694871d9c/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:afaeff7696e0ad9f02cbb8f56365ff4686ab205fcf9c4c5b6fdfaaa16549dd04", size = 473577, upload-time = "2025-10-14T15:05:22.306Z" }, - { url = "https://files.pythonhosted.org/packages/e1/f7/0a4467be0a56e80447c8529c9fce5b38eab4f513cb3d9bf82e7392a5696b/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f7eb7da0eb23aa2ba036d4f616d46906013a68caf61b7fdbe42fc8b25132e77", size = 455425, upload-time = "2025-10-14T15:05:23.348Z" }, - { url = "https://files.pythonhosted.org/packages/8e/e0/82583485ea00137ddf69bc84a2db88bd92ab4a6e3c405e5fb878ead8d0e7/watchfiles-1.1.1-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:831a62658609f0e5c64178211c942ace999517f5770fe9436be4c2faeba0c0ef", size = 628826, upload-time = "2025-10-14T15:05:24.398Z" }, - { url = "https://files.pythonhosted.org/packages/28/9a/a785356fccf9fae84c0cc90570f11702ae9571036fb25932f1242c82191c/watchfiles-1.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:f9a2ae5c91cecc9edd47e041a930490c31c3afb1f5e6d71de3dc671bfaca02bf", size = 622208, upload-time = "2025-10-14T15:05:25.45Z" }, - { url = "https://files.pythonhosted.org/packages/c3/f4/0872229324ef69b2c3edec35e84bd57a1289e7d3fe74588048ed8947a323/watchfiles-1.1.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:d1715143123baeeaeadec0528bb7441103979a1d5f6fd0e1f915383fea7ea6d5", size = 404315, upload-time = "2025-10-14T15:05:26.501Z" }, - { url = "https://files.pythonhosted.org/packages/7b/22/16d5331eaed1cb107b873f6ae1b69e9ced582fcf0c59a50cd84f403b1c32/watchfiles-1.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:39574d6370c4579d7f5d0ad940ce5b20db0e4117444e39b6d8f99db5676c52fd", size = 390869, upload-time = "2025-10-14T15:05:27.649Z" }, - { url = "https://files.pythonhosted.org/packages/b2/7e/5643bfff5acb6539b18483128fdc0ef2cccc94a5b8fbda130c823e8ed636/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7365b92c2e69ee952902e8f70f3ba6360d0d596d9299d55d7d386df84b6941fb", size = 449919, upload-time = "2025-10-14T15:05:28.701Z" }, - { url = "https://files.pythonhosted.org/packages/51/2e/c410993ba5025a9f9357c376f48976ef0e1b1aefb73b97a5ae01a5972755/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bfff9740c69c0e4ed32416f013f3c45e2ae42ccedd1167ef2d805c000b6c71a5", size = 460845, upload-time = "2025-10-14T15:05:30.064Z" }, - { url = "https://files.pythonhosted.org/packages/8e/a4/2df3b404469122e8680f0fcd06079317e48db58a2da2950fb45020947734/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b27cf2eb1dda37b2089e3907d8ea92922b673c0c427886d4edc6b94d8dfe5db3", size = 489027, upload-time = "2025-10-14T15:05:31.064Z" }, - { url = "https://files.pythonhosted.org/packages/ea/84/4587ba5b1f267167ee715b7f66e6382cca6938e0a4b870adad93e44747e6/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:526e86aced14a65a5b0ec50827c745597c782ff46b571dbfe46192ab9e0b3c33", size = 595615, upload-time = "2025-10-14T15:05:32.074Z" }, - { url = "https://files.pythonhosted.org/packages/6a/0f/c6988c91d06e93cd0bb3d4a808bcf32375ca1904609835c3031799e3ecae/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04e78dd0b6352db95507fd8cb46f39d185cf8c74e4cf1e4fbad1d3df96faf510", size = 474836, upload-time = "2025-10-14T15:05:33.209Z" }, - { url = "https://files.pythonhosted.org/packages/b4/36/ded8aebea91919485b7bbabbd14f5f359326cb5ec218cd67074d1e426d74/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c85794a4cfa094714fb9c08d4a218375b2b95b8ed1666e8677c349906246c05", size = 455099, upload-time = "2025-10-14T15:05:34.189Z" }, - { url = "https://files.pythonhosted.org/packages/98/e0/8c9bdba88af756a2fce230dd365fab2baf927ba42cd47521ee7498fd5211/watchfiles-1.1.1-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:74d5012b7630714b66be7b7b7a78855ef7ad58e8650c73afc4c076a1f480a8d6", size = 630626, upload-time = "2025-10-14T15:05:35.216Z" }, - { url = "https://files.pythonhosted.org/packages/2a/84/a95db05354bf2d19e438520d92a8ca475e578c647f78f53197f5a2f17aaf/watchfiles-1.1.1-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:8fbe85cb3201c7d380d3d0b90e63d520f15d6afe217165d7f98c9c649654db81", size = 622519, upload-time = "2025-10-14T15:05:36.259Z" }, - { url = "https://files.pythonhosted.org/packages/1d/ce/d8acdc8de545de995c339be67711e474c77d643555a9bb74a9334252bd55/watchfiles-1.1.1-cp314-cp314-win32.whl", hash = "sha256:3fa0b59c92278b5a7800d3ee7733da9d096d4aabcfabb9a928918bd276ef9b9b", size = 272078, upload-time = "2025-10-14T15:05:37.63Z" }, - { url = "https://files.pythonhosted.org/packages/c4/c9/a74487f72d0451524be827e8edec251da0cc1fcf111646a511ae752e1a3d/watchfiles-1.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:c2047d0b6cea13b3316bdbafbfa0c4228ae593d995030fda39089d36e64fc03a", size = 287664, upload-time = "2025-10-14T15:05:38.95Z" }, - { url = "https://files.pythonhosted.org/packages/df/b8/8ac000702cdd496cdce998c6f4ee0ca1f15977bba51bdf07d872ebdfc34c/watchfiles-1.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:842178b126593addc05acf6fce960d28bc5fae7afbaa2c6c1b3a7b9460e5be02", size = 277154, upload-time = "2025-10-14T15:05:39.954Z" }, - { url = "https://files.pythonhosted.org/packages/47/a8/e3af2184707c29f0f14b1963c0aace6529f9d1b8582d5b99f31bbf42f59e/watchfiles-1.1.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:88863fbbc1a7312972f1c511f202eb30866370ebb8493aef2812b9ff28156a21", size = 403820, upload-time = "2025-10-14T15:05:40.932Z" }, - { url = "https://files.pythonhosted.org/packages/c0/ec/e47e307c2f4bd75f9f9e8afbe3876679b18e1bcec449beca132a1c5ffb2d/watchfiles-1.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:55c7475190662e202c08c6c0f4d9e345a29367438cf8e8037f3155e10a88d5a5", size = 390510, upload-time = "2025-10-14T15:05:41.945Z" }, - { url = "https://files.pythonhosted.org/packages/d5/a0/ad235642118090f66e7b2f18fd5c42082418404a79205cdfca50b6309c13/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f53fa183d53a1d7a8852277c92b967ae99c2d4dcee2bfacff8868e6e30b15f7", size = 448408, upload-time = "2025-10-14T15:05:43.385Z" }, - { url = "https://files.pythonhosted.org/packages/df/85/97fa10fd5ff3332ae17e7e40e20784e419e28521549780869f1413742e9d/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6aae418a8b323732fa89721d86f39ec8f092fc2af67f4217a2b07fd3e93c6101", size = 458968, upload-time = "2025-10-14T15:05:44.404Z" }, - { url = "https://files.pythonhosted.org/packages/47/c2/9059c2e8966ea5ce678166617a7f75ecba6164375f3b288e50a40dc6d489/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f096076119da54a6080e8920cbdaac3dbee667eb91dcc5e5b78840b87415bd44", size = 488096, upload-time = "2025-10-14T15:05:45.398Z" }, - { url = "https://files.pythonhosted.org/packages/94/44/d90a9ec8ac309bc26db808a13e7bfc0e4e78b6fc051078a554e132e80160/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00485f441d183717038ed2e887a7c868154f216877653121068107b227a2f64c", size = 596040, upload-time = "2025-10-14T15:05:46.502Z" }, - { url = "https://files.pythonhosted.org/packages/95/68/4e3479b20ca305cfc561db3ed207a8a1c745ee32bf24f2026a129d0ddb6e/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a55f3e9e493158d7bfdb60a1165035f1cf7d320914e7b7ea83fe22c6023b58fc", size = 473847, upload-time = "2025-10-14T15:05:47.484Z" }, - { url = "https://files.pythonhosted.org/packages/4f/55/2af26693fd15165c4ff7857e38330e1b61ab8c37d15dc79118cdba115b7a/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c91ed27800188c2ae96d16e3149f199d62f86c7af5f5f4d2c61a3ed8cd3666c", size = 455072, upload-time = "2025-10-14T15:05:48.928Z" }, - { url = "https://files.pythonhosted.org/packages/66/1d/d0d200b10c9311ec25d2273f8aad8c3ef7cc7ea11808022501811208a750/watchfiles-1.1.1-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:311ff15a0bae3714ffb603e6ba6dbfba4065ab60865d15a6ec544133bdb21099", size = 629104, upload-time = "2025-10-14T15:05:49.908Z" }, - { url = "https://files.pythonhosted.org/packages/e3/bd/fa9bb053192491b3867ba07d2343d9f2252e00811567d30ae8d0f78136fe/watchfiles-1.1.1-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a916a2932da8f8ab582f242c065f5c81bed3462849ca79ee357dd9551b0e9b01", size = 622112, upload-time = "2025-10-14T15:05:50.941Z" }, + { url = "https://files.pythonhosted.org/packages/d1/4d/70a7feced9f87e2ff26dba42667290f41694fc64646c67261fbb8cab5d5c/watchfiles-1.2.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:01ea8d66f0693b9b60a6541c8d10263091ca9a9060d242f3c1f3143f9aad2c98", size = 399730, upload-time = "2026-05-18T04:31:38.162Z" }, + { url = "https://files.pythonhosted.org/packages/31/3a/0da302f2307aee316922806ebd5726c542cbd787c938271cf14a074c7daf/watchfiles-1.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7ba0480b9a74af058f43b337e937a451e109295c420916d68ad24e3dc02f5e44", size = 392842, upload-time = "2026-05-18T04:30:27.051Z" }, + { url = "https://files.pythonhosted.org/packages/db/ef/d5bdb705c224dbc256aa0c1ec47bf4e61ec52558f2afb44a71a1fe4d7015/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f34e26a19f91f710c08e0183429f0d1d15df734e6bc78c31e77b9ea9c433658", size = 452989, upload-time = "2026-05-18T04:31:11.945Z" }, + { url = "https://files.pythonhosted.org/packages/71/29/5495f2c1661949ef7a35e4d71111d129cfe7606414a26887a919d0a55406/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b4e77f6a55f858504069abd35d336a637555c09bca453dde1ee1e5ada8a6a1fb", size = 458978, upload-time = "2026-05-18T04:30:52.606Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8c/7f9c07c433811c2fffd93e13fdfb7135de9aab5f2ae41be08960fa0047dc/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0cb4d80e212f116474a545c21c912b445f16bb0cef9e6a73a498164223e14e2f", size = 490248, upload-time = "2026-05-18T04:31:36.003Z" }, + { url = "https://files.pythonhosted.org/packages/3c/11/d93632febc52fbc21be90231bb7c17fd5387f46c9076fd40a5f9c2ae6910/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b974946a10af379d425e2eef5b62f5c6ebeaccf91d45eaad6f5b27ecd4f91aa0", size = 571847, upload-time = "2026-05-18T04:31:10.862Z" }, + { url = "https://files.pythonhosted.org/packages/55/b4/383173e73aabb07ad1d9c7aa859d95437ac46a6d6a1e11005facda0c9d19/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86bc13c25a8d1fcd70b51d0ce7c9b65e90de5666fcbfd3e34957cc73ee19aeb5", size = 465974, upload-time = "2026-05-18T04:30:17.006Z" }, + { url = "https://files.pythonhosted.org/packages/a7/6c/89b1a230a78f57c52dd8893adb1f92f94411721b6ec12596c56d98c74356/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca148d73dea36c9763aaa351e4d7a51780ec1584217c45276f4fe8239c768b71", size = 454782, upload-time = "2026-05-18T04:30:35.656Z" }, + { url = "https://files.pythonhosted.org/packages/24/62/1732118367cfff0a9fce3bf62ff4bfded09ef5df21d9d446b858b3f70a96/watchfiles-1.2.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:c525543d91961c6955b2636b308569e84a1d1c5f5f2932041ab9ef46422f43e3", size = 465182, upload-time = "2026-05-18T04:30:20.846Z" }, + { url = "https://files.pythonhosted.org/packages/28/96/716f7e5f51339bf22963f3345f9f27d7f3b30e2eadc597e257c881dd3c53/watchfiles-1.2.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:a204794696ffb8f9b10fba6f7cb5216d42f3b2b71860ccac6b6e42f5f10973b0", size = 629841, upload-time = "2026-05-18T04:31:05.397Z" }, + { url = "https://files.pythonhosted.org/packages/4c/fe/c40783950fd771ccf66ab3ec2722d188a9af1c7f96c6e811f36e40c6e03f/watchfiles-1.2.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:10d86db20695afe7997ac9e1717637d6714a8d0220458c33f3d2061f54cec427", size = 658028, upload-time = "2026-05-18T04:31:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/71/72/4508db1856d1d87fcbb3b63f4839bab1b5682cb0e8d224d122263c09654a/watchfiles-1.2.0-cp313-cp313-win32.whl", hash = "sha256:eb283ee99e21ad6443c8cdb06ac5b34b1308c329cbdf03fa02b445363714c799", size = 275183, upload-time = "2026-05-18T04:30:59.57Z" }, + { url = "https://files.pythonhosted.org/packages/f9/36/14b76ca57652e5cc5fd1c11f32a261292c08a0d19a00351013c2549cbfb2/watchfiles-1.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:a0f27f01bee51861392bb6b7c4fdb290b27d1eb194e9e28788d68102a0e898d9", size = 288059, upload-time = "2026-05-18T04:32:07.937Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8d/0a85e395398d8d20fadfe5c5d32c726eee17a519e78fb356f2cf7531bffe/watchfiles-1.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:3651aa7058595e9cfb75d35dd5ada2bf9f48a5b8a0f3562821d3e210c507e077", size = 280186, upload-time = "2026-05-18T04:31:54.484Z" }, + { url = "https://files.pythonhosted.org/packages/37/68/36db056f1fdcc5f07302f56e631774d6835bcd6fa3ace402304621d5f9e5/watchfiles-1.2.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:faea288b6f0ab1902ef08f4ca6de005dccf856c4e0c4f21b8c5fce02d90a1b08", size = 399031, upload-time = "2026-05-18T04:30:44.576Z" }, + { url = "https://files.pythonhosted.org/packages/c1/64/01a9d6f66a82a5c101ce939274106cc72759d62427e153f01edd2b9f87c2/watchfiles-1.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:01859b11fd9fbca670f4d5da00fbac282cfea9bd67a2125d8b2833a3b5617ea9", size = 391205, upload-time = "2026-05-18T04:30:25.413Z" }, + { url = "https://files.pythonhosted.org/packages/84/2c/0a44fe058cb4bb7b8ede6b6670698bbb7c0400740e378d00022189b7b31d/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fff610d7bb2256a317bb1e96f0d7862c7aa8076733ee5df0fd41bbe76a24a4f4", size = 451892, upload-time = "2026-05-18T04:32:14.005Z" }, + { url = "https://files.pythonhosted.org/packages/67/a1/351e0d56cd35e6488b5c8b4fb11a809a5bc923e8fe8fed9faf8920be0c89/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b141a4891c995a039cd89e9a49e62df1dc8a559a5d1a6e4c7106d16c12777a55", size = 458867, upload-time = "2026-05-18T04:31:22.279Z" }, + { url = "https://files.pythonhosted.org/packages/d5/7d/9d09605187f1b838998624049fcf8bf47b73c1a3b76901fcac1782f62277/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f22943b7770483f6ea0721c6b11d022947a98eb0acae14694de034f4d0d38925", size = 490217, upload-time = "2026-05-18T04:31:43.657Z" }, + { url = "https://files.pythonhosted.org/packages/60/5d/a17a16eccb182f04188cd308ec24b1a71a9b5c4e7098269cf35d9fa56d02/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1bc6195825b7dcd217968bb1f801a60fd4c16e8eeab5bedc7fe917d7d5995ab4", size = 571458, upload-time = "2026-05-18T04:32:11.875Z" }, + { url = "https://files.pythonhosted.org/packages/d3/3d/4dd457062083ab1938e5dfd45032eb425cee2ac817287ca8ff4356183e5d/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4a4b147f5dca2a5d325a06a832fb43f345751adfbc63204aec30e0d9ca965a2", size = 464707, upload-time = "2026-05-18T04:30:43.492Z" }, + { url = "https://files.pythonhosted.org/packages/c6/71/ea8c57b128f5383de74d0c7d2d9c57ad7c9a65a930c451bd25d524b295b7/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4543579a9bdb0c9560039b4ffddbdb39545707659fbc430ce4c10f3f68d557f9", size = 454663, upload-time = "2026-05-18T04:30:16.061Z" }, + { url = "https://files.pythonhosted.org/packages/53/fd/2e812bf938406d7db351f0703ddd3fc6c061cf30d96153a77bc79a943a44/watchfiles-1.2.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:20aa0e708b920bde876a4aa82dc7dd6ebea228a63a67cda6632c2fc87b787efa", size = 463537, upload-time = "2026-05-18T04:31:44.9Z" }, + { url = "https://files.pythonhosted.org/packages/86/56/d17a7f1dd1bc3035f1072694a551301272f1739c2d8e319c927cb9e29b38/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:d413349d565dab74297f2a63e84a097936be69bf8f3b3801f27f380e32040f44", size = 629194, upload-time = "2026-05-18T04:31:14.141Z" }, + { url = "https://files.pythonhosted.org/packages/be/06/f1ff66bf5cae50aa4062779a0ecd0bbaf15e466195719074078947d9a17d/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:f28b2725eb8cce327b9b3ab02415c853011dc55c95832fe90de6bc56f5315f72", size = 656194, upload-time = "2026-05-18T04:31:47.14Z" }, + { url = "https://files.pythonhosted.org/packages/e7/54/a9c7ea9a82a4ac65e7004c0a03920b5cdd2f9c3b678757d9cd425aa51d53/watchfiles-1.2.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b8c8358484d5fa12ef34f05b7f4168eaf1932f408725ff6d023c33ec17bd79d4", size = 400205, upload-time = "2026-05-18T04:32:05.153Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5d/c9ab3534374a4a67450696905d6ef16a04405448b8dc52bd752ae50423d4/watchfiles-1.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9f04b092229ad2c50126dd3c922c8822e51e605993764a33058d4a791ab42281", size = 392508, upload-time = "2026-05-18T04:30:54.849Z" }, + { url = "https://files.pythonhosted.org/packages/26/ca/1ad30103535cf0cecd7b993e8d50edc5351b1820e38f2d22e3df58962feb/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a7ce236284f002a156f70add88efe5c70879cccbb658be0822c54b1306fc09d", size = 452448, upload-time = "2026-05-18T04:30:53.727Z" }, + { url = "https://files.pythonhosted.org/packages/37/a1/ceee2cdf2afbd715fa07758d39c9859513eae411b23196f7fd039e5feedd/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b9909cc2b48468b575eefa944919e1fe8a36c5849d5c7c168f80a8c1db69398e", size = 459605, upload-time = "2026-05-18T04:30:23.312Z" }, + { url = "https://files.pythonhosted.org/packages/e8/f6/421e30fd1cb3907a84ed92ab3f1983e37ba2dca015e9a894a048418417a2/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a37faaed405c67e28e6be45a1fa4f206ef5a2860f27c237db9fa30704c38242", size = 490757, upload-time = "2026-05-18T04:30:47.358Z" }, + { url = "https://files.pythonhosted.org/packages/41/b0/55ed1b97ed08be7bba6f9a541cac15f2a858e1d74d2b07b6da70a82aab00/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9649193aa27bd9ff2e80ff29bfaa93085496c7a3a377592823cc58b77ee88add", size = 568672, upload-time = "2026-05-18T04:30:38.915Z" }, + { url = "https://files.pythonhosted.org/packages/d1/cf/d8ae8a80dd7bafab395ea7681c10237311bbf34d37704a8c744e7cf31fc7/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e4ff8e37f99cf1da89e255e07c9c4b37c214038c4283707bdec308cb1b0ea1f", size = 464197, upload-time = "2026-05-18T04:30:09.914Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8a/3076c496ca8dafe0e8cd03fcebdfc47be4b1174b4e5b24ff6e396e6b3af2/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:054dc20fd2e3132b4c3883b4a00d72fd6e1f56fdaf89fccd12e8057d74cd74d7", size = 453181, upload-time = "2026-05-18T04:30:14.829Z" }, + { url = "https://files.pythonhosted.org/packages/e5/10/9745e17c98e7b8a86454df0a3c7b5686bd650383f1e9f26e4ebcbd6cc0c0/watchfiles-1.2.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e140ed30ebde76796b686e67c182cff10ea2fbab186fafd1560f74bb5a473a6e", size = 465109, upload-time = "2026-05-18T04:30:28.123Z" }, + { url = "https://files.pythonhosted.org/packages/8f/95/8ef4a95481d3e0cb52d62a06fa6e972e81424be2d9698b91a2fecca9904c/watchfiles-1.2.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:bb7e52ecf68ba46d22df23467b87cffeb2146908aa523ebfe803019618cfda06", size = 630653, upload-time = "2026-05-18T04:31:49.304Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e4/3b3bf36b0f829b50c6ebcb8d031583863c59f923d6a6af3d485e470d0fac/watchfiles-1.2.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:23282a321c8baf9b3a3c4afff673f9fe65eb7fdc2338d765ccad9d3d1916a5ba", size = 657838, upload-time = "2026-05-18T04:31:06.497Z" }, + { url = "https://files.pythonhosted.org/packages/21/b1/6cbbb50c1f3002ab568777d44aa21206dfb8807a840990c4037523b51812/watchfiles-1.2.0-cp314-cp314-win32.whl", hash = "sha256:c0db965c5f79aa49fe672d297cf1febc5ad149b658594944f49a54a2b96270a7", size = 275108, upload-time = "2026-05-18T04:30:06.891Z" }, + { url = "https://files.pythonhosted.org/packages/92/45/190ce6db8dcb4536682cf75d3889ff1a27182a58cb519d343cb6d9ea63d8/watchfiles-1.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:71283b39fd17e5408eb123bd37aeecfd9d54c81fc184421943208aadb879d103", size = 288441, upload-time = "2026-05-18T04:32:12.901Z" }, + { url = "https://files.pythonhosted.org/packages/74/0d/3eae1c2313ab08378431d907c3f8095ecca00f3eda33111cf4f0f2591799/watchfiles-1.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:c5c19526f4e54a00f2666a6c0e9e40d582c09e865055ea7378bf0009aab857b3", size = 280684, upload-time = "2026-05-18T04:31:26.902Z" }, + { url = "https://files.pythonhosted.org/packages/b1/75/fb64e6c25d6b5ca636d03df34ffb1c6e9873303e76d27967e045f8df088f/watchfiles-1.2.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:d73a585accffa5ae39c17264c36ec3166d2fad7000c780f5ef83b2722afb9dd2", size = 398857, upload-time = "2026-05-18T04:32:17.108Z" }, + { url = "https://files.pythonhosted.org/packages/73/4e/9f7adf01754cbf81843722ccfec169d8f26c69778281a302855cecd2ee08/watchfiles-1.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ae99b14c5f21e026e0e9d96f40e07d8570ebee6cafd9d8fc318354606daa7a28", size = 392413, upload-time = "2026-05-18T04:31:07.911Z" }, + { url = "https://files.pythonhosted.org/packages/47/c8/bec626bcc2d69f44b9acb24ce7d60ed7b16b73628eea747fcbd169d8edda/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4429f3b105524a10b72c3a819b091c495d2811d419c1e1e8df773a5a5974f831", size = 452409, upload-time = "2026-05-18T04:31:20.142Z" }, + { url = "https://files.pythonhosted.org/packages/00/b7/b6362068e81e7c556d155a34c35d40ac3ef42d747b06d7f6e5bf58e359c2/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:43d818978d06062d9b22c4fab2ebe44cf5213d42dc8e62bda8c2760cfa2eeb33", size = 458827, upload-time = "2026-05-18T04:32:06.219Z" }, + { url = "https://files.pythonhosted.org/packages/67/f8/9a813fa42afb1e0b4625e75f0479826644d3ee8dc287e093799bc01f390c/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b9f732dc58b2dbe69e464ccf8fff7a03b0dd0be439da4c0720d3558527d3d6b4", size = 490104, upload-time = "2026-05-18T04:31:56.034Z" }, + { url = "https://files.pythonhosted.org/packages/2f/bf/27dfb6094ca4c9aad21298b5525b6c53cb36121ee454331d05161e58d130/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f200104103feb097de4cab8fe4f5dd18a2026934c7dea98c55a2f5fd6d5a33b", size = 571360, upload-time = "2026-05-18T04:31:57.133Z" }, + { url = "https://files.pythonhosted.org/packages/fb/39/44a096d67270ea93df91d33877dbe91fbda3aa4f8ec2edf799d93eda8736/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:63ac26eefbf4af1741247d6fb68b11c49a25b2f7413fbd318a83a12aaa9cf666", size = 464644, upload-time = "2026-05-18T04:30:57.33Z" }, + { url = "https://files.pythonhosted.org/packages/0e/80/c7472203bad6268e3ef1ad260739704847898938ad7ea8b63a5131f46b50/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c4997d4e4a55f0d02b6cde327322daf3a0400e5df6c6b15948994bf72497925", size = 454771, upload-time = "2026-05-18T04:30:48.736Z" }, + { url = "https://files.pythonhosted.org/packages/51/cf/3b10b268b4b7f0fc26e9debb5eef1998b515887840f444cd3ec80c688755/watchfiles-1.2.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:4c887eba18b7945ac73067a8b4a66f21cd46c2539b2bc68588f7be6c7eb6d26b", size = 463494, upload-time = "2026-05-18T04:31:33.826Z" }, + { url = "https://files.pythonhosted.org/packages/3d/3e/a4302545cd589262a0dc7d140e86f7688eba3f9c72776c27f7e23b8864c4/watchfiles-1.2.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:3416ff151bb6b5a8d8d11664974fbef4d9305b9b2957839ab5a270468fd8df30", size = 629383, upload-time = "2026-05-18T04:31:15.596Z" }, + { url = "https://files.pythonhosted.org/packages/db/99/d5649df0a9a410d45b7c882304d0b790903ac9b6e8f2cfd12114e0c6b9f2/watchfiles-1.2.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:0e831a271c035d89789cffc386b6aa1375f39f1cd25eb7ca0997e4970d152fc5", size = 656093, upload-time = "2026-05-18T04:31:58.707Z" }, + { url = "https://files.pythonhosted.org/packages/92/b9/362702539275019a54dd2e94511b31a9b89c5f9e6a21966de7eb692549fc/watchfiles-1.2.0-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:37a6721cdf3f65dbb13aa9503510ccb4451603ac837e44d265d7992a597e1374", size = 400109, upload-time = "2026-05-18T04:31:16.879Z" }, + { url = "https://files.pythonhosted.org/packages/8f/75/71d5ba62db781e5587bded1d944c675374bc4aa37ff33d5018d98e8b6538/watchfiles-1.2.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2b37d10b5a63bd4d87e18472d80fa525bd670586fae62e5dd580452764879b65", size = 392167, upload-time = "2026-05-18T04:31:28.058Z" }, + { url = "https://files.pythonhosted.org/packages/3c/01/c66dd95d0423fe30d31820e2d1d5bda773764131bbb6ac0cb1cf303ac328/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a105bc2283f67e8fbec74253ec2d94925de92ed72c0393f1206bf326b7b7b69", size = 452372, upload-time = "2026-05-18T04:31:00.836Z" }, + { url = "https://files.pythonhosted.org/packages/91/15/2fe99557e72f85627c6a8eed50d889e8d101623e060a22ad75b875cb932d/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5327989a465505f05cfe06f04fa9d0c2fd5432bb243e10e6f012b1bdca3c8579", size = 459596, upload-time = "2026-05-18T04:31:34.96Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/d4acfa0023367428ed48351b3b9b267893037b6cadae55620c61c24bcfd4/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ecb47f183a8025b2aa18b546725c3657e542112ae9c0613a2af79b4fa8d04ad7", size = 490869, upload-time = "2026-05-18T04:31:59.923Z" }, + { url = "https://files.pythonhosted.org/packages/a4/5f/3164cbdce06c9fb95c4f7b9e2f9760b5e2797af43a9ecc317ef42a23a278/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8520a4ab0e37f770afc34459c4f8f7019e153f9124dc101c15538365875d1ab2", size = 571641, upload-time = "2026-05-18T04:32:00.948Z" }, + { url = "https://files.pythonhosted.org/packages/41/e6/85d3731c55e65cd7690f3f803d24c139588aaf863e4bf2148fe7a7fa1a19/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71cd71740ed2c15211ebb237ced4e39a1cdf6f80566e5fe95428da1626f4fde6", size = 464444, upload-time = "2026-05-18T04:30:34.298Z" }, + { url = "https://files.pythonhosted.org/packages/f4/7d/562641012b8b09872742c3b8adf9629ec479fd78f8d68ae4a0c13da8add6/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f88af53d6ddaf72179ef613ddc905e6f4785f712b49b80b3bef9f3525e6194b4", size = 453593, upload-time = "2026-05-18T04:31:23.464Z" }, + { url = "https://files.pythonhosted.org/packages/56/fe/cb8ef3d6f929d14158fdaaad9925985b7310abc9384dcd4d82dd0016fb59/watchfiles-1.2.0-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:cee9d5efd929efdac5f7e58f72b3376f676b64050a91c5b99a7094c5b2317488", size = 465096, upload-time = "2026-05-18T04:31:30.384Z" }, + { url = "https://files.pythonhosted.org/packages/25/91/80908e835e100527a9267147b08c0eee1fa6ab0ffec15edc04d1d44885f7/watchfiles-1.2.0-cp315-cp315-musllinux_1_1_aarch64.whl", hash = "sha256:b718bf356bbc15e559bd8ef41782b573b8ae0e3f177ab244b440568d7ea02cfb", size = 630638, upload-time = "2026-05-18T04:30:49.89Z" }, + { url = "https://files.pythonhosted.org/packages/46/4b/95ab2f256bb4af3cb2eb23b9317bda984ee6e0f11733a5c004a6c95b06e3/watchfiles-1.2.0-cp315-cp315-musllinux_1_1_x86_64.whl", hash = "sha256:922c0e019fe68b3ae392965a766b02a71ba1168c932cebc3733cd52c5fe5b377", size = 657684, upload-time = "2026-05-18T04:31:32.027Z" }, ] [[package]] diff --git a/website/vercel.json b/website/vercel.json deleted file mode 100644 index 7aa86301..00000000 --- a/website/vercel.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "redirects": [ - { - "source": "/:path*", - "has": [{ "type": "host", "value": "ktx.sh" }], - "destination": "https://docs.ktx.sh/:path*", - "permanent": true - } - ] -}