diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbfe41d5..3b331b02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,36 +83,8 @@ jobs: - name: Run TypeScript checks run: pnpm run check - slow-context-tests: - name: Slow context tests - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup pnpm - uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 - with: - run_install: false - - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: "24" - cache: "pnpm" - cache-dependency-path: "pnpm-lock.yaml" - - - name: Install TypeScript dependencies - run: pnpm install --frozen-lockfile - - - name: Build TypeScript packages - run: pnpm run build - - - name: Run slow context tests - run: pnpm --filter @ktx/context run test:slow - slow-cli-tests: - name: Slow CLI tests + name: Slow TypeScript tests runs-on: ubuntu-latest steps: - name: Checkout repository @@ -137,7 +109,7 @@ jobs: run: pnpm run build - name: Run slow CLI tests - run: pnpm --filter @ktx/cli run test:slow + run: pnpm --filter @kaelio/ktx run test:slow cli-smoke-tests: name: CLI smoke tests @@ -236,7 +208,7 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} slug: Kaelio/ktx - files: ./packages/cli/coverage/lcov.info,./packages/connector-bigquery/coverage/lcov.info,./packages/connector-clickhouse/coverage/lcov.info,./packages/connector-mysql/coverage/lcov.info,./packages/connector-postgres/coverage/lcov.info,./packages/connector-snowflake/coverage/lcov.info,./packages/connector-sqlite/coverage/lcov.info,./packages/connector-sqlserver/coverage/lcov.info,./packages/context/coverage/lcov.info,./packages/llm/coverage/lcov.info + files: ./packages/cli/coverage/lcov.info flags: typescript name: typescript disable_search: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 167681a6..cc2f483d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,8 +41,13 @@ repos: language: system pass_filenames: false - id: knip-dead-code - name: knip dead-code check - entry: pnpm exec knip --reporter compact + name: knip dead-code (auto-fix) + entry: pnpm exec knip --fix --reporter compact + language: system + pass_filenames: false + - id: knip-dead-code-production + name: knip dead-code (production mode) + entry: pnpm exec knip --production --reporter compact language: system pass_filenames: false diff --git a/AGENTS.md b/AGENTS.md index 1964d8c6..0f62b902 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # ktx Development Notes -**ktx** is a standalone open-source context layer for database agents. These +**ktx** is a standalone open-source context layer for data agents. These instructions apply to all agents working in this repository (Codex, Claude, Gemini, and similar tools). Do not assume an external app server, frontend, database migrations, ORPC contracts, or `python-service/` layout exist here. @@ -63,11 +63,10 @@ When rules conflict, follow this order: **ktx** is a pnpm + uv workspace. -- TypeScript packages: `packages/*` -- CLI package: `packages/cli` -- Core context package: `packages/context` -- LLM package: `packages/llm` -- Database connectors: `packages/connector-*` +- TypeScript package: `packages/cli` (the sole npm-published package source) +- Core context modules: `packages/cli/src/context/` +- LLM provider modules: `packages/cli/src/llm/` +- Database connector modules: `packages/cli/src/connectors//` - Python semantic layer: `python/ktx-sl` - **ktx** daemon: `python/ktx-daemon` - Examples and fixtures: `examples/` @@ -76,9 +75,8 @@ When rules conflict, follow this order: commit `.agents/`, `.claude/`, or `docs/superpowers/` to this public repository. -Some package names still contain `ktx` during the split. Do not mass-rename -symbols, package names, paths, or docs to `ktx` unless the task asks for that -rename. +Some source identifiers still contain historical package-oriented names. Do not +mass-rename symbols, paths, or docs unless the task asks for that rename. ## Quick Commands @@ -91,7 +89,7 @@ pnpm run type-check pnpm run test pnpm run check pnpm run dead-code -pnpm --filter @ktx/cli run smoke +pnpm --filter @kaelio/ktx run smoke pnpm --filter './packages/*' run build pnpm --filter './packages/*' run test pnpm --filter './packages/*' run type-check @@ -165,11 +163,11 @@ rules there with the same weight as the ones in this file. - Keep package exports, `types`, and built `dist` expectations aligned when changing public APIs. - Use `zod` schemas for runtime validation at CLI/config/API boundaries. -- Keep connector packages thin: connector-specific scanning/auth behavior - belongs in `packages/connector-*`; shared types and orchestration belong in - `packages/context`. -- Avoid circular package dependencies. Shared code should move to the lowest - sensible package, not be duplicated across connectors. +- Keep connector modules thin: connector-specific scanning/auth behavior + belongs in `packages/cli/src/connectors//`; shared types and + orchestration belong in `packages/cli/src/context/`. +- Avoid circular module dependencies. Shared code should move to the lowest + sensible module, not be duplicated across connectors. - Do not manually edit generated or built output under `dist/`; edit source and rebuild. @@ -179,7 +177,16 @@ rules there with the same weight as the ones in this file. analysis. These checks are intentionally part of CI and pre-commit because the normal development workflow is agent-based. -- Run `pnpm run dead-code` after TypeScript changes. +- `pnpm run dead-code` runs three checks: Biome (`dead-code:biome`), Knip + default-mode (`dead-code:knip`), and Knip production-mode + (`dead-code:knip:production`). All three must pass. +- Default-mode Knip catches dead code reachable from no entry at all (broken + graph). Production-mode Knip catches code reachable only via tests — + i.e. code that's tested but doesn't ship. +- Pre-commit runs `knip --fix` (auto-removes the `export` keyword from + symbols that are exported but unused) plus `knip --production` (alerts on + test-only paths). CI runs the same checks without `--fix` and fails on any + finding. - Treat Knip findings as investigation prompts, not automatic deletion orders. - Remove private dead code when you confirm there are no imports, dynamic references, generated references, or tests that still need it. @@ -190,6 +197,26 @@ normal development workflow is agent-based. - Update `knip.json` when adding dynamic entrypoints, generated files, package exports, CLI bins, or framework files that Knip cannot infer. +#### Internal exports for testability + +When a function, type, or constant must be exported solely so a unit test can +import it (i.e. it has no production cross-file consumer), annotate the +declaration with `/** @internal */` JSDoc. Knip's production-mode check +ignores `@internal` exports, so the convention keeps the gate clean without +silencing the rest of the file. + +```typescript +/** @internal */ +export function reindexHasErrors(result: ReindexResult): boolean { ... } +``` + +Do NOT use Vitest in-source testing (`if (import.meta.vitest)` blocks). Keep +tests in separate `*.test.ts(x)` files. + +If the only consumer of an export is its own test and the underlying behavior +isn't used in production, delete both the export AND the test — testing dead +code is still dead code. + ### CLI Standards - Use Commander for CLI command trees, arguments, options, help text, custom diff --git a/README.md b/README.md index a94b4a53..b6f0494e 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ ktx wiki "refund policy" --json ktx sl query --connection-id warehouse --measure orders.revenue --format sql ``` -During setup, choose **Ask data questions with ktx MCP** for client agents. +During setup, choose **Ask data questions with ktx MCP** for agent clients. Choose **Ask data questions + manage ktx with CLI commands** when an operator agent also needs pinned `ktx` admin commands. @@ -162,20 +162,14 @@ commands to run. If the output includes `ktx mcp start --project-dir ...`, run it before opening your agent. Claude Desktop uses its own launcher and prints separate skill upload steps under `.ktx/agents/claude/`. -## Workspace packages +## Workspace layout -| Package | Purpose | -|---------|---------| -| `packages/cli` | CLI entry point | -| `packages/context` | Core context engine | -| `packages/llm` | LLM and embedding providers | -| `packages/connector-bigquery` | BigQuery scan connector | -| `packages/connector-clickhouse` | ClickHouse scan connector | -| `packages/connector-mysql` | MySQL scan connector | -| `packages/connector-postgres` | Postgres scan connector | -| `packages/connector-snowflake` | Snowflake scan connector | -| `packages/connector-sqlite` | SQLite scan connector | -| `packages/connector-sqlserver` | SQL Server scan connector | +| Path | Purpose | +|------|---------| +| `packages/cli` | TypeScript CLI package and published npm package source | +| `packages/cli/src/context` | Core context engine | +| `packages/cli/src/llm` | LLM and embedding providers | +| `packages/cli/src/connectors` | Database scan connectors | | `python/ktx-sl` | Semantic-layer query planning | | `python/ktx-daemon` | Portable compute service | diff --git a/docs-site/components/semantic-layer-flow.tsx b/docs-site/components/semantic-layer-flow.tsx index b62d291f..9406ed1d 100644 --- a/docs-site/components/semantic-layer-flow.tsx +++ b/docs-site/components/semantic-layer-flow.tsx @@ -124,7 +124,7 @@ const agent: AgentNode = { position: { x: AGENT_X, y: AGENT_Y }, data: { variant: "single", - title: "Analytics agent", + title: "Data agent", subtitle: "Asks: monthly net revenue and open tickets per segment, high-value orders only, no test customers", }, diff --git a/docs-site/content/docs/community/contributing.mdx b/docs-site/content/docs/community/contributing.mdx index 10b12fe0..3d070993 100644 --- a/docs-site/content/docs/community/contributing.mdx +++ b/docs-site/content/docs/community/contributing.mdx @@ -22,8 +22,8 @@ documentation, connector coverage, and examples. | Area | Good first context | |------|--------------------| | CLI and setup | `packages/cli`, especially setup steps, command definitions, status checks, and smoke tests | -| Context engine | `packages/context`, including project config, ingest orchestration, and semantic search | -| Connectors | `packages/connector-*`, plus connector-specific tests and integration docs | +| Context engine | `packages/cli/src/context`, including project config, ingest orchestration, and semantic search | +| Connectors | `packages/cli/src/connectors/`, plus connector-specific tests and integration docs | | Python semantic layer | `python/ktx-sl` for planning and SQL compilation | | **ktx** daemon | `python/ktx-daemon` for the portable runtime API | | Documentation | `docs-site/content/docs` for public docs and `docs-site/tests` for docs behavior | @@ -50,7 +50,7 @@ pnpm install uv sync --all-groups ``` -`pnpm install` sets up all TypeScript packages in the workspace. +`pnpm install` sets up the TypeScript workspace. `uv sync --all-groups` installs Python dependencies for the semantic layer and daemon, including dev and test groups. @@ -60,11 +60,10 @@ daemon, including dev and test groups. pnpm run build ``` -This builds all TypeScript packages. You can also build individual packages: +This builds the TypeScript package. You can also build the package directly: ```bash -pnpm --filter @ktx/cli run build -pnpm --filter @ktx/context run build +pnpm --filter @kaelio/ktx run build ``` ### Link the CLI for local testing @@ -80,21 +79,15 @@ changes. ## Repository structure -**ktx** is a pnpm + uv workspace. TypeScript packages live in `packages/`, Python -projects in `python/`. +**ktx** is a pnpm + uv workspace. TypeScript source lives in `packages/cli`, +and Python projects live in `python/`. ```text packages/ - cli/ # CLI entry point and commands - context/ # Core context engine (scan, ingest, MCP, semantic layer) - llm/ # LLM client abstraction - connector-postgres/ # PostgreSQL connector - connector-snowflake/ # Snowflake connector - connector-bigquery/ # BigQuery connector - connector-mysql/ # MySQL connector - connector-sqlserver/ # SQL Server connector - connector-sqlite/ # SQLite connector - connector-posthog/ # PostHog connector + cli/ # CLI package and published npm package source + src/context/ # Core context engine (scan, ingest, MCP, semantic layer) + src/llm/ # LLM client abstraction + src/connectors/ # Database connectors python/ ktx-sl/ # Semantic layer - grain-aware query planning and SQL compilation @@ -105,7 +98,7 @@ scripts/ # Workspace scripts (benchmarks, verification, release) docs-site/ # Documentation site (Fumadocs) ``` -All TypeScript packages are ESM (`"type": "module"`) and use `NodeNext` module +The TypeScript package is ESM (`"type": "module"`) and uses `NodeNext` module resolution. The Python projects use `pyproject.toml` for dependency management. ## Running tests @@ -116,18 +109,17 @@ resolution. The Python projects use `pyproject.toml` for dependency management. # Run all tests pnpm run test -# Run tests for a specific package -pnpm --filter @ktx/cli run test -pnpm --filter @ktx/context run test +# Run tests for the TypeScript package +pnpm --filter @kaelio/ktx run test # Type-check all packages pnpm run type-check -# Type-check a specific package -pnpm --filter @ktx/context run type-check +# Type-check the TypeScript package +pnpm --filter @kaelio/ktx run type-check # CLI smoke test -pnpm --filter @ktx/cli run smoke +pnpm --filter @kaelio/ktx run smoke ``` ### Python @@ -164,43 +156,22 @@ uv run pytest -q ## Adding a connector -Database connectors live in `packages/connector-/`. Each connector -implements the `KtxScanConnector` interface from `@ktx/context`. +Database connectors live in `packages/cli/src/connectors//`. Each +connector implements the `KtxScanConnector` interface from the internal context +modules. -### Step 1: Scaffold the package +### Step 1: Scaffold the connector -Create a new directory at `packages/connector-/` with: +Create a new directory at `packages/cli/src/connectors//` with: ```text -packages/connector-/ - package.json - tsconfig.json - src/ - index.ts # Public exports - connector.ts # KtxScanConnector implementation - dialect.ts # SQL dialect handling +packages/cli/src/connectors// + index.ts # Internal connector exports + connector.ts # KtxScanConnector implementation + dialect.ts # SQL dialect handling ``` -The `package.json` should follow the pattern of existing connectors: - -```json -{ - "name": "@ktx/connector-", - "private": true, - "type": "module", - "main": "dist/index.js", - "types": "dist/index.d.ts", - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "dependencies": { - "@ktx/context": "workspace:*" - } -} -``` +Add any connector-specific npm dependency to `packages/cli/package.json`. ### Step 2: Implement the connector @@ -226,20 +197,20 @@ and statistics. ### Step 4: Wire it up -Register the new connector in `packages/context` so the CLI and scan -engine can instantiate it. Look at how existing connectors are registered for -the pattern. +Register the new connector in `packages/cli/src/local-scan-connectors.ts` and +`packages/cli/src/local-adapters.ts` so the CLI and scan engine can instantiate +it. Keep runtime loading dynamic when the connector is optional. ### Step 5: Test ```bash -pnpm --filter @ktx/connector- run build -pnpm --filter @ktx/connector- run type-check -pnpm --filter @ktx/connector- run test +pnpm --filter @kaelio/ktx run build +pnpm --filter @kaelio/ktx run type-check +pnpm --filter @kaelio/ktx run test ``` -Use `packages/connector-sqlite/` as a minimal reference and -`packages/connector-postgres/` as a full-featured one. +Use `packages/cli/src/connectors/sqlite/` as a minimal reference and +`packages/cli/src/connectors/postgres/` as a full-featured one. ## Code conventions diff --git a/docs-site/content/docs/integrations/agent-clients.mdx b/docs-site/content/docs/integrations/agent-clients.mdx index 23430e0c..f7281dda 100644 --- a/docs-site/content/docs/integrations/agent-clients.mdx +++ b/docs-site/content/docs/integrations/agent-clients.mdx @@ -6,9 +6,9 @@ description: Set up ktx with Claude Code, Claude Desktop, Cursor, Codex, and Ope **ktx** exposes context to end-user agents through MCP tools. The CLI remains the admin surface for setup, ingest, status, daemon lifecycle, and debugging. -Run `ktx setup` and select your client agent targets, or configure manually -using the snippets below. Choose **Ask data questions with ktx MCP** for client -agents. Choose **Ask data questions + manage ktx with CLI commands** only when +Run `ktx setup` and select your agent client targets, or configure manually +using the snippets below. Choose **Ask data questions with ktx MCP** for agent +clients. Choose **Ask data questions + manage ktx with CLI commands** only when a developer or operator agent also needs pinned `ktx` admin commands. ## Install with setup diff --git a/docs/release.md b/docs/release.md index 00362c99..3a72f54e 100644 --- a/docs/release.md +++ b/docs/release.md @@ -105,27 +105,36 @@ prior rc lineage is consumed cleanly. ## Release metadata semantic-release calls `scripts/update-public-release-version.mjs` during the -prepare step before the exec publish command runs. That script updates the -following files **inside the CI runner only**: +prepare step before the exec publish command runs. That script rewrites the +following files in lockstep: -- `package.json` with the semantic-release version. -- `release-policy.json` with `publicNpmPackageVersion`, npm publish settings, - and the published package smoke-test version. +- `package.json` and `packages/cli/package.json` with the semantic-release + version. +- `python/ktx-daemon/pyproject.toml` and `python/ktx-sl/pyproject.toml` with + the same version, normalized for PEP 440 (e.g. `0.1.0-rc.2` → + `0.1.0rc2`). Branch-prefixed npm releases skip this step because they are + not published to PyPI. +- `release-policy.json` with the npm publish settings, release mode, and + published package smoke-test version. The package version itself is not + stored here; it is derived from `packages/cli/package.json.version` by + `scripts/public-npm-release-metadata.mjs`, so there is only one place that + owns the version. -The artifact packaging, readiness, and smoke-test scripts read -`publicNpmPackageVersion` from `release-policy.json` within the same CI run. -Nothing reads these files at runtime — the daemon and CLI rely on the -published `package.json` (for the installed `@kaelio/ktx` package) or -`packages/cli/package.json` (for dev-tree runs from this repo, which always -report `0.0.0-private`). Because the metadata mutation never has to survive -the run, no commit is pushed back to `main`. The git tag plus the published -npm artifact carry the version forward. +The `@semantic-release/git` plugin then commits these files back to `main` +with the release tag (see `scripts/semantic-release-config.cjs`). As a +result, the dev tree always reflects the most recently published version — +there is no sentinel pin. `ktx --version` and the bundled Python wheel both +report whatever was last released. -The bundled Python runtime wheel also derives its version from -`publicNpmPackageVersion`. Stable npm versions are reused as-is, and rc -versions are normalized to Python's version format. For example, -`0.1.0-rc.2` becomes `0.1.0rc2` in the `kaelio-ktx` wheel filename and wheel -metadata. +At runtime the CLI reads its version from its own `package.json` via +`getKtxCliPackageInfo()` (`packages/cli/src/cli-runtime.ts`); the Python +daemon reads its version from installed-package metadata via +`importlib.metadata.version()` (`python/ktx-daemon/src/ktx_daemon/__init__.py`). + +The bundled Python runtime wheel also derives its version from the same +single source. Stable npm versions are reused as-is, and rc versions are +normalized to Python's version format. For example, `0.1.0-rc.2` becomes +`0.1.0rc2` in the `kaelio-ktx` wheel filename and wheel metadata. ## npm authentication diff --git a/examples/postgres-historic/README.md b/examples/postgres-historic/README.md index 30dcfde5..64fc2593 100644 --- a/examples/postgres-historic/README.md +++ b/examples/postgres-historic/README.md @@ -58,7 +58,7 @@ Create a project and enable query history: ```bash export WAREHOUSE_DATABASE_URL=postgresql://ktx_reader:ktx_reader@127.0.0.1:55432/analytics # pragma: allowlist secret -pnpm --filter @ktx/cli run build +pnpm --filter @kaelio/ktx run build node packages/cli/dist/bin.js --project-dir /tmp/ktx-postgres-historic setup \ --new \ --skip-agents \ diff --git a/examples/postgres-historic/scripts/smoke.sh b/examples/postgres-historic/scripts/smoke.sh index 1fe10b70..947eeb88 100755 --- a/examples/postgres-historic/scripts/smoke.sh +++ b/examples/postgres-historic/scripts/smoke.sh @@ -198,8 +198,7 @@ NODE } cd "$KTX_ROOT" -pnpm --filter @ktx/context run build -pnpm --filter @ktx/cli run build +pnpm --filter @kaelio/ktx run build docker compose -f "$COMPOSE_FILE" up -d --wait "$EXAMPLE_DIR/scripts/generate-workload.sh" base diff --git a/knip.json b/knip.json index 6caf525c..08939c28 100644 --- a/knip.json +++ b/knip.json @@ -2,121 +2,41 @@ "$schema": "https://unpkg.com/knip@6/schema.json", "workspaces": { ".": { - "entry": ["scripts/**/*.mjs"], - "project": ["scripts/**/*.mjs"], - "ignoreDependencies": [ - "@semantic-release/commit-analyzer", - "@semantic-release/github", - "@semantic-release/npm", - "@semantic-release/release-notes-generator", - "conventional-changelog-conventionalcommits" + "entry": [ + "scripts/**/*.mjs", + "scripts/**/*.cjs", + ".releaserc.cjs!" ] }, "packages/cli": { "entry": [ - "src/index.ts", - "src/bin.ts", - "src/**/*.test.ts", - "src/**/*.test.tsx", - "scripts/**/*.mjs" - ], - "project": ["src/**/*.{ts,tsx}", "scripts/**/*.mjs", "vitest.config.ts"] - }, - "packages/context": { - "entry": [ - "src/index.ts", - "src/agent/index.ts", - "src/core/index.ts", - "src/connections/index.ts", - "src/daemon/index.ts", - "src/ingest/index.ts", - "src/ingest/memory-flow/index.ts", - "src/ingest/metabase-mapping.ts", - "src/scan/index.ts", - "src/search/index.ts", - "src/sql-analysis/index.ts", - "src/memory/index.ts", - "src/mcp/index.ts", - "src/project/index.ts", - "src/prompts/index.ts", - "src/skills/index.ts", - "src/sl/index.ts", - "src/sl/descriptions.ts", - "src/tools/index.ts", - "src/wiki/index.ts", - "src/**/*.test.ts", - "scripts/**/*.mjs" - ], - "project": ["src/**/*.ts", "scripts/**/*.mjs", "vitest.config.ts"] - }, - "packages/llm": { - "entry": ["src/index.ts", "src/**/*.test.ts"], - "project": ["src/**/*.ts", "vitest.config.ts"] - }, - "packages/connector-*": { - "entry": ["src/index.ts", "src/**/*.test.ts"], - "project": ["src/**/*.ts"] + "src/print-command-tree.ts!", + "scripts/**/*.mjs", + "src/**/*.test-utils.ts", + "src/**/acceptance-fixtures.ts", + "src/context/scan/relationship-benchmarks.ts!", + "src/context/scan/relationship-benchmark-report.ts!" + ] }, "docs-site": { "entry": [ - "app/**/*.{ts,tsx}", - "components/**/*.{ts,tsx}", - "lib/**/*.{ts,tsx}", - "middleware.ts", - "next.config.mjs", - "source.config.ts", - "tests/**/*.mjs" + "components/**/*.{ts,tsx}!", + "source.config.ts!" ], - "project": [ - "app/**/*.{ts,tsx}", - "components/**/*.{ts,tsx}", - "lib/**/*.{ts,tsx}", - "*.ts", - "*.mjs", - "tests/**/*.mjs" - ], - "ignoreDependencies": ["tailwindcss"] + "ignoreDependencies": [ + "tailwindcss" + ] } }, - "ignore": [ - "**/dist/**", - "**/coverage/**", - "**/.next/**", - "**/node_modules/**", - "**/*.gen.ts", - "**/*.generated.ts" + "ignoreDependencies": [ + "@semantic-release/commit-analyzer", + "@semantic-release/github", + "@semantic-release/npm", + "@semantic-release/release-notes-generator", + "conventional-changelog-conventionalcommits" ], - "ignoreBinaries": ["uv", "lsof"], - "ignoreIssues": { - "packages/cli/src/clack.ts": ["exports"], - "packages/cli/src/commands/connection-metabase-setup.ts": ["exports", "types"], - "packages/cli/src/ingest.test-utils.ts": ["exports"], - "packages/cli/src/io/symbols.ts": ["exports"], - "packages/cli/src/managed-python-command.ts": ["types"], - "packages/cli/src/managed-python-daemon.ts": ["types"], - "packages/cli/src/managed-python-http.ts": ["exports", "types"], - "packages/cli/src/managed-python-runtime.ts": ["types"], - "packages/cli/src/memory-flow-tui.tsx": ["types"], - "packages/cli/src/next-steps.ts": ["exports"], - "packages/cli/src/print-command-tree.ts": ["exports"], - "packages/cli/src/setup-agents.ts": ["exports", "types"], - "packages/cli/src/setup-context.ts": ["types"], - "packages/cli/src/setup-demo-tour.ts": ["exports"], - "packages/cli/src/setup-models.ts": ["exports"], - "packages/cli/src/setup-project.ts": ["types"], - "packages/cli/src/setup-ready-menu.ts": ["types"], - "packages/cli/src/setup-sources.ts": ["types"], - "packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts": ["exports", "types"], - "packages/context/src/ingest/adapters/lookml/pull-config.ts": ["exports"], - "packages/context/src/ingest/adapters/metabase/serialize-card.ts": ["types"], - "packages/context/src/ingest/adapters/metabase/types.ts": ["exports"], - "packages/context/src/ingest/adapters/metricflow/parse.ts": ["types"], - "packages/context/src/ingest/ports.ts": ["types"], - "packages/context/src/ingest/stages/stage-3-work-units.ts": ["types"], - "packages/context/src/ingest/stages/stage-index.types.ts": ["types"], - "packages/context/src/project/config.ts": ["types"], - "packages/context/src/scan/relationship-candidates.ts": ["types"], - "packages/context/src/scan/relationship-diagnostics.ts": ["types"], - "packages/context/src/tools/context-evidence-tool-store.ts": ["types"] - } + "ignoreBinaries": [ + "uv", + "lsof" + ] } diff --git a/package.json b/package.json index 5c8fe060..9fcd583a 100644 --- a/package.json +++ b/package.json @@ -19,10 +19,11 @@ "artifacts:verify-manifest": "node scripts/package-artifacts.mjs verify-manifest", "build": "pnpm --filter './packages/*' run build", "check": "node scripts/check-boundaries.mjs && node --test scripts/*.test.mjs && pnpm --filter './packages/*' run build && pnpm --filter './packages/*' run test", - "dead-code": "pnpm run dead-code:biome && pnpm run dead-code:knip", + "dead-code": "pnpm run dead-code:biome && pnpm run dead-code:knip && pnpm run dead-code:knip:production", "dead-code:biome": "biome ci . --formatter-enabled=false --assist-enabled=false", "dead-code:fix": "biome check . --formatter-enabled=false --assist-enabled=false --write && knip --fix --format", "dead-code:knip": "knip --reporter compact", + "dead-code:knip:production": "knip --production --reporter compact", "docs": "kill $(lsof -ti:3000) 2>/dev/null; pnpm --filter ktx-docs run dev", "ktx": "node scripts/run-ktx.mjs", "link:dev": "node scripts/link-dev-cli.mjs", @@ -39,16 +40,18 @@ "semantic-release": "semantic-release", "semantic-release:debug": "semantic-release --dry-run --debug", "semantic-release:dry-run": "semantic-release --dry-run --no-ci", - "smoke": "pnpm run build && pnpm --filter @ktx/cli run smoke", + "smoke": "pnpm run build && pnpm --filter @kaelio/ktx run smoke", "test": "node --test scripts/*.test.mjs && pnpm --filter './packages/*' run test", "test:coverage": "pnpm run test:coverage:ts && pnpm run test:coverage:py", "test:coverage:py": "uv run pytest --cov=python/ktx-sl/semantic_layer --cov=python/ktx-daemon/src/ktx_daemon --cov-report=xml:coverage/python.xml --cov-report=term", "test:coverage:ts": "pnpm --filter './packages/*' run build && pnpm --filter './packages/*' run test --coverage --coverage.reporter=lcov --coverage.exclude='dist/**' && node scripts/normalize-lcov-paths.mjs packages/*/coverage/lcov.info", - "test:slow": "pnpm --filter @ktx/context run test:slow && pnpm --filter @ktx/cli run test:slow", + "test:slow": "pnpm --filter @kaelio/ktx run test:slow", "type-check": "pnpm --filter './packages/*' run type-check" }, "devDependencies": { "@biomejs/biome": "^2.4.15", + "@electric-sql/pglite": "^0.4.5", + "@electric-sql/pglite-socket": "^0.1.5", "@semantic-release/commit-analyzer": "^13.0.1", "@semantic-release/exec": "^7.1.0", "@semantic-release/git": "^10.0.1", @@ -59,6 +62,7 @@ "better-sqlite3": "^12.10.0", "conventional-changelog-conventionalcommits": "^9.3.1", "knip": "^6.12.2", + "pg": "^8.20.0", "semantic-release": "^25.0.3", "typescript": "^6.0.3", "yaml": "^2.9.0" diff --git a/packages/cli/package.json b/packages/cli/package.json index 75355189..296276c0 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,8 +1,7 @@ { - "name": "@ktx/cli", + "name": "@kaelio/ktx", "version": "0.4.1", - "description": "CLI wrapper for ktx context packages", - "private": true, + "description": "Standalone ktx context layer for data agents", "type": "module", "engines": { "node": ">=22.0.0" @@ -24,41 +23,68 @@ "dist", "assets" ], + "publishConfig": { + "access": "public", + "provenance": true + }, "scripts": { "assets:demo": "node scripts/build-demo-assets.mjs", "build": "tsc -p tsconfig.json && node scripts/copy-runtime-assets.mjs && node ../../scripts/prepare-cli-bin.mjs", - "clean": "node -e \"fs.rmSync('dist', { recursive: true, force: true }); fs.rmSync('node_modules/.cache/tsc.tsbuildinfo', { force: true })\"", + "clean": "node -e \"fs.rmSync('dist', { recursive: true, force: true })\"", "docs:commands": "pnpm run build && node dist/print-command-tree.js", "smoke": "vitest run src/standalone-smoke.test.ts src/example-smoke.test.ts --testTimeout 30000", - "test": "vitest run --exclude src/standalone-smoke.test.ts --exclude src/example-smoke.test.ts --exclude src/setup-databases.test.ts --exclude src/scan.test.ts --exclude src/commands/connection-metabase-setup.test.ts --exclude src/setup-models.test.ts --exclude src/setup-sources.test.ts --exclude src/setup.test.ts --exclude src/connection.test.ts --exclude src/setup-embeddings.test.ts --exclude src/ingest.test.ts --exclude src/commands/connection-mapping.test.ts --exclude src/ingest-viz.test.ts --exclude src/demo.test.ts --exclude src/setup-project.test.ts --exclude src/sl.test.ts --exclude src/local-scan-connectors.test.ts --exclude src/commands/connection-notion.test.ts", - "test:slow": "vitest run src/setup-databases.test.ts src/scan.test.ts src/commands/connection-metabase-setup.test.ts src/setup-models.test.ts src/setup-sources.test.ts src/setup.test.ts src/connection.test.ts src/setup-embeddings.test.ts src/ingest.test.ts src/commands/connection-mapping.test.ts src/ingest-viz.test.ts src/demo.test.ts src/setup-project.test.ts src/sl.test.ts src/local-scan-connectors.test.ts src/commands/connection-notion.test.ts --testTimeout 30000", - "type-check": "tsc -p tsconfig.json --noEmit" + "test": "vitest run --exclude src/standalone-smoke.test.ts --exclude src/example-smoke.test.ts --exclude src/setup-databases.test.ts --exclude src/scan.test.ts --exclude src/commands/connection-metabase-setup.test.ts --exclude src/setup-models.test.ts --exclude src/setup-sources.test.ts --exclude src/setup.test.ts --exclude src/connection.test.ts --exclude src/setup-embeddings.test.ts --exclude src/ingest.test.ts --exclude src/commands/connection-mapping.test.ts --exclude src/ingest-viz.test.ts --exclude src/demo.test.ts --exclude src/setup-project.test.ts --exclude src/sl.test.ts --exclude src/local-scan-connectors.test.ts --exclude src/commands/connection-notion.test.ts --exclude src/context/scan/local-scan.test.ts --exclude src/context/mcp/local-project-ports.test.ts --exclude src/context/ingest/local-stage-ingest.test.ts --exclude src/context/sl/pglite-sl-search-prototype.test.ts --exclude src/context/core/git.service.test.ts --exclude src/context/ingest/local-adapters.test.ts --exclude src/context/ingest/local-bundle-ingest.test.ts --exclude src/context/ingest/local-metabase-ingest.test.ts --exclude src/context/sl/local-sl.test.ts --exclude src/context/search/pglite-owner-process.test.ts --exclude src/context/scan/local-enrichment-artifacts.test.ts --exclude src/context/search/pglite-spike.test.ts --exclude src/context/wiki/local-knowledge.test.ts --exclude src/context/sl/local-query.test.ts --exclude src/context/scan/relationship-review-decisions.test.ts --exclude src/context/scan/relationship-profiling.test.ts", + "test:slow": "vitest run src/setup-databases.test.ts src/scan.test.ts src/commands/connection-metabase-setup.test.ts src/setup-models.test.ts src/setup-sources.test.ts src/setup.test.ts src/connection.test.ts src/setup-embeddings.test.ts src/ingest.test.ts src/commands/connection-mapping.test.ts src/ingest-viz.test.ts src/demo.test.ts src/setup-project.test.ts src/sl.test.ts src/local-scan-connectors.test.ts src/commands/connection-notion.test.ts src/context/scan/local-scan.test.ts src/context/mcp/local-project-ports.test.ts src/context/ingest/local-stage-ingest.test.ts src/context/sl/pglite-sl-search-prototype.test.ts src/context/core/git.service.test.ts src/context/ingest/local-adapters.test.ts src/context/ingest/local-bundle-ingest.test.ts src/context/ingest/local-metabase-ingest.test.ts src/context/sl/local-sl.test.ts src/context/search/pglite-owner-process.test.ts src/context/scan/local-enrichment-artifacts.test.ts src/context/search/pglite-spike.test.ts src/context/wiki/local-knowledge.test.ts src/context/sl/local-query.test.ts src/context/scan/relationship-review-decisions.test.ts src/context/scan/relationship-profiling.test.ts --testTimeout 30000", + "type-check": "tsc -p tsconfig.json --noEmit", + "relationships:benchmarks": "pnpm --silent run build && node ../../scripts/relationship-benchmark-report.mjs", + "relationships:benchmarks:test": "KTX_RUN_RELATIONSHIP_BENCHMARKS=1 vitest run src/context/scan/relationship-benchmarks.test.ts", + "search:pglite-spike": "node ../../scripts/pglite-hybrid-search-spike.mjs", + "search:pglite-owner-prototype": "node ../../scripts/pglite-owner-process-prototype.mjs", + "search:pglite-sl-prototype": "node ../../scripts/pglite-sl-search-prototype.mjs" }, "dependencies": { + "@ai-sdk/anthropic": "3.0.77", + "@ai-sdk/devtools": "0.0.17", + "@ai-sdk/google-vertex": "^4.0.128", + "@anthropic-ai/claude-agent-sdk": "0.3.142", "@clack/prompts": "1.4.0", + "@clickhouse/client": "^1.18.4", "@commander-js/extra-typings": "14.0.0", - "@ktx/connector-bigquery": "workspace:*", - "@ktx/connector-clickhouse": "workspace:*", - "@ktx/connector-mysql": "workspace:*", - "@ktx/connector-postgres": "workspace:*", - "@ktx/connector-snowflake": "workspace:*", - "@ktx/connector-sqlite": "workspace:*", - "@ktx/connector-sqlserver": "workspace:*", - "@ktx/context": "workspace:*", - "@ktx/llm": "workspace:*", + "@google-cloud/bigquery": "^8.3.1", + "@looker/sdk": "^26.8.0", + "@looker/sdk-node": "^26.8.0", + "@looker/sdk-rtl": "^21.6.5", "@modelcontextprotocol/sdk": "^1.29.0", + "@notionhq/client": "^5.21.0", + "ai": "^6.0.180", + "better-sqlite3": "^12.10.0", "commander": "14.0.3", "fflate": "^0.8.2", + "handlebars": "^4.7.9", "ink": "^7.0.2", + "lookml-parser": "7.1.0", + "minimatch": "^10.2.5", + "mssql": "^12.5.2", + "mysql2": "^3.22.3", + "openai": "^6.37.0", + "p-limit": "^7.3.0", + "pg": "^8.20.0", "react": "^19.2.6", + "simple-git": "3.36.0", + "snowflake-sdk": "^2.4.1", + "yaml": "^2.9.0", "zod": "^4.4.3" }, "devDependencies": { + "@electric-sql/pglite": "^0.4.5", + "@electric-sql/pglite-socket": "^0.1.5", "@types/better-sqlite3": "^7.6.13", + "@types/mssql": "^12.3.0", "@types/node": "^25.7.0", + "@types/pg": "^8.20.0", "@types/react": "^19.2.14", "@vitest/coverage-v8": "^4.1.6", - "better-sqlite3": "^12.10.0", + "ajv": "8.20.0", "ink-testing-library": "^4.0.0", "typescript": "^6.0.3", "vitest": "^4.1.6" @@ -66,11 +92,11 @@ "license": "Apache-2.0", "repository": { "type": "git", - "url": "git+https://github.com/kaelio/ktx.git", + "url": "https://github.com/Kaelio/ktx", "directory": "packages/cli" }, "bugs": { - "url": "https://github.com/kaelio/ktx/issues" + "url": "https://github.com/Kaelio/ktx/issues" }, - "homepage": "https://github.com/kaelio/ktx#readme" + "homepage": "https://github.com/Kaelio/ktx#readme" } diff --git a/packages/cli/scripts/copy-runtime-assets.mjs b/packages/cli/scripts/copy-runtime-assets.mjs index a7c75658..579cb8e8 100644 --- a/packages/cli/scripts/copy-runtime-assets.mjs +++ b/packages/cli/scripts/copy-runtime-assets.mjs @@ -3,9 +3,14 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; const packageRoot = fileURLToPath(new URL('..', import.meta.url)); +const promptsSource = join(packageRoot, 'src', 'prompts'); +const promptsTarget = join(packageRoot, 'dist', 'prompts'); const skillsSource = join(packageRoot, 'src', 'skills'); const skillsTarget = join(packageRoot, 'dist', 'skills'); +await rm(promptsTarget, { recursive: true, force: true }); await rm(skillsTarget, { recursive: true, force: true }); +await mkdir(dirname(promptsTarget), { recursive: true }); await mkdir(dirname(skillsTarget), { recursive: true }); +await cp(promptsSource, promptsTarget, { recursive: true }); await cp(skillsSource, skillsTarget, { recursive: true }); diff --git a/packages/cli/src/admin-reindex.test.ts b/packages/cli/src/admin-reindex.test.ts index 0f14486a..dace420f 100644 --- a/packages/cli/src/admin-reindex.test.ts +++ b/packages/cli/src/admin-reindex.test.ts @@ -1,11 +1,11 @@ import { createRequire } from 'node:module'; -import type { ReindexSummary } from '@ktx/context/index-sync'; +import type { ReindexSummary } from './context/index-sync/types.js'; import { describe, expect, it, vi } from 'vitest'; import { renderReindexJson, renderReindexPlain, reindexHasErrors } from './admin-reindex.js'; import { runKtxCli } from './index.js'; -const cliVersion = (createRequire(import.meta.url)('@ktx/cli/package.json') as { version: string }) +const cliVersion = (createRequire(import.meta.url)('@kaelio/ktx/package.json') as { version: string }) .version; function makeIo(options: { stdoutIsTTY?: boolean } = {}) { diff --git a/packages/cli/src/admin-reindex.ts b/packages/cli/src/admin-reindex.ts index 5d978d59..82e3bbaf 100644 --- a/packages/cli/src/admin-reindex.ts +++ b/packages/cli/src/admin-reindex.ts @@ -1,6 +1,8 @@ -import { KtxIngestEmbeddingPortAdapter, type KtxEmbeddingPort } from '@ktx/context'; -import { reindexLocalIndexes, type ReindexScopeResult, type ReindexSummary } from '@ktx/context/index-sync'; -import { loadKtxProject } from '@ktx/context/project'; +import { KtxIngestEmbeddingPortAdapter } from './context/llm/embedding-port.js'; +import type { KtxEmbeddingPort } from './context/core/embedding.js'; +import { reindexLocalIndexes } from './context/index-sync/reindex.js'; +import type { ReindexScopeResult, ReindexSummary } from './context/index-sync/types.js'; +import { loadKtxProject } from './context/project/project.js'; import { Option, type Command } from '@commander-js/extra-typings'; import { cancel, intro, log, note, outro } from '@clack/prompts'; import type { KtxCliCommandContext } from './cli-program.js'; @@ -55,10 +57,12 @@ function quotePlainValue(value: string): string { return `"${value.replaceAll('\\', '\\\\').replaceAll('"', '\\"')}"`; } +/** @internal */ export function reindexHasErrors(summary: ReindexSummary): boolean { return summary.scopes.some((scope) => scope.error); } +/** @internal */ export function renderReindexPlain(summary: ReindexSummary, io: KtxCliIo): void { const updateKey = summary.force ? 'rebuilt' : 'updated'; for (const scope of summary.scopes) { @@ -88,6 +92,7 @@ export function renderReindexPlain(summary: ReindexSummary, io: KtxCliIo): void ); } +/** @internal */ export function renderReindexJson(summary: ReindexSummary, io: KtxCliIo): void { io.stdout.write(`${JSON.stringify({ kind: 'reindex', data: summary, meta: { command: 'admin reindex' } }, null, 2)}\n`); } diff --git a/packages/cli/src/admin.ts b/packages/cli/src/admin.ts index af23c192..6c04f82f 100644 --- a/packages/cli/src/admin.ts +++ b/packages/cli/src/admin.ts @@ -50,7 +50,7 @@ export function registerAdminCommands(program: Command, context: KtxCliCommandCo .description('Print a JSON Schema describing ktx.yaml (for editors and LLM agents)') .option('--output ', 'Write the schema to a file instead of stdout') .action(async (options: { output?: string }) => { - const { generateKtxProjectConfigJsonSchema } = await import('@ktx/context/project'); + const { generateKtxProjectConfigJsonSchema } = await import('./context/project/config.js');; const json = `${JSON.stringify(generateKtxProjectConfigJsonSchema(), null, 2)}\n`; if (options.output) { const { writeFile } = await import('node:fs/promises'); diff --git a/packages/cli/src/clack.ts b/packages/cli/src/clack.ts index ce6f5872..55d3e802 100644 --- a/packages/cli/src/clack.ts +++ b/packages/cli/src/clack.ts @@ -26,7 +26,7 @@ export interface KtxCliPromptAdapter { spinner(): KtxCliSpinner; } -export class KtxCliPromptCancelledError extends Error { +class KtxCliPromptCancelledError extends Error { constructor(message = 'Operation cancelled.') { super(message); this.name = 'KtxCliPromptCancelledError'; diff --git a/packages/cli/src/claude-code-prompt-caching.ts b/packages/cli/src/claude-code-prompt-caching.ts index 78cd5764..a7c0fa54 100644 --- a/packages/cli/src/claude-code-prompt-caching.ts +++ b/packages/cli/src/claude-code-prompt-caching.ts @@ -1,4 +1,4 @@ -import type { KtxProjectLlmConfig } from '@ktx/context/project'; +import type { KtxProjectLlmConfig } from './context/project/config.js'; const CLAUDE_CODE_IGNORED_PROMPT_CACHING_FIELDS = [ 'systemTtl', diff --git a/packages/cli/src/cli-program.test.ts b/packages/cli/src/cli-program.test.ts index 91e2ed8f..565a5d56 100644 --- a/packages/cli/src/cli-program.test.ts +++ b/packages/cli/src/cli-program.test.ts @@ -12,9 +12,8 @@ function stubIo(): KtxCliIo { function stubPackageInfo(): KtxCliPackageInfo { return { - name: '@ktx/cli', + name: '@kaelio/ktx', version: '0.0.0-test', - contextPackageName: '@ktx/context', }; } diff --git a/packages/cli/src/cli-project.test.ts b/packages/cli/src/cli-project.test.ts deleted file mode 100644 index df5aeb71..00000000 --- a/packages/cli/src/cli-project.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { buildDefaultKtxProjectConfig, type KtxLocalProject, type KtxProjectConfig } from '@ktx/context/project'; -import { loadKtxCliProject } from './cli-project.js'; - -function projectWithConfig(config: KtxProjectConfig): KtxLocalProject { - return { - projectDir: '/work/proj', - configPath: '/work/proj/ktx.yaml', - config, - coreConfig: {} as KtxLocalProject['coreConfig'], - git: {} as KtxLocalProject['git'], - fileStore: {} as KtxLocalProject['fileStore'], - }; -} - -describe('loadKtxCliProject', () => { - it('delegates to loadKtxProject and returns the project unchanged', async () => { - const project = projectWithConfig(buildDefaultKtxProjectConfig()); - const loadProject = vi.fn(async () => project); - - const result = await loadKtxCliProject({ projectDir: '/work/proj' }, { loadProject }); - - expect(result).toBe(project); - expect(loadProject).toHaveBeenCalledWith({ projectDir: '/work/proj' }); - }); -}); diff --git a/packages/cli/src/cli-project.ts b/packages/cli/src/cli-project.ts deleted file mode 100644 index 1dfd5aef..00000000 --- a/packages/cli/src/cli-project.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project'; - -export interface LoadKtxCliProjectOptions { - projectDir: string; -} - -export interface LoadKtxCliProjectDeps { - loadProject?: typeof loadKtxProject; -} - -/** - * Thin wrapper around `loadKtxProject`. Kept as a single entrypoint so the CLI can grow shared - * pre-load behavior later (telemetry, project lock, etc.). Today it does no extra work. - */ -export async function loadKtxCliProject( - options: LoadKtxCliProjectOptions, - deps: LoadKtxCliProjectDeps = {}, -): Promise { - return (deps.loadProject ?? loadKtxProject)({ projectDir: options.projectDir }); -} diff --git a/packages/cli/src/cli-runtime.ts b/packages/cli/src/cli-runtime.ts index 523945f5..68089720 100644 --- a/packages/cli/src/cli-runtime.ts +++ b/packages/cli/src/cli-runtime.ts @@ -20,7 +20,6 @@ const requirePackageJson = createRequire(import.meta.url); export interface KtxCliPackageInfo { name: string; version: string; - contextPackageName: '@ktx/context'; } export interface KtxCliIo { @@ -67,12 +66,11 @@ export function packageInfoFromJson(packageJson: unknown): KtxCliPackageInfo { return { name: packageJson.name, version: assertCliVersion(packageJson.version, `${packageJson.name}/package.json`), - contextPackageName: '@ktx/context', }; } async function runInit(args: { projectDir: string; force: boolean }, io: KtxCliIo): Promise { - const { initKtxProject } = await import('@ktx/context/project'); + const { initKtxProject } = await import('./context/project/project.js');; const result = await initKtxProject({ projectDir: args.projectDir, force: args.force, diff --git a/packages/cli/src/commands/mcp-commands.test.ts b/packages/cli/src/commands/mcp-commands.test.ts index 29235bce..dfcd1946 100644 --- a/packages/cli/src/commands/mcp-commands.test.ts +++ b/packages/cli/src/commands/mcp-commands.test.ts @@ -11,7 +11,7 @@ function makeContext(overrides: Partial = {}): KtxCliComma stderr: { write: vi.fn() }, }, deps: {}, - packageInfo: { name: '@ktx/cli', version: '0.0.0-test', contextPackageName: '@ktx/context' }, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, setExitCode: (code) => { exitCode = code; }, diff --git a/packages/cli/src/commands/sql-commands.test.ts b/packages/cli/src/commands/sql-commands.test.ts index f690b305..4f2c0277 100644 --- a/packages/cli/src/commands/sql-commands.test.ts +++ b/packages/cli/src/commands/sql-commands.test.ts @@ -11,7 +11,7 @@ function makeContext(overrides: Partial = {}): KtxCliComma stderr: { write: vi.fn() }, }, deps: {}, - packageInfo: { name: '@ktx/cli', version: '0.0.0-test', contextPackageName: '@ktx/context' }, + packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' }, setExitCode: (code) => { exitCode = code; }, diff --git a/packages/cli/src/connection.test.ts b/packages/cli/src/connection.test.ts index 88f4b921..b05a6f16 100644 --- a/packages/cli/src/connection.test.ts +++ b/packages/cli/src/connection.test.ts @@ -1,9 +1,12 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import type { LookerClient, MetabaseRuntimeClient, NotionClient } from '@ktx/context/ingest'; -import { initKtxProject, parseKtxProjectConfig, serializeKtxProjectConfig } from '@ktx/context/project'; -import type { KtxConnectionDriver, KtxScanConnector } from '@ktx/context/scan'; +import type { LookerClient } from './context/ingest/adapters/looker/client.js'; +import type { MetabaseRuntimeClient } from './context/ingest/adapters/metabase/client-port.js'; +import type { NotionClient } from './context/ingest/adapters/notion/notion-client.js'; +import { initKtxProject } from './context/project/project.js'; +import { parseKtxProjectConfig, serializeKtxProjectConfig } from './context/project/config.js'; +import type { KtxConnectionDriver, KtxScanConnector } from './context/scan/types.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runKtxConnection } from './connection.js'; diff --git a/packages/cli/src/connection.ts b/packages/cli/src/connection.ts index 06d02922..174fe5ad 100644 --- a/packages/cli/src/connection.ts +++ b/packages/cli/src/connection.ts @@ -1,19 +1,15 @@ -import { - DEFAULT_METABASE_CLIENT_CONFIG, - DefaultLookerConnectionClientFactory, - DefaultMetabaseConnectionClientFactory, - type LookerClient, - type MetabaseRuntimeClient, - type NotionBotInfo, - NotionClient, - createLocalLookerCredentialResolver, - metabaseRuntimeConfigFromLocalConnection, - testRepoConnection, -} from '@ktx/context/ingest'; -import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from '@ktx/context/connections'; -import { resolveKtxConfigReference } from '@ktx/context/core'; -import { type KtxLocalProject, loadKtxProject } from '@ktx/context/project'; -import type { KtxScanConnector } from '@ktx/context/scan'; +import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory } from './context/ingest/adapters/metabase/client.js'; +import { DefaultLookerConnectionClientFactory } from './context/ingest/adapters/looker/factory.js'; +import type { LookerClient } from './context/ingest/adapters/looker/client.js'; +import type { MetabaseRuntimeClient } from './context/ingest/adapters/metabase/client-port.js'; +import { type NotionBotInfo, NotionClient } from './context/ingest/adapters/notion/notion-client.js'; +import { createLocalLookerCredentialResolver } from './context/ingest/adapters/looker/local-looker.adapter.js'; +import { metabaseRuntimeConfigFromLocalConnection } from './context/ingest/adapters/metabase/local-metabase.adapter.js'; +import { testRepoConnection } from './context/ingest/repo-fetch.js'; +import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from './context/connections/notion-config.js'; +import { resolveKtxConfigReference } from './context/core/config-reference.js'; +import { type KtxLocalProject, loadKtxProject } from './context/project/project.js'; +import type { KtxScanConnector } from './context/scan/types.js'; import type { KtxCliIo } from './index.js'; import { bold, dim, green, red, SYMBOLS } from './io/symbols.js'; import { createKtxCliScanConnector } from './local-scan-connectors.js'; diff --git a/packages/connector-bigquery/src/connector.test.ts b/packages/cli/src/connectors/bigquery/connector.test.ts similarity index 96% rename from packages/connector-bigquery/src/connector.test.ts rename to packages/cli/src/connectors/bigquery/connector.test.ts index 0de940df..5a890612 100644 --- a/packages/connector-bigquery/src/connector.test.ts +++ b/packages/cli/src/connectors/bigquery/connector.test.ts @@ -1,15 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { - bigQueryConnectionConfigFromConfig, - createBigQueryLiveDatabaseIntrospection, - isKtxBigQueryConnectionConfig, - type KtxBigQueryClient, - KtxBigQueryScanConnector, - type KtxBigQueryClientFactory, - type KtxBigQueryDataset, - type KtxBigQueryQueryJob, - type KtxBigQueryTableRef, -} from './index.js'; +import { bigQueryConnectionConfigFromConfig, isKtxBigQueryConnectionConfig, type KtxBigQueryClient, KtxBigQueryScanConnector, type KtxBigQueryClientFactory, type KtxBigQueryDataset, type KtxBigQueryQueryJob, type KtxBigQueryTableRef } from '../../connectors/bigquery/connector.js'; +import { createBigQueryLiveDatabaseIntrospection } from '../../connectors/bigquery/live-database-introspection.js'; function fakeClientFactory(): KtxBigQueryClientFactory { const queryResults = vi.fn(async (): ReturnType => [ diff --git a/packages/connector-bigquery/src/connector.ts b/packages/cli/src/connectors/bigquery/connector.ts similarity index 96% rename from packages/connector-bigquery/src/connector.ts rename to packages/cli/src/connectors/bigquery/connector.ts index c69d6030..1c6c964b 100644 --- a/packages/connector-bigquery/src/connector.ts +++ b/packages/cli/src/connectors/bigquery/connector.ts @@ -1,24 +1,6 @@ import { BigQuery, type TableField } from '@google-cloud/bigquery'; -import { assertReadOnlySql, limitSqlForExecution } from '@ktx/context/connections'; -import { - createKtxConnectorCapabilities, - type KtxColumnSampleInput, - type KtxColumnSampleResult, - type KtxColumnStatsInput, - type KtxColumnStatsResult, - type KtxQueryResult, - type KtxReadOnlyQueryInput, - type KtxScanConnector, - type KtxScanContext, - type KtxScanInput, - type KtxSchemaColumn, - type KtxSchemaSnapshot, - type KtxSchemaTable, - type KtxTableListEntry, - type KtxTableRef, - type KtxTableSampleInput, - type KtxTableSampleResult, -} from '@ktx/context/scan'; +import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; +import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js'; import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; @@ -57,12 +39,14 @@ export interface KtxBigQueryColumnDistinctValuesResult { cardinality: number; } +/** @internal */ export interface KtxBigQueryQueryJob { getQueryResults(): Promise< [Array>, unknown, { schema?: { fields?: TableField[] } }?, ...unknown[]] >; } +/** @internal */ export interface KtxBigQueryTableRef { id?: string; metadata?: { type?: string }; @@ -81,6 +65,7 @@ export interface KtxBigQueryTableRef { >; } +/** @internal */ export interface KtxBigQueryDataset { get(): Promise; getTables(): Promise<[KtxBigQueryTableRef[], ...unknown[]]>; @@ -223,6 +208,7 @@ export function isKtxBigQueryConnectionConfig( return String(connection?.driver ?? '').toLowerCase() === 'bigquery'; } +/** @internal */ export function bigQueryConnectionConfigFromConfig(input: { connectionId: string; connection: KtxBigQueryConnectionConfig | undefined; diff --git a/packages/connector-bigquery/src/dialect.test.ts b/packages/cli/src/connectors/bigquery/dialect.test.ts similarity index 100% rename from packages/connector-bigquery/src/dialect.test.ts rename to packages/cli/src/connectors/bigquery/dialect.test.ts diff --git a/packages/connector-bigquery/src/dialect.ts b/packages/cli/src/connectors/bigquery/dialect.ts similarity index 98% rename from packages/connector-bigquery/src/dialect.ts rename to packages/cli/src/connectors/bigquery/dialect.ts index f4cf2cd2..02d904ed 100644 --- a/packages/connector-bigquery/src/dialect.ts +++ b/packages/cli/src/connectors/bigquery/dialect.ts @@ -1,4 +1,4 @@ -import type { KtxSchemaDimensionType, KtxTableRef } from '@ktx/context/scan'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type BigQueryTableNameRef = Pick & Partial>; diff --git a/packages/connector-bigquery/src/live-database-introspection.ts b/packages/cli/src/connectors/bigquery/live-database-introspection.ts similarity index 84% rename from packages/connector-bigquery/src/live-database-introspection.ts rename to packages/cli/src/connectors/bigquery/live-database-introspection.ts index 79271e0e..5e854b9e 100644 --- a/packages/connector-bigquery/src/live-database-introspection.ts +++ b/packages/cli/src/connectors/bigquery/live-database-introspection.ts @@ -1,5 +1,5 @@ -import type { LiveDatabaseIntrospectionPort } from '@ktx/context/ingest'; -import type { KtxProjectConnectionConfig } from '@ktx/context/project'; +import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; import { KtxBigQueryScanConnector, type KtxBigQueryClientFactory, diff --git a/packages/connector-clickhouse/src/connector.test.ts b/packages/cli/src/connectors/clickhouse/connector.test.ts similarity index 96% rename from packages/connector-clickhouse/src/connector.test.ts rename to packages/cli/src/connectors/clickhouse/connector.test.ts index 4f8b7f52..6ff60299 100644 --- a/packages/connector-clickhouse/src/connector.test.ts +++ b/packages/cli/src/connectors/clickhouse/connector.test.ts @@ -1,11 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { - clickHouseClientConfigFromConfig, - createClickHouseLiveDatabaseIntrospection, - isKtxClickHouseConnectionConfig, - KtxClickHouseScanConnector, - type KtxClickHouseClientFactory, -} from './index.js'; +import { clickHouseClientConfigFromConfig, isKtxClickHouseConnectionConfig, KtxClickHouseScanConnector, type KtxClickHouseClientFactory } from '../../connectors/clickhouse/connector.js'; +import { createClickHouseLiveDatabaseIntrospection } from '../../connectors/clickhouse/live-database-introspection.js'; function result(payload: T) { return { diff --git a/packages/connector-clickhouse/src/connector.ts b/packages/cli/src/connectors/clickhouse/connector.ts similarity index 96% rename from packages/connector-clickhouse/src/connector.ts rename to packages/cli/src/connectors/clickhouse/connector.ts index 4b39c943..714ccfb1 100644 --- a/packages/connector-clickhouse/src/connector.ts +++ b/packages/cli/src/connectors/clickhouse/connector.ts @@ -1,24 +1,6 @@ import { createClient } from '@clickhouse/client'; -import { assertReadOnlySql, limitSqlForExecution } from '@ktx/context/connections'; -import { - createKtxConnectorCapabilities, - type KtxColumnSampleInput, - type KtxColumnSampleResult, - type KtxColumnStatsInput, - type KtxColumnStatsResult, - type KtxQueryResult, - type KtxReadOnlyQueryInput, - type KtxScanConnector, - type KtxScanContext, - type KtxScanInput, - type KtxSchemaColumn, - type KtxSchemaSnapshot, - type KtxSchemaTable, - type KtxTableRef, - type KtxTableSampleInput, - type KtxTableListEntry, - type KtxTableSampleResult, -} from '@ktx/context/scan'; +import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; +import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js'; import { readFileSync } from 'node:fs'; import { Agent as HttpsAgent } from 'node:https'; import { homedir } from 'node:os'; @@ -198,6 +180,7 @@ export function isKtxClickHouseConnectionConfig( return String(connection?.driver ?? '').toLowerCase() === 'clickhouse'; } +/** @internal */ export function clickHouseClientConfigFromConfig(input: { connectionId: string; connection: KtxClickHouseConnectionConfig | undefined; diff --git a/packages/connector-clickhouse/src/dialect.test.ts b/packages/cli/src/connectors/clickhouse/dialect.test.ts similarity index 100% rename from packages/connector-clickhouse/src/dialect.test.ts rename to packages/cli/src/connectors/clickhouse/dialect.test.ts diff --git a/packages/connector-clickhouse/src/dialect.ts b/packages/cli/src/connectors/clickhouse/dialect.ts similarity index 98% rename from packages/connector-clickhouse/src/dialect.ts rename to packages/cli/src/connectors/clickhouse/dialect.ts index bff03ae0..48452ea6 100644 --- a/packages/connector-clickhouse/src/dialect.ts +++ b/packages/cli/src/connectors/clickhouse/dialect.ts @@ -1,4 +1,4 @@ -import type { KtxSchemaDimensionType, KtxTableRef } from '@ktx/context/scan'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type ClickHouseTableNameRef = Pick & Partial>; diff --git a/packages/connector-clickhouse/src/live-database-introspection.ts b/packages/cli/src/connectors/clickhouse/live-database-introspection.ts similarity index 86% rename from packages/connector-clickhouse/src/live-database-introspection.ts rename to packages/cli/src/connectors/clickhouse/live-database-introspection.ts index a7b333cd..1e0ec918 100644 --- a/packages/connector-clickhouse/src/live-database-introspection.ts +++ b/packages/cli/src/connectors/clickhouse/live-database-introspection.ts @@ -1,5 +1,5 @@ -import type { LiveDatabaseIntrospectionPort } from '@ktx/context/ingest'; -import type { KtxProjectConnectionConfig } from '@ktx/context/project'; +import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; import { KtxClickHouseScanConnector, type KtxClickHouseClientFactory, diff --git a/packages/connector-mysql/src/connector.test.ts b/packages/cli/src/connectors/mysql/connector.test.ts similarity index 97% rename from packages/connector-mysql/src/connector.test.ts rename to packages/cli/src/connectors/mysql/connector.test.ts index c5c5a3fa..64b576e2 100644 --- a/packages/connector-mysql/src/connector.test.ts +++ b/packages/cli/src/connectors/mysql/connector.test.ts @@ -1,12 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; import type { FieldPacket, RowDataPacket } from 'mysql2/promise'; -import { - createMysqlLiveDatabaseIntrospection, - isKtxMysqlConnectionConfig, - KtxMysqlScanConnector, - mysqlConnectionPoolConfigFromConfig, - type KtxMysqlPoolFactory, -} from './index.js'; +import { createMysqlLiveDatabaseIntrospection } from '../../connectors/mysql/live-database-introspection.js'; +import { isKtxMysqlConnectionConfig, KtxMysqlScanConnector, mysqlConnectionPoolConfigFromConfig, type KtxMysqlPoolFactory } from '../../connectors/mysql/connector.js'; function mysqlResult(rows: Record[], fields: Array<{ name: string; type?: number }>): [RowDataPacket[], FieldPacket[]] { return [rows as RowDataPacket[], fields as FieldPacket[]]; diff --git a/packages/connector-mysql/src/connector.ts b/packages/cli/src/connectors/mysql/connector.ts similarity index 97% rename from packages/connector-mysql/src/connector.ts rename to packages/cli/src/connectors/mysql/connector.ts index 62bb1880..9cf242f5 100644 --- a/packages/connector-mysql/src/connector.ts +++ b/packages/cli/src/connectors/mysql/connector.ts @@ -2,27 +2,8 @@ import mysql, { type FieldPacket, type Pool, type RowDataPacket } from 'mysql2/p import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; -import { assertReadOnlySql, limitSqlForExecution } from '@ktx/context/connections'; -import { - createKtxConnectorCapabilities, - type KtxColumnSampleInput, - type KtxColumnSampleResult, - type KtxColumnStatsInput, - type KtxColumnStatsResult, - type KtxQueryResult, - type KtxReadOnlyQueryInput, - type KtxScanConnector, - type KtxScanContext, - type KtxScanInput, - type KtxSchemaColumn, - type KtxTableListEntry, - type KtxSchemaForeignKey, - type KtxSchemaSnapshot, - type KtxSchemaTable, - type KtxTableRef, - type KtxTableSampleInput, - type KtxTableSampleResult, -} from '@ktx/context/scan'; +import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; +import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxTableListEntry, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js'; import { KtxMysqlDialect } from './dialect.js'; export interface KtxMysqlConnectionConfig { @@ -237,6 +218,7 @@ export function isKtxMysqlConnectionConfig( return String(connection?.driver ?? '').toLowerCase() === 'mysql'; } +/** @internal */ export function mysqlConnectionPoolConfigFromConfig(input: { connectionId: string; connection: KtxMysqlConnectionConfig | undefined; diff --git a/packages/connector-mysql/src/dialect.test.ts b/packages/cli/src/connectors/mysql/dialect.test.ts similarity index 100% rename from packages/connector-mysql/src/dialect.test.ts rename to packages/cli/src/connectors/mysql/dialect.test.ts diff --git a/packages/connector-mysql/src/dialect.ts b/packages/cli/src/connectors/mysql/dialect.ts similarity index 98% rename from packages/connector-mysql/src/dialect.ts rename to packages/cli/src/connectors/mysql/dialect.ts index d641beb6..d61db36c 100644 --- a/packages/connector-mysql/src/dialect.ts +++ b/packages/cli/src/connectors/mysql/dialect.ts @@ -1,4 +1,4 @@ -import type { KtxSchemaDimensionType, KtxTableRef } from '@ktx/context/scan'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type MysqlTableNameRef = Pick & Partial>; diff --git a/packages/connector-mysql/src/live-database-introspection.ts b/packages/cli/src/connectors/mysql/live-database-introspection.ts similarity index 85% rename from packages/connector-mysql/src/live-database-introspection.ts rename to packages/cli/src/connectors/mysql/live-database-introspection.ts index b88d2fa6..ea649761 100644 --- a/packages/connector-mysql/src/live-database-introspection.ts +++ b/packages/cli/src/connectors/mysql/live-database-introspection.ts @@ -1,5 +1,5 @@ -import type { LiveDatabaseIntrospectionPort } from '@ktx/context/ingest'; -import type { KtxProjectConnectionConfig } from '@ktx/context/project'; +import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; import { KtxMysqlScanConnector, type KtxMysqlConnectionConfig, diff --git a/packages/connector-postgres/src/connector.test.ts b/packages/cli/src/connectors/postgres/connector.test.ts similarity index 97% rename from packages/connector-postgres/src/connector.test.ts rename to packages/cli/src/connectors/postgres/connector.test.ts index 8093acda..cf595f5c 100644 --- a/packages/connector-postgres/src/connector.test.ts +++ b/packages/cli/src/connectors/postgres/connector.test.ts @@ -1,11 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { - createPostgresLiveDatabaseIntrospection, - isKtxPostgresConnectionConfig, - KtxPostgresScanConnector, - postgresPoolConfigFromConfig, - type KtxPostgresPoolFactory, -} from './index.js'; +import { createPostgresLiveDatabaseIntrospection } from '../../connectors/postgres/live-database-introspection.js'; +import { isKtxPostgresConnectionConfig, KtxPostgresScanConnector, postgresPoolConfigFromConfig, type KtxPostgresPoolFactory } from '../../connectors/postgres/connector.js'; interface FakeQueryResult { rows: Record[]; diff --git a/packages/connector-postgres/src/connector.ts b/packages/cli/src/connectors/postgres/connector.ts similarity index 97% rename from packages/connector-postgres/src/connector.ts rename to packages/cli/src/connectors/postgres/connector.ts index 7f5ed65c..36a2bda6 100644 --- a/packages/connector-postgres/src/connector.ts +++ b/packages/cli/src/connectors/postgres/connector.ts @@ -1,27 +1,8 @@ import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; -import { assertReadOnlySql, limitSqlForExecution } from '@ktx/context/connections'; -import { - createKtxConnectorCapabilities, - type KtxColumnSampleInput, - type KtxColumnSampleResult, - type KtxColumnStatsInput, - type KtxColumnStatsResult, - type KtxQueryResult, - type KtxReadOnlyQueryInput, - type KtxScanConnector, - type KtxScanContext, - type KtxScanInput, - type KtxSchemaColumn, - type KtxSchemaForeignKey, - type KtxSchemaSnapshot, - type KtxSchemaTable, - type KtxTableListEntry, - type KtxTableRef, - type KtxTableSampleInput, - type KtxTableSampleResult, -} from '@ktx/context/scan'; +import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; +import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js'; import { Pool } from 'pg'; import { KtxPostgresDialect } from './dialect.js'; @@ -297,6 +278,7 @@ export function isKtxPostgresConnectionConfig( return driver === 'postgres' || driver === 'postgresql'; } +/** @internal */ export function postgresPoolConfigFromConfig(input: { connectionId: string; connection: KtxPostgresConnectionConfig | undefined; diff --git a/packages/connector-postgres/src/dialect.test.ts b/packages/cli/src/connectors/postgres/dialect.test.ts similarity index 100% rename from packages/connector-postgres/src/dialect.test.ts rename to packages/cli/src/connectors/postgres/dialect.test.ts diff --git a/packages/connector-postgres/src/dialect.ts b/packages/cli/src/connectors/postgres/dialect.ts similarity index 98% rename from packages/connector-postgres/src/dialect.ts rename to packages/cli/src/connectors/postgres/dialect.ts index 989d780c..ea0590b8 100644 --- a/packages/connector-postgres/src/dialect.ts +++ b/packages/cli/src/connectors/postgres/dialect.ts @@ -1,4 +1,4 @@ -import type { KtxSchemaDimensionType, KtxTableRef } from '@ktx/context/scan'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type PostgresTableNameRef = Pick & Partial>; diff --git a/packages/connector-postgres/src/historic-sql-query-client.test.ts b/packages/cli/src/connectors/postgres/historic-sql-query-client.test.ts similarity index 100% rename from packages/connector-postgres/src/historic-sql-query-client.test.ts rename to packages/cli/src/connectors/postgres/historic-sql-query-client.test.ts diff --git a/packages/connector-postgres/src/historic-sql-query-client.ts b/packages/cli/src/connectors/postgres/historic-sql-query-client.ts similarity index 91% rename from packages/connector-postgres/src/historic-sql-query-client.ts rename to packages/cli/src/connectors/postgres/historic-sql-query-client.ts index b0e620ed..58f0f8eb 100644 --- a/packages/connector-postgres/src/historic-sql-query-client.ts +++ b/packages/cli/src/connectors/postgres/historic-sql-query-client.ts @@ -1,4 +1,4 @@ -import type { KtxPostgresQueryClient } from '@ktx/context/ingest'; +import type { KtxPostgresQueryClient } from '../../context/ingest/adapters/historic-sql/types.js'; import { KtxPostgresScanConnector, type KtxPostgresScanConnectorOptions } from './connector.js'; export type KtxPostgresHistoricSqlQueryClientOptions = KtxPostgresScanConnectorOptions; diff --git a/packages/connector-postgres/src/live-database-introspection.ts b/packages/cli/src/connectors/postgres/live-database-introspection.ts similarity index 85% rename from packages/connector-postgres/src/live-database-introspection.ts rename to packages/cli/src/connectors/postgres/live-database-introspection.ts index ad025676..83e29489 100644 --- a/packages/connector-postgres/src/live-database-introspection.ts +++ b/packages/cli/src/connectors/postgres/live-database-introspection.ts @@ -1,5 +1,5 @@ -import type { LiveDatabaseIntrospectionPort } from '@ktx/context/ingest'; -import type { KtxProjectConnectionConfig } from '@ktx/context/project'; +import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; import { KtxPostgresScanConnector, type KtxPostgresConnectionConfig, diff --git a/packages/connector-snowflake/src/connector.test.ts b/packages/cli/src/connectors/snowflake/connector.test.ts similarity index 96% rename from packages/connector-snowflake/src/connector.test.ts rename to packages/cli/src/connectors/snowflake/connector.test.ts index a49be885..7b2e600f 100644 --- a/packages/connector-snowflake/src/connector.test.ts +++ b/packages/cli/src/connectors/snowflake/connector.test.ts @@ -1,12 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { - createSnowflakeLiveDatabaseIntrospection, - isKtxSnowflakeConnectionConfig, - KtxSnowflakeScanConnector, - snowflakeConnectionConfigFromConfig, - type KtxSnowflakeDriver, - type KtxSnowflakeDriverFactory, -} from './index.js'; +import { createSnowflakeLiveDatabaseIntrospection } from '../../connectors/snowflake/live-database-introspection.js'; +import { isKtxSnowflakeConnectionConfig, KtxSnowflakeScanConnector, snowflakeConnectionConfigFromConfig, type KtxSnowflakeDriver, type KtxSnowflakeDriverFactory } from '../../connectors/snowflake/connector.js'; function fakeDriverFactory(): KtxSnowflakeDriverFactory { const driver: KtxSnowflakeDriver = { diff --git a/packages/connector-snowflake/src/connector.ts b/packages/cli/src/connectors/snowflake/connector.ts similarity index 97% rename from packages/connector-snowflake/src/connector.ts rename to packages/cli/src/connectors/snowflake/connector.ts index 39179369..5b0dfcca 100644 --- a/packages/connector-snowflake/src/connector.ts +++ b/packages/cli/src/connectors/snowflake/connector.ts @@ -2,26 +2,8 @@ import { createPrivateKey } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; -import { assertReadOnlySql, limitSqlForExecution } from '@ktx/context/connections'; -import { - createKtxConnectorCapabilities, - type KtxColumnSampleInput, - type KtxColumnSampleResult, - type KtxColumnStatsInput, - type KtxColumnStatsResult, - type KtxQueryResult, - type KtxReadOnlyQueryInput, - type KtxScanConnector, - type KtxScanContext, - type KtxScanInput, - type KtxSchemaColumn, - type KtxSchemaSnapshot, - type KtxSchemaTable, - type KtxTableRef, - type KtxTableSampleInput, - type KtxTableListEntry, - type KtxTableSampleResult, -} from '@ktx/context/scan'; +import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; +import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js'; import * as snowflake from 'snowflake-sdk'; import { KtxSnowflakeDialect } from './dialect.js'; @@ -196,6 +178,7 @@ export function isKtxSnowflakeConnectionConfig( return String(connection?.driver ?? '').toLowerCase() === 'snowflake'; } +/** @internal */ export function snowflakeConnectionConfigFromConfig(input: { connectionId: string; connection: KtxSnowflakeConnectionConfig | undefined; diff --git a/packages/connector-snowflake/src/dialect.test.ts b/packages/cli/src/connectors/snowflake/dialect.test.ts similarity index 100% rename from packages/connector-snowflake/src/dialect.test.ts rename to packages/cli/src/connectors/snowflake/dialect.test.ts diff --git a/packages/connector-snowflake/src/dialect.ts b/packages/cli/src/connectors/snowflake/dialect.ts similarity index 98% rename from packages/connector-snowflake/src/dialect.ts rename to packages/cli/src/connectors/snowflake/dialect.ts index b105a49d..db508134 100644 --- a/packages/connector-snowflake/src/dialect.ts +++ b/packages/cli/src/connectors/snowflake/dialect.ts @@ -1,4 +1,4 @@ -import type { KtxSchemaDimensionType, KtxTableRef } from '@ktx/context/scan'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type SnowflakeTableNameRef = Pick & Partial>; diff --git a/packages/connector-snowflake/src/live-database-introspection.ts b/packages/cli/src/connectors/snowflake/live-database-introspection.ts similarity index 86% rename from packages/connector-snowflake/src/live-database-introspection.ts rename to packages/cli/src/connectors/snowflake/live-database-introspection.ts index 419fa5e4..58812c1a 100644 --- a/packages/connector-snowflake/src/live-database-introspection.ts +++ b/packages/cli/src/connectors/snowflake/live-database-introspection.ts @@ -1,5 +1,5 @@ -import type { LiveDatabaseIntrospectionPort } from '@ktx/context/ingest'; -import type { KtxProjectConnectionConfig } from '@ktx/context/project'; +import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; import { KtxSnowflakeScanConnector, type KtxSnowflakeConnectionConfig, diff --git a/packages/connector-sqlite/src/connector.test.ts b/packages/cli/src/connectors/sqlite/connector.test.ts similarity index 97% rename from packages/connector-sqlite/src/connector.test.ts rename to packages/cli/src/connectors/sqlite/connector.test.ts index c3c11d64..77ec4b3c 100644 --- a/packages/connector-sqlite/src/connector.test.ts +++ b/packages/cli/src/connectors/sqlite/connector.test.ts @@ -4,12 +4,8 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { - createSqliteLiveDatabaseIntrospection, - isKtxSqliteConnectionConfig, - KtxSqliteScanConnector, - sqliteDatabasePathFromConfig, -} from './index.js'; +import { createSqliteLiveDatabaseIntrospection } from '../../connectors/sqlite/live-database-introspection.js'; +import { isKtxSqliteConnectionConfig, KtxSqliteScanConnector, sqliteDatabasePathFromConfig } from '../../connectors/sqlite/connector.js'; describe('KtxSqliteScanConnector', () => { let tempDir: string; diff --git a/packages/connector-sqlite/src/connector.ts b/packages/cli/src/connectors/sqlite/connector.ts similarity index 94% rename from packages/connector-sqlite/src/connector.ts rename to packages/cli/src/connectors/sqlite/connector.ts index 6979d7ee..e915c776 100644 --- a/packages/connector-sqlite/src/connector.ts +++ b/packages/cli/src/connectors/sqlite/connector.ts @@ -3,25 +3,9 @@ import { existsSync, readFileSync, statSync } from 'node:fs'; import { homedir } from 'node:os'; import { isAbsolute, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { assertReadOnlySql, limitSqlForExecution, normalizeQueryRows } from '@ktx/context/connections'; -import { - createKtxConnectorCapabilities, - type KtxColumnSampleInput, - type KtxColumnSampleResult, - type KtxColumnStatsInput, - type KtxColumnStatsResult, - type KtxQueryResult, - type KtxReadOnlyQueryInput, - type KtxScanConnector, - type KtxScanContext, - type KtxScanInput, - type KtxSchemaForeignKey, - type KtxSchemaSnapshot, - type KtxSchemaTable, - type KtxTableRef, - type KtxTableSampleInput, - type KtxTableSampleResult, -} from '@ktx/context/scan'; +import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; +import { normalizeQueryRows } from '../../context/connections/query-executor.js'; +import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js'; import { KtxSqliteDialect } from './dialect.js'; export interface KtxSqliteConnectionConfig { @@ -31,6 +15,7 @@ export interface KtxSqliteConnectionConfig { [key: string]: unknown; } +/** @internal */ export interface SqliteDatabasePathInput { connectionId: string; projectDir?: string; @@ -142,6 +127,7 @@ export function isKtxSqliteConnectionConfig( return driver === 'sqlite' || driver === 'sqlite3'; } +/** @internal */ export function sqliteDatabasePathFromConfig(input: SqliteDatabasePathInput): string { const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxSqliteConnectionConfig(input.connection)) { diff --git a/packages/connector-sqlite/src/dialect.test.ts b/packages/cli/src/connectors/sqlite/dialect.test.ts similarity index 100% rename from packages/connector-sqlite/src/dialect.test.ts rename to packages/cli/src/connectors/sqlite/dialect.test.ts diff --git a/packages/connector-sqlite/src/dialect.ts b/packages/cli/src/connectors/sqlite/dialect.ts similarity index 98% rename from packages/connector-sqlite/src/dialect.ts rename to packages/cli/src/connectors/sqlite/dialect.ts index a61a1ca1..b5771b62 100644 --- a/packages/connector-sqlite/src/dialect.ts +++ b/packages/cli/src/connectors/sqlite/dialect.ts @@ -1,4 +1,4 @@ -import type { KtxSchemaDimensionType, KtxTableRef } from '@ktx/context/scan'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type SqliteTableNameRef = Pick & Partial>; diff --git a/packages/connector-sqlite/src/live-database-introspection.ts b/packages/cli/src/connectors/sqlite/live-database-introspection.ts similarity index 83% rename from packages/connector-sqlite/src/live-database-introspection.ts rename to packages/cli/src/connectors/sqlite/live-database-introspection.ts index 6e45ac7d..62a1f8c5 100644 --- a/packages/connector-sqlite/src/live-database-introspection.ts +++ b/packages/cli/src/connectors/sqlite/live-database-introspection.ts @@ -1,5 +1,5 @@ -import type { LiveDatabaseIntrospectionPort } from '@ktx/context/ingest'; -import type { KtxProjectConnectionConfig } from '@ktx/context/project'; +import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; import { KtxSqliteScanConnector, type KtxSqliteConnectionConfig } from './connector.js'; export interface CreateSqliteLiveDatabaseIntrospectionOptions { diff --git a/packages/connector-sqlserver/src/connector.test.ts b/packages/cli/src/connectors/sqlserver/connector.test.ts similarity index 97% rename from packages/connector-sqlserver/src/connector.test.ts rename to packages/cli/src/connectors/sqlserver/connector.test.ts index b7915fa8..bd9a8af1 100644 --- a/packages/connector-sqlserver/src/connector.test.ts +++ b/packages/cli/src/connectors/sqlserver/connector.test.ts @@ -1,12 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { - createSqlServerLiveDatabaseIntrospection, - isKtxSqlServerConnectionConfig, - KtxSqlServerScanConnector, - sqlServerConnectionPoolConfigFromConfig, - type KtxSqlServerPoolFactory, - type KtxSqlServerQueryResult, -} from './index.js'; +import { createSqlServerLiveDatabaseIntrospection } from '../../connectors/sqlserver/live-database-introspection.js'; +import { isKtxSqlServerConnectionConfig, KtxSqlServerScanConnector, sqlServerConnectionPoolConfigFromConfig, type KtxSqlServerPoolFactory, type KtxSqlServerQueryResult } from '../../connectors/sqlserver/connector.js'; function recordset>( rows: T[], diff --git a/packages/connector-sqlserver/src/connector.ts b/packages/cli/src/connectors/sqlserver/connector.ts similarity index 97% rename from packages/connector-sqlserver/src/connector.ts rename to packages/cli/src/connectors/sqlserver/connector.ts index 73a46aab..d9c227d7 100644 --- a/packages/connector-sqlserver/src/connector.ts +++ b/packages/cli/src/connectors/sqlserver/connector.ts @@ -1,24 +1,5 @@ -import { assertReadOnlySql } from '@ktx/context/connections'; -import { - createKtxConnectorCapabilities, - type KtxColumnSampleInput, - type KtxColumnSampleResult, - type KtxColumnStatsInput, - type KtxColumnStatsResult, - type KtxQueryResult, - type KtxReadOnlyQueryInput, - type KtxScanConnector, - type KtxScanContext, - type KtxScanInput, - type KtxSchemaColumn, - type KtxSchemaForeignKey, - type KtxSchemaSnapshot, - type KtxSchemaTable, - type KtxTableListEntry, - type KtxTableRef, - type KtxTableSampleInput, - type KtxTableSampleResult, -} from '@ktx/context/scan'; +import { assertReadOnlySql } from '../../context/connections/read-only-sql.js'; +import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js'; import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; @@ -50,6 +31,7 @@ export interface KtxSqlServerPoolConfig { pool: { max: number; min: number; idleTimeoutMillis: number }; } +/** @internal */ export interface KtxSqlServerQueryResult { recordset?: Array> & { columns?: Record }; } @@ -239,6 +221,7 @@ export function isKtxSqlServerConnectionConfig( return String(connection?.driver ?? '').toLowerCase() === 'sqlserver'; } +/** @internal */ export function sqlServerConnectionPoolConfigFromConfig(input: { connectionId: string; connection: KtxSqlServerConnectionConfig | undefined; diff --git a/packages/connector-sqlserver/src/dialect.test.ts b/packages/cli/src/connectors/sqlserver/dialect.test.ts similarity index 100% rename from packages/connector-sqlserver/src/dialect.test.ts rename to packages/cli/src/connectors/sqlserver/dialect.test.ts diff --git a/packages/connector-sqlserver/src/dialect.ts b/packages/cli/src/connectors/sqlserver/dialect.ts similarity index 98% rename from packages/connector-sqlserver/src/dialect.ts rename to packages/cli/src/connectors/sqlserver/dialect.ts index 67897876..8444317d 100644 --- a/packages/connector-sqlserver/src/dialect.ts +++ b/packages/cli/src/connectors/sqlserver/dialect.ts @@ -1,4 +1,4 @@ -import type { KtxSchemaDimensionType, KtxTableRef } from '@ktx/context/scan'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type SqlServerTableNameRef = Pick & Partial>; diff --git a/packages/connector-sqlserver/src/live-database-introspection.ts b/packages/cli/src/connectors/sqlserver/live-database-introspection.ts similarity index 86% rename from packages/connector-sqlserver/src/live-database-introspection.ts rename to packages/cli/src/connectors/sqlserver/live-database-introspection.ts index 7c7a2b53..6bd54ba1 100644 --- a/packages/connector-sqlserver/src/live-database-introspection.ts +++ b/packages/cli/src/connectors/sqlserver/live-database-introspection.ts @@ -1,5 +1,5 @@ -import type { LiveDatabaseIntrospectionPort } from '@ktx/context/ingest'; -import type { KtxProjectConnectionConfig } from '@ktx/context/project'; +import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; import { KtxSqlServerScanConnector, type KtxSqlServerConnectionConfig, diff --git a/packages/cli/src/context-build-view.test.ts b/packages/cli/src/context-build-view.test.ts index efe2f445..c1550219 100644 --- a/packages/cli/src/context-build-view.test.ts +++ b/packages/cli/src/context-build-view.test.ts @@ -1,4 +1,4 @@ -import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from '@ktx/context/project'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; import { describe, expect, it, vi } from 'vitest'; import type { KtxPublicIngestProject, KtxPublicIngestTargetResult } from './public-ingest.js'; import { diff --git a/packages/cli/src/context-build-view.ts b/packages/cli/src/context-build-view.ts index 7f47e33f..c734c6b9 100644 --- a/packages/cli/src/context-build-view.ts +++ b/packages/cli/src/context-build-view.ts @@ -1,4 +1,4 @@ -import type { KtxProgressPort, KtxProgressUpdateOptions } from '@ktx/context/scan'; +import type { KtxProgressPort, KtxProgressUpdateOptions } from './context/scan/types.js'; import type { KtxCliIo } from './index.js'; import type { KtxIngestProgressUpdate } from './ingest.js'; import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; @@ -444,17 +444,20 @@ export function renderContextBuildView( const ESC_K_RE = new RegExp(`${ESC.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\[K`, 'g'); const ANSI_RE = /\x1b\[[0-9;]*m/g; +/** @internal */ export function extractProgressMessage(chunk: string): string | null { const cleaned = chunk.replace(/^\r/, '').replace(ESC_K_RE, '').replace(/\n$/, '').trim(); const match = cleaned.match(/^\[(\d+)%\]\s*(.+)$/); return match ? `[${match[1]}%] ${match[2]}` : null; } +/** @internal */ export function parseScanSummary(output: string): string | null { const match = output.match(/(\d+) changes? across (\d+) tables?/); return match ? `${match[2]} tables` : null; } +/** @internal */ export function parseIngestSummary(output: string): string | null { const savedMemory = output.match(/Saved memory: (.+)/); if (savedMemory) return savedMemory[1]; @@ -560,6 +563,7 @@ function collectSourceProgress(targets: ContextBuildTargetState[]): ContextBuild }); } +/** @internal */ export function viewStateFromSourceProgress( sources: ContextBuildSourceProgressUpdate[], now: number, diff --git a/packages/context/src/connections/connection-type.ts b/packages/cli/src/context/connections/connection-type.ts similarity index 100% rename from packages/context/src/connections/connection-type.ts rename to packages/cli/src/context/connections/connection-type.ts diff --git a/packages/context/src/connections/dialects.test.ts b/packages/cli/src/context/connections/dialects.test.ts similarity index 100% rename from packages/context/src/connections/dialects.test.ts rename to packages/cli/src/context/connections/dialects.test.ts diff --git a/packages/context/src/connections/dialects.ts b/packages/cli/src/context/connections/dialects.ts similarity index 99% rename from packages/context/src/connections/dialects.ts rename to packages/cli/src/context/connections/dialects.ts index afac4bd2..75a8ae4c 100644 --- a/packages/context/src/connections/dialects.ts +++ b/packages/cli/src/context/connections/dialects.ts @@ -1,6 +1,6 @@ import type { KtxSchemaDimensionType, KtxTableRef } from '../scan/types.js'; -export type SupportedDriver = +type SupportedDriver = | 'postgres' | 'postgresql' | 'mysql' diff --git a/packages/context/src/connections/local-query-executor.test.ts b/packages/cli/src/context/connections/local-query-executor.test.ts similarity index 100% rename from packages/context/src/connections/local-query-executor.test.ts rename to packages/cli/src/context/connections/local-query-executor.test.ts diff --git a/packages/context/src/connections/local-query-executor.ts b/packages/cli/src/context/connections/local-query-executor.ts similarity index 100% rename from packages/context/src/connections/local-query-executor.ts rename to packages/cli/src/context/connections/local-query-executor.ts diff --git a/packages/context/src/connections/local-warehouse-descriptor.test.ts b/packages/cli/src/context/connections/local-warehouse-descriptor.test.ts similarity index 100% rename from packages/context/src/connections/local-warehouse-descriptor.test.ts rename to packages/cli/src/context/connections/local-warehouse-descriptor.test.ts diff --git a/packages/context/src/connections/local-warehouse-descriptor.ts b/packages/cli/src/context/connections/local-warehouse-descriptor.ts similarity index 100% rename from packages/context/src/connections/local-warehouse-descriptor.ts rename to packages/cli/src/context/connections/local-warehouse-descriptor.ts diff --git a/packages/context/src/connections/notion-config.test.ts b/packages/cli/src/context/connections/notion-config.test.ts similarity index 100% rename from packages/context/src/connections/notion-config.test.ts rename to packages/cli/src/context/connections/notion-config.test.ts diff --git a/packages/context/src/connections/notion-config.ts b/packages/cli/src/context/connections/notion-config.ts similarity index 98% rename from packages/context/src/connections/notion-config.ts rename to packages/cli/src/context/connections/notion-config.ts index 24dd5d4b..79ebe23d 100644 --- a/packages/context/src/connections/notion-config.ts +++ b/packages/cli/src/context/connections/notion-config.ts @@ -8,7 +8,7 @@ import { } from '../ingest/adapters/notion/types.js'; import type { KtxProjectConnectionConfig } from '../project/config.js'; -export const KTX_NOTION_ORG_KNOWLEDGE_WARNING = +const KTX_NOTION_ORG_KNOWLEDGE_WARNING = 'Anything accessible to this Notion integration can become organization knowledge.'; type KtxNotionCrawlMode = 'all_accessible' | 'selected_roots'; @@ -39,6 +39,7 @@ export type KtxNotionConnectionConfig = Omit< max_knowledge_updates_per_run: number; }; +/** @internal */ export interface RedactedKtxNotionConnectionConfig { driver: 'notion'; hasAuthToken: boolean; @@ -152,6 +153,7 @@ export function parseNotionConnectionConfig(raw: unknown): KtxNotionConnectionCo }; } +/** @internal */ export function redactNotionConnectionConfig(config: KtxNotionConnectionConfig): RedactedKtxNotionConnectionConfig { return { driver: 'notion', @@ -171,6 +173,7 @@ function expandHome(path: string): string { return path === '~' || path.startsWith('~/') ? resolve(homedir(), path.slice(2)) : path; } +/** @internal */ export async function resolveNotionAuthToken( authTokenRef: string, options: ResolveNotionTokenOptions = {}, diff --git a/packages/context/src/connections/postgres-query-executor.test.ts b/packages/cli/src/context/connections/postgres-query-executor.test.ts similarity index 100% rename from packages/context/src/connections/postgres-query-executor.test.ts rename to packages/cli/src/context/connections/postgres-query-executor.test.ts diff --git a/packages/context/src/connections/postgres-query-executor.ts b/packages/cli/src/context/connections/postgres-query-executor.ts similarity index 100% rename from packages/context/src/connections/postgres-query-executor.ts rename to packages/cli/src/context/connections/postgres-query-executor.ts diff --git a/packages/context/src/connections/query-executor.ts b/packages/cli/src/context/connections/query-executor.ts similarity index 88% rename from packages/context/src/connections/query-executor.ts rename to packages/cli/src/context/connections/query-executor.ts index d3ca68bc..e169d164 100644 --- a/packages/context/src/connections/query-executor.ts +++ b/packages/cli/src/context/connections/query-executor.ts @@ -1,4 +1,4 @@ -import type { KtxProjectConnectionConfig } from '../project/index.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; export interface KtxSqlQueryExecutionInput { connectionId: string; diff --git a/packages/context/src/connections/read-only-sql.test.ts b/packages/cli/src/context/connections/read-only-sql.test.ts similarity index 100% rename from packages/context/src/connections/read-only-sql.test.ts rename to packages/cli/src/context/connections/read-only-sql.test.ts diff --git a/packages/context/src/connections/read-only-sql.ts b/packages/cli/src/context/connections/read-only-sql.ts similarity index 100% rename from packages/context/src/connections/read-only-sql.ts rename to packages/cli/src/context/connections/read-only-sql.ts diff --git a/packages/context/src/connections/sqlite-query-executor.test.ts b/packages/cli/src/context/connections/sqlite-query-executor.test.ts similarity index 100% rename from packages/context/src/connections/sqlite-query-executor.test.ts rename to packages/cli/src/context/connections/sqlite-query-executor.test.ts diff --git a/packages/context/src/connections/sqlite-query-executor.ts b/packages/cli/src/context/connections/sqlite-query-executor.ts similarity index 99% rename from packages/context/src/connections/sqlite-query-executor.ts rename to packages/cli/src/context/connections/sqlite-query-executor.ts index 2a87ef7d..22c69005 100644 --- a/packages/context/src/connections/sqlite-query-executor.ts +++ b/packages/cli/src/context/connections/sqlite-query-executor.ts @@ -49,6 +49,7 @@ function sqlitePathFromUrl(url: string): string { return url; } +/** @internal */ export function sqliteDatabasePathFromConnection(input: KtxSqlQueryExecutionInput): string { const driver = connectionDriver(input); if (driver !== 'sqlite' && driver !== 'sqlite3') { diff --git a/packages/context/src/core/config-reference.test.ts b/packages/cli/src/context/core/config-reference.test.ts similarity index 100% rename from packages/context/src/core/config-reference.test.ts rename to packages/cli/src/context/core/config-reference.test.ts diff --git a/packages/context/src/core/config-reference.ts b/packages/cli/src/context/core/config-reference.ts similarity index 98% rename from packages/context/src/core/config-reference.ts rename to packages/cli/src/context/core/config-reference.ts index 7aa69e69..f1f235ca 100644 --- a/packages/context/src/core/config-reference.ts +++ b/packages/cli/src/context/core/config-reference.ts @@ -2,6 +2,7 @@ import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; +/** @internal */ export function resolveKtxHomePath(path: string): string { if (path === '~') { return homedir(); diff --git a/packages/context/src/core/config.ts b/packages/cli/src/context/core/config.ts similarity index 93% rename from packages/context/src/core/config.ts rename to packages/cli/src/context/core/config.ts index 21e5a277..d61d98d2 100644 --- a/packages/context/src/core/config.ts +++ b/packages/cli/src/context/core/config.ts @@ -1,10 +1,10 @@ -export interface KtxStorageConfig { +interface KtxStorageConfig { configDir?: string; homeDir?: string; worktreesDir?: string; } -export interface KtxGitConfig { +interface KtxGitConfig { userName: string; userEmail: string; bootstrapMessage?: string; diff --git a/packages/context/src/core/embedding.ts b/packages/cli/src/context/core/embedding.ts similarity index 100% rename from packages/context/src/core/embedding.ts rename to packages/cli/src/context/core/embedding.ts diff --git a/packages/context/src/core/file-store.ts b/packages/cli/src/context/core/file-store.ts similarity index 100% rename from packages/context/src/core/file-store.ts rename to packages/cli/src/context/core/file-store.ts diff --git a/packages/context/src/core/git-env.ts b/packages/cli/src/context/core/git-env.ts similarity index 100% rename from packages/context/src/core/git-env.ts rename to packages/cli/src/context/core/git-env.ts diff --git a/packages/context/src/core/git.service.assert-worktree-clean.test.ts b/packages/cli/src/context/core/git.service.assert-worktree-clean.test.ts similarity index 100% rename from packages/context/src/core/git.service.assert-worktree-clean.test.ts rename to packages/cli/src/context/core/git.service.assert-worktree-clean.test.ts diff --git a/packages/context/src/core/git.service.delete-directories.test.ts b/packages/cli/src/context/core/git.service.delete-directories.test.ts similarity index 100% rename from packages/context/src/core/git.service.delete-directories.test.ts rename to packages/cli/src/context/core/git.service.delete-directories.test.ts diff --git a/packages/context/src/core/git.service.patch.test.ts b/packages/cli/src/context/core/git.service.patch.test.ts similarity index 100% rename from packages/context/src/core/git.service.patch.test.ts rename to packages/cli/src/context/core/git.service.patch.test.ts diff --git a/packages/context/src/core/git.service.reset-hard.test.ts b/packages/cli/src/context/core/git.service.reset-hard.test.ts similarity index 100% rename from packages/context/src/core/git.service.reset-hard.test.ts rename to packages/cli/src/context/core/git.service.reset-hard.test.ts diff --git a/packages/context/src/core/git.service.test.ts b/packages/cli/src/context/core/git.service.test.ts similarity index 100% rename from packages/context/src/core/git.service.test.ts rename to packages/cli/src/context/core/git.service.test.ts diff --git a/packages/context/src/core/git.service.ts b/packages/cli/src/context/core/git.service.ts similarity index 100% rename from packages/context/src/core/git.service.ts rename to packages/cli/src/context/core/git.service.ts diff --git a/packages/context/src/core/redaction.ts b/packages/cli/src/context/core/redaction.ts similarity index 98% rename from packages/context/src/core/redaction.ts rename to packages/cli/src/context/core/redaction.ts index 87241742..0ba5bab5 100644 --- a/packages/context/src/core/redaction.ts +++ b/packages/cli/src/context/core/redaction.ts @@ -1,3 +1,4 @@ +/** @internal */ export const REDACTED_KTX_CREDENTIAL_VALUE = ''; const SENSITIVE_FIELD_NAME = /(password|secret|token|api[_-]?key|private[_-]?key|passphrase|credential|authorization|url)/i; diff --git a/packages/context/src/core/session-worktree.service.test.ts b/packages/cli/src/context/core/session-worktree.service.test.ts similarity index 100% rename from packages/context/src/core/session-worktree.service.test.ts rename to packages/cli/src/context/core/session-worktree.service.test.ts diff --git a/packages/context/src/core/session-worktree.service.ts b/packages/cli/src/context/core/session-worktree.service.ts similarity index 98% rename from packages/context/src/core/session-worktree.service.ts rename to packages/cli/src/context/core/session-worktree.service.ts index f001d58b..4667478b 100644 --- a/packages/context/src/core/session-worktree.service.ts +++ b/packages/cli/src/context/core/session-worktree.service.ts @@ -5,7 +5,7 @@ import { GitService } from './git.service.js'; export type SessionOutcome = 'success' | 'empty' | 'conflict' | 'crash'; -export interface SentinelPayload { +interface SentinelPayload { outcome: SessionOutcome; at: string; chatId: string; diff --git a/packages/context/src/daemon/semantic-layer-compute.test.ts b/packages/cli/src/context/daemon/semantic-layer-compute.test.ts similarity index 100% rename from packages/context/src/daemon/semantic-layer-compute.test.ts rename to packages/cli/src/context/daemon/semantic-layer-compute.test.ts diff --git a/packages/context/src/daemon/semantic-layer-compute.ts b/packages/cli/src/context/daemon/semantic-layer-compute.ts similarity index 93% rename from packages/context/src/daemon/semantic-layer-compute.ts rename to packages/cli/src/context/daemon/semantic-layer-compute.ts index a48239eb..f416b169 100644 --- a/packages/context/src/daemon/semantic-layer-compute.ts +++ b/packages/cli/src/context/daemon/semantic-layer-compute.ts @@ -4,21 +4,21 @@ import { URL } from 'node:url'; import { spawn } from 'node:child_process'; import type { ResolvedSemanticLayerSource, SemanticLayerQueryInput } from '../sl/types.js'; -export interface KtxSemanticLayerComputeQueryResult { +interface KtxSemanticLayerComputeQueryResult { sql: string; dialect: string; columns: Array>; plan: Record; } -export interface KtxSemanticLayerComputeValidationResult { +interface KtxSemanticLayerComputeValidationResult { valid: boolean; errors: string[]; warnings: string[]; perSourceWarnings: Record; } -export interface KtxSemanticLayerSourceGenerationColumnInput { +interface KtxSemanticLayerSourceGenerationColumnInput { name: string; type: string; primaryKey?: boolean; @@ -26,7 +26,7 @@ export interface KtxSemanticLayerSourceGenerationColumnInput { comment?: string | null; } -export interface KtxSemanticLayerSourceGenerationTableInput { +interface KtxSemanticLayerSourceGenerationTableInput { name: string; catalog?: string | null; db?: string | null; @@ -34,7 +34,7 @@ export interface KtxSemanticLayerSourceGenerationTableInput { columns: KtxSemanticLayerSourceGenerationColumnInput[]; } -export interface KtxSemanticLayerSourceGenerationLinkInput { +interface KtxSemanticLayerSourceGenerationLinkInput { fromTable: string; fromColumn: string; toTable: string; @@ -42,13 +42,13 @@ export interface KtxSemanticLayerSourceGenerationLinkInput { relationshipType: string; } -export interface KtxSemanticLayerSourceGenerationInput { +interface KtxSemanticLayerSourceGenerationInput { tables: KtxSemanticLayerSourceGenerationTableInput[]; links: KtxSemanticLayerSourceGenerationLinkInput[]; dialect?: string; } -export interface KtxSemanticLayerSourceGenerationResult { +interface KtxSemanticLayerSourceGenerationResult { sources: Array>; sourceCount: number; } @@ -75,14 +75,14 @@ export interface KtxSemanticLayerComputePort { generateSources(input: KtxSemanticLayerSourceGenerationInput): Promise; } -export type KtxDaemonCommand = 'semantic-query' | 'semantic-validate' | 'semantic-generate-sources'; +type KtxDaemonCommand = 'semantic-query' | 'semantic-validate' | 'semantic-generate-sources'; -export type KtxDaemonJsonRunner = ( +type KtxDaemonJsonRunner = ( subcommand: KtxDaemonCommand, payload: Record, ) => Promise>; -export type KtxDaemonHttpJsonRunner = (path: string, payload: Record) => Promise>; +type KtxDaemonHttpJsonRunner = (path: string, payload: Record) => Promise>; export interface PythonSemanticLayerComputeOptions { command?: string; @@ -92,6 +92,7 @@ export interface PythonSemanticLayerComputeOptions { runJson?: KtxDaemonJsonRunner; } +/** @internal */ export interface HttpSemanticLayerComputeOptions { baseUrl: string; requestJson?: KtxDaemonHttpJsonRunner; @@ -272,6 +273,7 @@ export function createPythonSemanticLayerComputePort( }; } +/** @internal */ export function createHttpSemanticLayerComputePort( options: HttpSemanticLayerComputeOptions, ): KtxSemanticLayerComputePort { diff --git a/packages/context/src/index-sync/reindex.test.ts b/packages/cli/src/context/index-sync/reindex.test.ts similarity index 98% rename from packages/context/src/index-sync/reindex.test.ts rename to packages/cli/src/context/index-sync/reindex.test.ts index beb62342..90c6a178 100644 --- a/packages/context/src/index-sync/reindex.test.ts +++ b/packages/cli/src/context/index-sync/reindex.test.ts @@ -2,8 +2,8 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import type { KtxEmbeddingPort } from '../core/index.js'; -import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../project/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { SqliteKnowledgeIndex } from '../wiki/sqlite-knowledge-index.js'; import { reindexLocalIndexes } from './reindex.js'; diff --git a/packages/context/src/index-sync/reindex.ts b/packages/cli/src/context/index-sync/reindex.ts similarity index 89% rename from packages/context/src/index-sync/reindex.ts rename to packages/cli/src/context/index-sync/reindex.ts index d0cbe29a..14252a5e 100644 --- a/packages/context/src/index-sync/reindex.ts +++ b/packages/cli/src/context/index-sync/reindex.ts @@ -1,8 +1,12 @@ import { readdir, stat } from 'node:fs/promises'; import { join, relative } from 'node:path'; -import { ktxLocalStateDbPath, type KtxLocalProject } from '../project/index.js'; -import { loadLocalSlSourceRecords, SlSearchService, SqliteSlSourcesIndex } from '../sl/index.js'; -import { KnowledgeWikiService, SqliteKnowledgeIndex } from '../wiki/index.js'; +import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { loadLocalSlSourceRecords } from '../../context/sl/local-sl.js'; +import { SlSearchService } from '../../context/sl/sl-search.service.js'; +import { SqliteSlSourcesIndex } from '../../context/sl/sqlite-sl-sources-index.js'; +import { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; +import { SqliteKnowledgeIndex } from '../../context/wiki/sqlite-knowledge-index.js'; import type { ReindexOptions, ReindexScopeResult, ReindexSummary, ReindexWorkResult } from './types.js'; type DiscoveredScope = @@ -41,7 +45,7 @@ async function childDirectories(path: string): Promise { } } -export async function discoverReindexScopes(project: KtxLocalProject): Promise { +async function discoverReindexScopes(project: KtxLocalProject): Promise { const scopes: DiscoveredScope[] = []; if (await directoryExists(join(project.projectDir, 'wiki/global'))) { scopes.push({ kind: 'wiki', scope: 'GLOBAL', scopeId: null, label: 'global' }); diff --git a/packages/context/src/index-sync/types.ts b/packages/cli/src/context/index-sync/types.ts similarity index 90% rename from packages/context/src/index-sync/types.ts rename to packages/cli/src/context/index-sync/types.ts index 39970b57..5a038182 100644 --- a/packages/context/src/index-sync/types.ts +++ b/packages/cli/src/context/index-sync/types.ts @@ -1,4 +1,4 @@ -import type { KtxEmbeddingPort } from '../core/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; export interface ReindexOptions { force: boolean; diff --git a/packages/context/src/ingest/action-identity.test.ts b/packages/cli/src/context/ingest/action-identity.test.ts similarity index 100% rename from packages/context/src/ingest/action-identity.test.ts rename to packages/cli/src/context/ingest/action-identity.test.ts diff --git a/packages/context/src/ingest/action-identity.ts b/packages/cli/src/context/ingest/action-identity.ts similarity index 85% rename from packages/context/src/ingest/action-identity.ts rename to packages/cli/src/context/ingest/action-identity.ts index d0111d09..18c25e27 100644 --- a/packages/context/src/ingest/action-identity.ts +++ b/packages/cli/src/context/ingest/action-identity.ts @@ -1,4 +1,4 @@ -import type { MemoryAction } from '../memory/index.js'; +import type { MemoryAction } from '../../context/memory/types.js'; export function actionTargetConnectionId(action: MemoryAction, runConnectionId: string): string { return action.target === 'sl' ? (action.targetConnectionId ?? runConnectionId) : runConnectionId; diff --git a/packages/context/src/ingest/adapters/dbt-descriptions/parse-schema.test.ts b/packages/cli/src/context/ingest/adapters/dbt-descriptions/parse-schema.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/dbt-descriptions/parse-schema.test.ts rename to packages/cli/src/context/ingest/adapters/dbt-descriptions/parse-schema.test.ts diff --git a/packages/context/src/ingest/adapters/dbt-descriptions/parse-schema.ts b/packages/cli/src/context/ingest/adapters/dbt-descriptions/parse-schema.ts similarity index 97% rename from packages/context/src/ingest/adapters/dbt-descriptions/parse-schema.ts rename to packages/cli/src/context/ingest/adapters/dbt-descriptions/parse-schema.ts index a8f07a72..2fb7e7e3 100644 --- a/packages/context/src/ingest/adapters/dbt-descriptions/parse-schema.ts +++ b/packages/cli/src/context/ingest/adapters/dbt-descriptions/parse-schema.ts @@ -1,9 +1,9 @@ import { createHash } from 'node:crypto'; import { parse as parseYaml } from 'yaml'; -import { type KtxLogger, noopLogger } from '../../../core/index.js'; +import { type KtxLogger, noopLogger } from '../../../../context/core/config.js'; import { resolveJinjaVariables } from '../../dbt-shared/project-vars.js'; -export interface DbtParsedColumn { +interface DbtParsedColumn { name: string; description: string | null; dataType: string | null; @@ -12,20 +12,20 @@ export interface DbtParsedColumn { enumValuesDbt?: string[]; } -export interface DbtDataTestRef { +interface DbtDataTestRef { name: string; package: string; kwargs?: Record; } -export interface DbtColumnConstraints { +interface DbtColumnConstraints { dbt: { not_null?: boolean; unique?: boolean; }; } -export interface DbtParsedRelationship { +interface DbtParsedRelationship { fromTable: string; fromColumn: string; toTable: string; @@ -35,7 +35,7 @@ export interface DbtParsedRelationship { description?: string; } -export interface DbtParsedTable { +interface DbtParsedTable { name: string; description: string | null; database: string | null; @@ -126,6 +126,7 @@ type DbtSchemaDataTest = [key: string]: unknown; }; +/** @internal */ export function parseDbtSchemaFile(content: string, options: ParseDbtSchemaOptions = {}): DbtSchemaParseResult { return new DbtSchemaParser(options.logger ?? noopLogger).parseFile(content, options); } @@ -138,13 +139,6 @@ export function parseDbtSchemaFiles( return new DbtSchemaParser(options.logger ?? noopLogger).parseFiles(files, variables, options.projectName ?? null); } -export function computeDbtSchemaHash(files: DbtSchemaFile[]): string { - const combined = [...files] - .sort((a, b) => a.path.localeCompare(b.path)) - .map((file) => `${file.path}:${file.content}`) - .join('\n'); - return createHash('sha256').update(combined).digest('hex').substring(0, 16); -} class DbtSchemaParser { constructor(private readonly logger: KtxLogger) {} diff --git a/packages/context/src/ingest/adapters/dbt/chunk.test.ts b/packages/cli/src/context/ingest/adapters/dbt/chunk.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/dbt/chunk.test.ts rename to packages/cli/src/context/ingest/adapters/dbt/chunk.test.ts diff --git a/packages/context/src/ingest/adapters/dbt/chunk.ts b/packages/cli/src/context/ingest/adapters/dbt/chunk.ts similarity index 100% rename from packages/context/src/ingest/adapters/dbt/chunk.ts rename to packages/cli/src/context/ingest/adapters/dbt/chunk.ts diff --git a/packages/context/src/ingest/adapters/dbt/dbt.adapter.test.ts b/packages/cli/src/context/ingest/adapters/dbt/dbt.adapter.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/dbt/dbt.adapter.test.ts rename to packages/cli/src/context/ingest/adapters/dbt/dbt.adapter.test.ts diff --git a/packages/context/src/ingest/adapters/dbt/dbt.adapter.ts b/packages/cli/src/context/ingest/adapters/dbt/dbt.adapter.ts similarity index 100% rename from packages/context/src/ingest/adapters/dbt/dbt.adapter.ts rename to packages/cli/src/context/ingest/adapters/dbt/dbt.adapter.ts diff --git a/packages/context/src/ingest/adapters/dbt/detect.ts b/packages/cli/src/context/ingest/adapters/dbt/detect.ts similarity index 100% rename from packages/context/src/ingest/adapters/dbt/detect.ts rename to packages/cli/src/context/ingest/adapters/dbt/detect.ts diff --git a/packages/context/src/ingest/adapters/dbt/fetch.test.ts b/packages/cli/src/context/ingest/adapters/dbt/fetch.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/dbt/fetch.test.ts rename to packages/cli/src/context/ingest/adapters/dbt/fetch.test.ts diff --git a/packages/context/src/ingest/adapters/dbt/fetch.ts b/packages/cli/src/context/ingest/adapters/dbt/fetch.ts similarity index 100% rename from packages/context/src/ingest/adapters/dbt/fetch.ts rename to packages/cli/src/context/ingest/adapters/dbt/fetch.ts diff --git a/packages/context/src/ingest/adapters/dbt/parse.test.ts b/packages/cli/src/context/ingest/adapters/dbt/parse.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/dbt/parse.test.ts rename to packages/cli/src/context/ingest/adapters/dbt/parse.test.ts diff --git a/packages/context/src/ingest/adapters/dbt/parse.ts b/packages/cli/src/context/ingest/adapters/dbt/parse.ts similarity index 98% rename from packages/context/src/ingest/adapters/dbt/parse.ts rename to packages/cli/src/context/ingest/adapters/dbt/parse.ts index 2402df36..3465c4ad 100644 --- a/packages/context/src/ingest/adapters/dbt/parse.ts +++ b/packages/cli/src/context/ingest/adapters/dbt/parse.ts @@ -3,6 +3,7 @@ import { join, relative } from 'node:path'; const YAML_EXT_RE = /\.(ya?ml)$/i; +/** @internal */ export function normalizeDbtPath(path: string): string { return path.replaceAll('\\', '/'); } diff --git a/packages/context/src/ingest/adapters/fake/fake.adapter.ts b/packages/cli/src/context/ingest/adapters/fake/fake.adapter.ts similarity index 100% rename from packages/context/src/ingest/adapters/fake/fake.adapter.ts rename to packages/cli/src/context/ingest/adapters/fake/fake.adapter.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts b/packages/cli/src/context/ingest/adapters/historic-sql/bigquery-query-history-reader.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/bigquery-query-history-reader.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/buckets.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/buckets.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/buckets.test.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/buckets.test.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/buckets.ts b/packages/cli/src/context/ingest/adapters/historic-sql/buckets.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/buckets.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/buckets.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/chunk-unified.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/chunk-unified.test.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts b/packages/cli/src/context/ingest/adapters/historic-sql/chunk-unified.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/chunk-unified.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/detect.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/detect.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/detect.test.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/detect.test.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/detect.ts b/packages/cli/src/context/ingest/adapters/historic-sql/detect.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/detect.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/detect.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/errors.ts b/packages/cli/src/context/ingest/adapters/historic-sql/errors.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/errors.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/errors.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/evidence-tool.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/evidence-tool.test.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts b/packages/cli/src/context/ingest/adapters/historic-sql/evidence-tool.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/evidence-tool.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/evidence.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/evidence.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/evidence.test.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/evidence.test.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/evidence.ts b/packages/cli/src/context/ingest/adapters/historic-sql/evidence.ts similarity index 88% rename from packages/context/src/ingest/adapters/historic-sql/evidence.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/evidence.ts index 18c01a85..ddf26aed 100644 --- a/packages/context/src/ingest/adapters/historic-sql/evidence.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/evidence.ts @@ -9,6 +9,7 @@ function safeEvidenceSegment(value: string): string { return segment; } +/** @internal */ export const historicSqlTableUsageEvidenceSchema = z.object({ kind: z.literal('table_usage'), connectionId: z.string().min(1), @@ -16,15 +17,14 @@ export const historicSqlTableUsageEvidenceSchema = z.object({ rawPath: z.string().min(1), usage: tableUsageOutputSchema, }); -export type HistoricSqlTableUsageEvidence = z.infer; +/** @internal */ export const historicSqlPatternEvidenceSchema = z.object({ kind: z.literal('pattern'), connectionId: z.string().min(1), rawPath: z.string().min(1), pattern: patternOutputSchema, }); -export type HistoricSqlPatternEvidence = z.infer; export const historicSqlEvidenceEnvelopeSchema = z.discriminatedUnion('kind', [ historicSqlTableUsageEvidenceSchema, diff --git a/packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/historic-sql.adapter.test.ts similarity index 97% rename from packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/historic-sql.adapter.test.ts index 3b5697d0..80df5c26 100644 --- a/packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/historic-sql.adapter.test.ts @@ -2,7 +2,7 @@ import { mkdtemp } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import type { SqlAnalysisPort } from '../../../sql-analysis/index.js'; +import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js'; import type { SourceAdapter } from '../../types.js'; import { HistoricSqlSourceAdapter } from './historic-sql.adapter.js'; import type { HistoricSqlReader } from './types.js'; diff --git a/packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts b/packages/cli/src/context/ingest/adapters/historic-sql/historic-sql.adapter.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/historic-sql.adapter.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts similarity index 97% rename from packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts index 971a05ec..62d5ba42 100644 --- a/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts @@ -2,14 +2,9 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import YAML from 'yaml'; -import type { AgentRunnerPort, RunLoopParams } from '../../../llm/index.js'; -import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../../project/index.js'; -import { - type SqlAnalysisBatchItem, - type SqlAnalysisBatchResult, - type SqlAnalysisDialect, - type SqlAnalysisPort, -} from '../../../sql-analysis/index.js'; +import type { AgentRunnerPort, RunLoopParams } from '../../../../context/llm/runtime-port.js'; +import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../../../context/project/project.js'; +import type { SqlAnalysisBatchItem, SqlAnalysisBatchResult, SqlAnalysisDialect, SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js'; import { searchLocalSlSources } from '../../../sl/local-sl.js'; import { searchLocalKnowledgePages } from '../../../wiki/local-knowledge.js'; import { runLocalIngest } from '../../local-ingest.js'; diff --git a/packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/pattern-inputs.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/pattern-inputs.test.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts b/packages/cli/src/context/ingest/adapters/historic-sql/pattern-inputs.ts similarity index 91% rename from packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/pattern-inputs.ts index c9380239..025fa43c 100644 --- a/packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/pattern-inputs.ts @@ -1,13 +1,14 @@ import { Buffer } from 'node:buffer'; import type { StagedPatternsInput } from './types.js'; -export const HISTORIC_SQL_PATTERN_WORKUNIT_DIR = 'patterns-input'; +const HISTORIC_SQL_PATTERN_WORKUNIT_DIR = 'patterns-input'; +/** @internal */ export const HISTORIC_SQL_PATTERN_WORKUNIT_MAX_BYTES = 110_000; -export const HISTORIC_SQL_PATTERN_WORKUNIT_PATH_RE = /^patterns-input\/part-\d{4}\.json$/; +const HISTORIC_SQL_PATTERN_WORKUNIT_PATH_RE = /^patterns-input\/part-\d{4}\.json$/; type PatternTemplate = StagedPatternsInput['templates'][number]; -export interface HistoricSqlPatternInputShard { +interface HistoricSqlPatternInputShard { path: string; input: StagedPatternsInput; byteLength: number; @@ -27,10 +28,11 @@ export function isHistoricSqlPatternInputShardPath(path: string): boolean { return HISTORIC_SQL_PATTERN_WORKUNIT_PATH_RE.test(path); } -export function serializeStagedPatternsInput(input: StagedPatternsInput): string { +function serializeStagedPatternsInput(input: StagedPatternsInput): string { return `${JSON.stringify(input, null, 2)}\n`; } +/** @internal */ export function serializedStagedPatternsInputByteLength(input: StagedPatternsInput): number { return Buffer.byteLength(serializeStagedPatternsInput(input), 'utf-8'); } diff --git a/packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts b/packages/cli/src/context/ingest/adapters/historic-sql/postgres-pgss-reader.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/postgres-pgss-reader.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/projection.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/projection.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/projection.test.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/projection.test.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/projection.ts b/packages/cli/src/context/ingest/adapters/historic-sql/projection.ts similarity index 99% rename from packages/context/src/ingest/adapters/historic-sql/projection.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/projection.ts index 56b1e360..272a96dc 100644 --- a/packages/context/src/ingest/adapters/historic-sql/projection.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/projection.ts @@ -1,7 +1,7 @@ import { access, mkdir, readdir, readFile, rename, writeFile } from 'node:fs/promises'; import { dirname, join, relative } from 'node:path'; import YAML from 'yaml'; -import type { MemoryAction } from '../../../memory/index.js'; +import type { MemoryAction } from '../../../../context/memory/types.js'; import { rawSourcesDirForSync } from '../../raw-sources-paths.js'; import type { FinalizationOverrideReplay } from '../../types.js'; import { mergeUsagePreservingExternal } from '../live-database/manifest.js'; diff --git a/packages/context/src/ingest/adapters/historic-sql/redaction.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/redaction.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/redaction.test.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/redaction.test.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/redaction.ts b/packages/cli/src/context/ingest/adapters/historic-sql/redaction.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/redaction.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/redaction.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/skill-schemas.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/skill-schemas.test.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts b/packages/cli/src/context/ingest/adapters/historic-sql/skill-schemas.ts similarity index 92% rename from packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/skill-schemas.ts index 340cd5b1..a8ba985e 100644 --- a/packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/skill-schemas.ts @@ -26,6 +26,6 @@ export const patternOutputSchema = z.object({ slRefs: z.array(z.string()), constituentTemplateIds: z.array(z.string()), }); -export type PatternOutput = z.infer; +/** @internal */ export const patternsArraySchema = z.array(patternOutputSchema); diff --git a/packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts b/packages/cli/src/context/ingest/adapters/historic-sql/snowflake-query-history-reader.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/snowflake-query-history-reader.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.test.ts similarity index 99% rename from packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.test.ts index d09a4d40..d49c3a1d 100644 --- a/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, readFile, readdir } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it, vi } from 'vitest'; -import type { SqlAnalysisPort } from '../../../sql-analysis/index.js'; +import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js'; import { stageHistoricSqlAggregatedSnapshot } from './stage-unified.js'; import type { AggregatedTemplate, HistoricSqlReader } from './types.js'; diff --git a/packages/context/src/ingest/adapters/historic-sql/stage-unified.ts b/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.ts similarity index 99% rename from packages/context/src/ingest/adapters/historic-sql/stage-unified.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.ts index 18675b11..70997648 100644 --- a/packages/context/src/ingest/adapters/historic-sql/stage-unified.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.ts @@ -1,6 +1,6 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; -import type { SqlAnalysisPort } from '../../../sql-analysis/index.js'; +import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js'; import { bucketDistinctUsers, bucketErrorRate, diff --git a/packages/context/src/ingest/adapters/historic-sql/types.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/types.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/historic-sql/types.test.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/types.test.ts diff --git a/packages/context/src/ingest/adapters/historic-sql/types.ts b/packages/cli/src/context/ingest/adapters/historic-sql/types.ts similarity index 96% rename from packages/context/src/ingest/adapters/historic-sql/types.ts rename to packages/cli/src/context/ingest/adapters/historic-sql/types.ts index ddb1846a..1d256b13 100644 --- a/packages/context/src/ingest/adapters/historic-sql/types.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/types.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import type { SqlAnalysisPort } from '../../../sql-analysis/index.js'; +import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js'; export const HISTORIC_SQL_SOURCE_KEY = 'historic-sql' as const; @@ -115,9 +115,8 @@ export const stagedManifestSchema = z.object({ probeWarnings: z.array(z.string()), staleArchiveAfterDays: z.number().int().positive().default(90), }); -export type StagedManifest = z.infer; -export interface HistoricSqlProbeResult { +interface HistoricSqlProbeResult { warnings: string[]; info?: string[]; } diff --git a/packages/context/src/ingest/adapters/live-database/chunk.test.ts b/packages/cli/src/context/ingest/adapters/live-database/chunk.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/live-database/chunk.test.ts rename to packages/cli/src/context/ingest/adapters/live-database/chunk.test.ts diff --git a/packages/context/src/ingest/adapters/live-database/chunk.ts b/packages/cli/src/context/ingest/adapters/live-database/chunk.ts similarity index 100% rename from packages/context/src/ingest/adapters/live-database/chunk.ts rename to packages/cli/src/context/ingest/adapters/live-database/chunk.ts diff --git a/packages/context/src/ingest/adapters/live-database/daemon-introspection.test.ts b/packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/live-database/daemon-introspection.test.ts rename to packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.test.ts diff --git a/packages/context/src/ingest/adapters/live-database/daemon-introspection.ts b/packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.ts similarity index 98% rename from packages/context/src/ingest/adapters/live-database/daemon-introspection.ts rename to packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.ts index d20b0adc..ff01fda9 100644 --- a/packages/context/src/ingest/adapters/live-database/daemon-introspection.ts +++ b/packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.ts @@ -8,9 +8,9 @@ import type { KtxSchemaColumn, KtxSchemaForeignKey, KtxSchemaSnapshot, KtxSchema import { inferKtxDimensionType, normalizeKtxNativeType } from '../../../scan/type-normalization.js'; import type { LiveDatabaseIntrospectionPort } from './types.js'; -export type KtxDaemonDatabaseIntrospectionCommand = 'database-introspect'; +type KtxDaemonDatabaseIntrospectionCommand = 'database-introspect'; -export type KtxDaemonDatabaseJsonRunner = ( +type KtxDaemonDatabaseJsonRunner = ( subcommand: KtxDaemonDatabaseIntrospectionCommand, payload: Record, ) => Promise>; diff --git a/packages/context/src/ingest/adapters/live-database/live-database.adapter.test.ts b/packages/cli/src/context/ingest/adapters/live-database/live-database.adapter.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/live-database/live-database.adapter.test.ts rename to packages/cli/src/context/ingest/adapters/live-database/live-database.adapter.test.ts diff --git a/packages/context/src/ingest/adapters/live-database/live-database.adapter.ts b/packages/cli/src/context/ingest/adapters/live-database/live-database.adapter.ts similarity index 100% rename from packages/context/src/ingest/adapters/live-database/live-database.adapter.ts rename to packages/cli/src/context/ingest/adapters/live-database/live-database.adapter.ts diff --git a/packages/context/src/ingest/adapters/live-database/manifest.test.ts b/packages/cli/src/context/ingest/adapters/live-database/manifest.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/live-database/manifest.test.ts rename to packages/cli/src/context/ingest/adapters/live-database/manifest.test.ts diff --git a/packages/context/src/ingest/adapters/live-database/manifest.ts b/packages/cli/src/context/ingest/adapters/live-database/manifest.ts similarity index 98% rename from packages/context/src/ingest/adapters/live-database/manifest.ts rename to packages/cli/src/context/ingest/adapters/live-database/manifest.ts index c6a6e2d5..3c35b463 100644 --- a/packages/context/src/ingest/adapters/live-database/manifest.ts +++ b/packages/cli/src/context/ingest/adapters/live-database/manifest.ts @@ -22,7 +22,7 @@ const HISTORIC_SQL_MANAGED_USAGE_KEYS = new Set([ 'staleSince', ]); -export interface LiveDatabaseManifestColumn { +interface LiveDatabaseManifestColumn { name: string; type: string; pk?: boolean; @@ -37,7 +37,7 @@ export interface LiveDatabaseManifestJoinEntry { source: string; } -export interface LiveDatabaseManifestTableEntry { +interface LiveDatabaseManifestTableEntry { table: string; descriptions?: Record; usage?: TableUsageOutput; diff --git a/packages/context/src/ingest/adapters/live-database/stage.test.ts b/packages/cli/src/context/ingest/adapters/live-database/stage.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/live-database/stage.test.ts rename to packages/cli/src/context/ingest/adapters/live-database/stage.test.ts diff --git a/packages/context/src/ingest/adapters/live-database/stage.ts b/packages/cli/src/context/ingest/adapters/live-database/stage.ts similarity index 99% rename from packages/context/src/ingest/adapters/live-database/stage.ts rename to packages/cli/src/context/ingest/adapters/live-database/stage.ts index 03e60f5a..ba925986 100644 --- a/packages/context/src/ingest/adapters/live-database/stage.ts +++ b/packages/cli/src/context/ingest/adapters/live-database/stage.ts @@ -33,6 +33,7 @@ function tableSortKey(table: KtxTableRef): string { return `${table.catalog ?? ''}\u0000${table.db ?? ''}\u0000${table.name}`; } +/** @internal */ export function liveDatabaseTablePath(table: KtxTableRef): string { return `${LIVE_DATABASE_TABLES_DIR}/${encodePathPart(table.catalog)}.${encodePathPart(table.db)}.${encodePathPart( table.name, diff --git a/packages/context/src/ingest/adapters/live-database/types.ts b/packages/cli/src/context/ingest/adapters/live-database/types.ts similarity index 100% rename from packages/context/src/ingest/adapters/live-database/types.ts rename to packages/cli/src/context/ingest/adapters/live-database/types.ts diff --git a/packages/context/src/ingest/adapters/looker/chunk.test.ts b/packages/cli/src/context/ingest/adapters/looker/chunk.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/chunk.test.ts rename to packages/cli/src/context/ingest/adapters/looker/chunk.test.ts diff --git a/packages/context/src/ingest/adapters/looker/chunk.ts b/packages/cli/src/context/ingest/adapters/looker/chunk.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/chunk.ts rename to packages/cli/src/context/ingest/adapters/looker/chunk.ts diff --git a/packages/context/src/ingest/adapters/looker/client-boundary.test.ts b/packages/cli/src/context/ingest/adapters/looker/client-boundary.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/client-boundary.test.ts rename to packages/cli/src/context/ingest/adapters/looker/client-boundary.test.ts diff --git a/packages/context/src/ingest/adapters/looker/client.test.ts b/packages/cli/src/context/ingest/adapters/looker/client.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/client.test.ts rename to packages/cli/src/context/ingest/adapters/looker/client.test.ts diff --git a/packages/context/src/ingest/adapters/looker/client.ts b/packages/cli/src/context/ingest/adapters/looker/client.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/client.ts rename to packages/cli/src/context/ingest/adapters/looker/client.ts diff --git a/packages/context/src/ingest/adapters/looker/daemon-table-identifier-parser.test.ts b/packages/cli/src/context/ingest/adapters/looker/daemon-table-identifier-parser.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/daemon-table-identifier-parser.test.ts rename to packages/cli/src/context/ingest/adapters/looker/daemon-table-identifier-parser.test.ts diff --git a/packages/context/src/ingest/adapters/looker/daemon-table-identifier-parser.ts b/packages/cli/src/context/ingest/adapters/looker/daemon-table-identifier-parser.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/daemon-table-identifier-parser.ts rename to packages/cli/src/context/ingest/adapters/looker/daemon-table-identifier-parser.ts diff --git a/packages/context/src/ingest/adapters/looker/detect.test.ts b/packages/cli/src/context/ingest/adapters/looker/detect.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/detect.test.ts rename to packages/cli/src/context/ingest/adapters/looker/detect.test.ts diff --git a/packages/context/src/ingest/adapters/looker/detect.ts b/packages/cli/src/context/ingest/adapters/looker/detect.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/detect.ts rename to packages/cli/src/context/ingest/adapters/looker/detect.ts diff --git a/packages/context/src/ingest/adapters/looker/evidence-documents.test.ts b/packages/cli/src/context/ingest/adapters/looker/evidence-documents.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/evidence-documents.test.ts rename to packages/cli/src/context/ingest/adapters/looker/evidence-documents.test.ts diff --git a/packages/context/src/ingest/adapters/looker/evidence-documents.ts b/packages/cli/src/context/ingest/adapters/looker/evidence-documents.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/evidence-documents.ts rename to packages/cli/src/context/ingest/adapters/looker/evidence-documents.ts diff --git a/packages/context/src/ingest/adapters/looker/factory.test.ts b/packages/cli/src/context/ingest/adapters/looker/factory.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/factory.test.ts rename to packages/cli/src/context/ingest/adapters/looker/factory.test.ts diff --git a/packages/context/src/ingest/adapters/looker/factory.ts b/packages/cli/src/context/ingest/adapters/looker/factory.ts similarity index 97% rename from packages/context/src/ingest/adapters/looker/factory.ts rename to packages/cli/src/context/ingest/adapters/looker/factory.ts index e80d781c..64fdfa42 100644 --- a/packages/context/src/ingest/adapters/looker/factory.ts +++ b/packages/cli/src/context/ingest/adapters/looker/factory.ts @@ -7,6 +7,7 @@ export interface LookerCredentialResolver { resolve(lookerConnectionId: string): Promise; } +/** @internal */ export interface LookerConnectionClientFactory { createClient(lookerConnectionId: string): Promise; } @@ -23,6 +24,7 @@ export class DefaultLookerConnectionClientFactory implements LookerConnectionCli } } +/** @internal */ export class DefaultLookerClientFactory implements LookerClientFactory { constructor(private readonly inner: LookerConnectionClientFactory) {} diff --git a/packages/context/src/ingest/adapters/looker/fetch-report.test.ts b/packages/cli/src/context/ingest/adapters/looker/fetch-report.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/fetch-report.test.ts rename to packages/cli/src/context/ingest/adapters/looker/fetch-report.test.ts diff --git a/packages/context/src/ingest/adapters/looker/fetch-report.ts b/packages/cli/src/context/ingest/adapters/looker/fetch-report.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/fetch-report.ts rename to packages/cli/src/context/ingest/adapters/looker/fetch-report.ts diff --git a/packages/context/src/ingest/adapters/looker/fetch.test.ts b/packages/cli/src/context/ingest/adapters/looker/fetch.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/fetch.test.ts rename to packages/cli/src/context/ingest/adapters/looker/fetch.test.ts diff --git a/packages/context/src/ingest/adapters/looker/fetch.ts b/packages/cli/src/context/ingest/adapters/looker/fetch.ts similarity index 99% rename from packages/context/src/ingest/adapters/looker/fetch.ts rename to packages/cli/src/context/ingest/adapters/looker/fetch.ts index 2086b48c..9a09ac1f 100644 --- a/packages/context/src/ingest/adapters/looker/fetch.ts +++ b/packages/cli/src/context/ingest/adapters/looker/fetch.ts @@ -1,11 +1,11 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; +import type { ParsedTargetTable } from '../../parsed-target-table.js'; import type { FetchContext } from '../../types.js'; import { writeLookerEvidenceDocuments } from './evidence-documents.js'; import { writeLookerFetchReport } from './fetch-report.js'; import { type LookerPullConfig, - type ParsedTargetTable, parseLookerPullConfig, STAGED_FILES, type StagedDashboardFile, @@ -31,7 +31,7 @@ import { stagedUserFileSchema, } from './types.js'; -export interface LookerEntityRef { +interface LookerEntityRef { id: string; updatedAt?: string | null; } diff --git a/packages/context/src/ingest/adapters/looker/local-looker.adapter.ts b/packages/cli/src/context/ingest/adapters/looker/local-looker.adapter.ts similarity index 71% rename from packages/context/src/ingest/adapters/looker/local-looker.adapter.ts rename to packages/cli/src/context/ingest/adapters/looker/local-looker.adapter.ts index 47299373..ea8ba658 100644 --- a/packages/context/src/ingest/adapters/looker/local-looker.adapter.ts +++ b/packages/cli/src/context/ingest/adapters/looker/local-looker.adapter.ts @@ -1,11 +1,8 @@ -import type { KtxLocalProject, KtxProjectConnectionConfig } from '../../../project/index.js'; -import type { LookerClientLogger } from './client.js'; +import type { KtxLocalProject } from '../../../../context/project/project.js'; +import type { KtxProjectConnectionConfig } from '../../../../context/project/config.js'; import { - DefaultLookerClientFactory, - DefaultLookerConnectionClientFactory, type LookerCredentialResolver, } from './factory.js'; -import { LookerSourceAdapter } from './looker.adapter.js'; function stringField(value: unknown): string | null { return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null; @@ -54,16 +51,3 @@ export function createLocalLookerCredentialResolver( }, }; } - -export function createLocalLookerSourceAdapter( - project: KtxLocalProject, - env: NodeJS.ProcessEnv = process.env, - logger?: LookerClientLogger, -): LookerSourceAdapter { - const connectionFactory = new DefaultLookerConnectionClientFactory(createLocalLookerCredentialResolver(project, env), { - ...(logger ? { logger } : {}), - }); - return new LookerSourceAdapter({ - clientFactory: new DefaultLookerClientFactory(connectionFactory), - }); -} diff --git a/packages/context/src/ingest/adapters/looker/local-runtime-store.test.ts b/packages/cli/src/context/ingest/adapters/looker/local-runtime-store.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/local-runtime-store.test.ts rename to packages/cli/src/context/ingest/adapters/looker/local-runtime-store.test.ts diff --git a/packages/context/src/ingest/adapters/looker/local-runtime-store.ts b/packages/cli/src/context/ingest/adapters/looker/local-runtime-store.ts similarity index 98% rename from packages/context/src/ingest/adapters/looker/local-runtime-store.ts rename to packages/cli/src/context/ingest/adapters/looker/local-runtime-store.ts index a7bc7111..fa6560b2 100644 --- a/packages/context/src/ingest/adapters/looker/local-runtime-store.ts +++ b/packages/cli/src/context/ingest/adapters/looker/local-runtime-store.ts @@ -5,7 +5,7 @@ import type { LookerWarehouseConnectionInfo } from './client.js'; import type { LookerConnectionMapping } from './mapping.js'; import type { LookerRuntimeCursors } from './types.js'; -export type LocalLookerMappingSource = 'ktx.yaml' | 'cli' | 'refresh'; +type LocalLookerMappingSource = 'ktx.yaml' | 'cli' | 'refresh'; interface LocalLookerRuntimeStoreOptions { dbPath: string; @@ -41,7 +41,7 @@ export interface ClearLocalLookerMappingsInput { lookerConnectionName?: string; } -export interface LookerSourceStateReader { +interface LookerSourceStateReader { readMappings(lookerConnectionId: string): Promise; readCursors(lookerConnectionId: string): Promise; } diff --git a/packages/context/src/ingest/adapters/looker/looker.adapter.test.ts b/packages/cli/src/context/ingest/adapters/looker/looker.adapter.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/looker.adapter.test.ts rename to packages/cli/src/context/ingest/adapters/looker/looker.adapter.test.ts diff --git a/packages/context/src/ingest/adapters/looker/looker.adapter.ts b/packages/cli/src/context/ingest/adapters/looker/looker.adapter.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/looker.adapter.ts rename to packages/cli/src/context/ingest/adapters/looker/looker.adapter.ts diff --git a/packages/context/src/ingest/adapters/looker/mapping.test.ts b/packages/cli/src/context/ingest/adapters/looker/mapping.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/mapping.test.ts rename to packages/cli/src/context/ingest/adapters/looker/mapping.test.ts diff --git a/packages/context/src/ingest/adapters/looker/mapping.ts b/packages/cli/src/context/ingest/adapters/looker/mapping.ts similarity index 95% rename from packages/context/src/ingest/adapters/looker/mapping.ts rename to packages/cli/src/context/ingest/adapters/looker/mapping.ts index 92ba82ce..cbd80e80 100644 --- a/packages/context/src/ingest/adapters/looker/mapping.ts +++ b/packages/cli/src/context/ingest/adapters/looker/mapping.ts @@ -1,13 +1,8 @@ +import type { ParsedTargetTable } from '../../parsed-target-table.js'; import type { LookerWarehouseConnectionInfo } from './client.js'; -import type { - LookerPullConfig, - LookerRuntimeCursors, - ParsedTargetTable, - StagedExploreFile, - StagedLookmlModelsFile, -} from './types.js'; +import type { LookerPullConfig, LookerRuntimeCursors, StagedExploreFile, StagedLookmlModelsFile } from './types.js'; -export const LOOKER_DIALECT_TO_CONNECTION_TYPE = { +const LOOKER_DIALECT_TO_CONNECTION_TYPE = { bigquery: 'BIGQUERY', bigquery_standard_sql: 'BIGQUERY', snowflake: 'SNOWFLAKE', @@ -21,6 +16,7 @@ export const LOOKER_DIALECT_TO_CONNECTION_TYPE = { clickhouse: 'CLICKHOUSE', } as const; +/** @internal */ export type LookerWarehouseTargetConnectionType = (typeof LOOKER_DIALECT_TO_CONNECTION_TYPE)[keyof typeof LOOKER_DIALECT_TO_CONNECTION_TYPE]; @@ -38,6 +34,7 @@ export interface LookerTargetConnection { connection_params?: Record | null; } +/** @internal */ export interface LookerMappingCandidateConnection extends LookerTargetConnection {} export interface LookerMappingDrift { @@ -94,6 +91,7 @@ export async function discoverLookerConnections( return client.listLookerConnections(); } +/** @internal */ export function lookerDialectToConnectionType(dialect: string | null): LookerWarehouseTargetConnectionType | null { if (!dialect) { return null; @@ -103,10 +101,12 @@ export function lookerDialectToConnectionType(dialect: string | null): LookerWar ); } +/** @internal */ export function sqlglotDialectForConnectionType(connectionType: string): string | null { return SQLGLOT_DIALECT_BY_CONNECTION_TYPE[connectionType as LookerWarehouseTargetConnectionType] ?? null; } +/** @internal */ export function validateLookerWarehouseTarget(connectionType: string): { ok: true } | { ok: false; reason: string } { return sqlglotDialectForConnectionType(connectionType) ? { ok: true } @@ -116,7 +116,7 @@ export function validateLookerWarehouseTarget(connectionType: string): { ok: tru }; } -export function extractWarehouseHost(params: unknown, connectionType: string): string | null { +function extractWarehouseHost(params: unknown, connectionType: string): string | null { const record = isRecord(params) ? params : {}; switch (connectionType) { case 'POSTGRESQL': @@ -131,7 +131,7 @@ export function extractWarehouseHost(params: unknown, connectionType: string): s } } -export function extractWarehouseDatabase(params: unknown, connectionType: string): string | null { +function extractWarehouseDatabase(params: unknown, connectionType: string): string | null { const record = isRecord(params) ? params : {}; switch (connectionType) { case 'POSTGRESQL': @@ -147,14 +147,15 @@ export function extractWarehouseDatabase(params: unknown, connectionType: string } } -export function normalizeHost(value: string | null): string | null { +function normalizeHost(value: string | null): string | null { return value ? value.toLowerCase().replace(/:\d+$/, '') : null; } -export function normalizeName(value: string | null): string | null { +function normalizeName(value: string | null): string | null { return value ? value.toLowerCase() : null; } +/** @internal */ export function suggestKtxConnectionForLookerConnection(args: { lookerConnection: LookerWarehouseConnectionInfo; candidateConnections: LookerMappingCandidateConnection[]; @@ -229,6 +230,7 @@ export function validateLookerMappings(args: { return errors.length === 0 ? { ok: true } : { ok: false, errors }; } +/** @internal */ export function refreshLookerMappingPlaceholders(args: { stored: LookerConnectionMapping[]; live: LookerWarehouseConnectionInfo[]; @@ -269,6 +271,7 @@ export function refreshLookerMappingPlaceholders(args: { return { mappings: [...byName.values()], changed }; } +/** @internal */ export function collectExploreParseItems(args: { explore: StagedExploreFile; connectionMappings: Record; @@ -314,6 +317,7 @@ export function collectExploreParseItems(args: { return { parsedTargetTables, parseItems }; } +/** @internal */ export function projectParsedIdentifier(row: LookerParsedIdentifier | undefined): ParsedTargetTable { if (!row) { return { ok: false, reason: 'parse_error', detail: 'Python parser response was missing this key.' }; diff --git a/packages/cli/src/context/ingest/adapters/looker/reconcile.test.ts b/packages/cli/src/context/ingest/adapters/looker/reconcile.test.ts new file mode 100644 index 00000000..09e8685f --- /dev/null +++ b/packages/cli/src/context/ingest/adapters/looker/reconcile.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from 'vitest'; +import { buildLookerReconcileNotes } from './reconcile.js'; + +describe('buildLookerReconcileNotes', () => { + it('instructs reconciliation to record subsumed provenance', () => { + expect(buildLookerReconcileNotes()).toEqual([ + [ + 'Looker runtime API-derived SL sources use looker____.', + 'If the unprefixed file-adapter source __ exists, prefer it in wiki sl_refs, delete or avoid the API-derived source, and call emit_artifact_resolution with actionType="subsumed" for the API raw explore path.', + ].join(' '), + ]); + }); +}); diff --git a/packages/cli/src/context/ingest/adapters/looker/reconcile.ts b/packages/cli/src/context/ingest/adapters/looker/reconcile.ts new file mode 100644 index 00000000..1dde846f --- /dev/null +++ b/packages/cli/src/context/ingest/adapters/looker/reconcile.ts @@ -0,0 +1,8 @@ +export function buildLookerReconcileNotes(): string[] { + return [ + [ + 'Looker runtime API-derived SL sources use looker____.', + 'If the unprefixed file-adapter source __ exists, prefer it in wiki sl_refs, delete or avoid the API-derived source, and call emit_artifact_resolution with actionType="subsumed" for the API raw explore path.', + ].join(' '), + ]; +} diff --git a/packages/context/src/ingest/adapters/looker/scope.test.ts b/packages/cli/src/context/ingest/adapters/looker/scope.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/looker/scope.test.ts rename to packages/cli/src/context/ingest/adapters/looker/scope.test.ts diff --git a/packages/context/src/ingest/adapters/looker/scope.ts b/packages/cli/src/context/ingest/adapters/looker/scope.ts similarity index 94% rename from packages/context/src/ingest/adapters/looker/scope.ts rename to packages/cli/src/context/ingest/adapters/looker/scope.ts index feabde72..f91d31b6 100644 --- a/packages/context/src/ingest/adapters/looker/scope.ts +++ b/packages/cli/src/context/ingest/adapters/looker/scope.ts @@ -15,7 +15,7 @@ export async function describeLookerScope(stagedDir: string): Promise { +async function readLookerScope(stagedDir: string): Promise { try { const body = await readFile(join(stagedDir, STAGED_FILES.scope), 'utf-8'); return stagedLookerScopeFileSchema.parse(JSON.parse(body)); @@ -27,6 +27,7 @@ export async function readLookerScope(stagedDir: string): Promise { diff --git a/packages/context/src/ingest/adapters/looker/tools/looker-query-to-sl.tool.ts b/packages/cli/src/context/ingest/adapters/looker/tools/looker-query-to-sl.tool.ts similarity index 95% rename from packages/context/src/ingest/adapters/looker/tools/looker-query-to-sl.tool.ts rename to packages/cli/src/context/ingest/adapters/looker/tools/looker-query-to-sl.tool.ts index 67bb68c6..400d123e 100644 --- a/packages/context/src/ingest/adapters/looker/tools/looker-query-to-sl.tool.ts +++ b/packages/cli/src/context/ingest/adapters/looker/tools/looker-query-to-sl.tool.ts @@ -1,14 +1,15 @@ import { tool } from 'ai'; import { z } from 'zod'; -import type { ToolOutput } from '../../../../tools/index.js'; -import { type ParsedTargetTable, stagedLookerQuerySchema } from '../types.js'; +import type { ToolOutput } from '../../../../../context/tools/base-tool.js'; +import type { ParsedTargetTable } from '../../../parsed-target-table.js'; +import { stagedLookerQuerySchema } from '../types.js'; const lookerUsageInputSchema = z.object({ queryCount30d: z.number().int().nonnegative().default(0), uniqueUsers30d: z.number().int().nonnegative().default(0), }); -export const lookerQueryToSlInputSchema = z.object({ +const lookerQueryToSlInputSchema = z.object({ query: stagedLookerQuerySchema, contentTitle: z.string().min(1).optional(), contentType: z.enum(['look', 'dashboard_tile']).default('look'), @@ -19,17 +20,17 @@ export type LookerQueryToSlInput = z.input; type LookerTargetStatus = 'mapped' | 'unmapped' | 'unparseable' | 'missing_target_table'; -export interface LookerSlFieldProposal { +interface LookerSlFieldProposal { name: string; lookerField: string; } -export interface LookerSlMeasureProposal extends LookerSlFieldProposal { +interface LookerSlMeasureProposal extends LookerSlFieldProposal { expr: string; description: string; } -export interface LookerSlSegmentProposal { +interface LookerSlSegmentProposal { name: string; filters: Record; suggestedPredicate: string; @@ -88,6 +89,7 @@ function targetNotes(status: LookerTargetStatus, targetTable: ParsedTargetTable ]; } +/** @internal */ export function buildLookerSlProposal(raw: LookerQueryToSlInput): LookerSlProposal { const input = lookerQueryToSlInputSchema.parse(raw); const sourceName = `looker__${toSlName(input.query.model)}__${toSlName(input.query.view)}`; @@ -181,7 +183,7 @@ export function createLookerQueryToSlTool() { }); } -export function formatLookerSlProposal(proposal: LookerSlProposal): string { +function formatLookerSlProposal(proposal: LookerSlProposal): string { const lines = [ '## Looker query SL proposal', '', diff --git a/packages/context/src/ingest/adapters/looker/types.test.ts b/packages/cli/src/context/ingest/adapters/looker/types.test.ts similarity index 99% rename from packages/context/src/ingest/adapters/looker/types.test.ts rename to packages/cli/src/context/ingest/adapters/looker/types.test.ts index 2d517d50..2a4c2b8c 100644 --- a/packages/context/src/ingest/adapters/looker/types.test.ts +++ b/packages/cli/src/context/ingest/adapters/looker/types.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it } from 'vitest'; +import { parsedTargetTableSchema } from '../../parsed-target-table.js'; import { lookerPullConfigSchema, parseLookerPullConfig, - parsedTargetTableSchema, stagedDashboardFileSchema, stagedExploreFileSchema, stagedLookerFetchIssueSchema, diff --git a/packages/context/src/ingest/adapters/looker/types.ts b/packages/cli/src/context/ingest/adapters/looker/types.ts similarity index 97% rename from packages/context/src/ingest/adapters/looker/types.ts rename to packages/cli/src/context/ingest/adapters/looker/types.ts index ea515b9d..37ff362a 100644 --- a/packages/context/src/ingest/adapters/looker/types.ts +++ b/packages/cli/src/context/ingest/adapters/looker/types.ts @@ -5,17 +5,16 @@ import { parsedTargetTableSchema } from '../../parsed-target-table.js'; const lookerIdSchema = z.union([z.string(), z.number().int()]).transform(String); const nullableLookerIdSchema = z.union([lookerIdSchema, z.null()]).default(null); -export const lookerConnectionIdSchema = z.string().min(1).regex(/^[A-Za-z0-9_-]+$/); +const lookerConnectionIdSchema = z.string().min(1).regex(/^[A-Za-z0-9_-]+$/); -export { parsedTargetTableSchema, type ParsedTargetTable } from '../../parsed-target-table.js'; - -export const lookerRuntimeCursorsSchema = z.object({ +const lookerRuntimeCursorsSchema = z.object({ dashboardsLastSyncedAt: z.iso.datetime().nullable().default(null), looksLastSyncedAt: z.iso.datetime().nullable().default(null), }); export type LookerRuntimeCursors = z.infer; +/** @internal */ export const lookerPullConfigSchema = z.object({ lookerConnectionId: lookerConnectionIdSchema.optional(), instanceBaseUrl: z.url().optional(), @@ -216,6 +215,7 @@ const stagedLookerFetchIssueKindSchema = z.enum([ 'lookml_connection_mismatch', ]); +/** @internal */ export const stagedLookerFetchIssueSchema = z.object({ rawPath: z.string().min(1), entityType: z.enum(['dashboard', 'look', 'explore', 'signals', 'lookml_models', 'looker_connection_mapping']), diff --git a/packages/context/src/ingest/adapters/lookml/chunk.test.ts b/packages/cli/src/context/ingest/adapters/lookml/chunk.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/lookml/chunk.test.ts rename to packages/cli/src/context/ingest/adapters/lookml/chunk.test.ts diff --git a/packages/context/src/ingest/adapters/lookml/chunk.ts b/packages/cli/src/context/ingest/adapters/lookml/chunk.ts similarity index 100% rename from packages/context/src/ingest/adapters/lookml/chunk.ts rename to packages/cli/src/context/ingest/adapters/lookml/chunk.ts diff --git a/packages/context/src/ingest/adapters/lookml/detect.test.ts b/packages/cli/src/context/ingest/adapters/lookml/detect.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/lookml/detect.test.ts rename to packages/cli/src/context/ingest/adapters/lookml/detect.test.ts diff --git a/packages/context/src/ingest/adapters/lookml/detect.ts b/packages/cli/src/context/ingest/adapters/lookml/detect.ts similarity index 100% rename from packages/context/src/ingest/adapters/lookml/detect.ts rename to packages/cli/src/context/ingest/adapters/lookml/detect.ts diff --git a/packages/context/src/ingest/adapters/lookml/fetch-report.test.ts b/packages/cli/src/context/ingest/adapters/lookml/fetch-report.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/lookml/fetch-report.test.ts rename to packages/cli/src/context/ingest/adapters/lookml/fetch-report.test.ts diff --git a/packages/context/src/ingest/adapters/lookml/fetch-report.ts b/packages/cli/src/context/ingest/adapters/lookml/fetch-report.ts similarity index 99% rename from packages/context/src/ingest/adapters/lookml/fetch-report.ts rename to packages/cli/src/context/ingest/adapters/lookml/fetch-report.ts index e626b392..75231f73 100644 --- a/packages/context/src/ingest/adapters/lookml/fetch-report.ts +++ b/packages/cli/src/context/ingest/adapters/lookml/fetch-report.ts @@ -4,7 +4,9 @@ import * as z from 'zod'; import type { SourceFetchReport } from '../../types.js'; import type { ParsedLookmlProject } from './parse.js'; +/** @internal */ export const LOOKML_FETCH_REPORT_FILE = 'lookml-fetch-report.json'; +/** @internal */ export const LOOKML_MISMATCHED_MODELS_FILE = 'lookml-mismatched-models.json'; const fetchIssueKindSchema = z.enum([ diff --git a/packages/context/src/ingest/adapters/lookml/fetch.test.ts b/packages/cli/src/context/ingest/adapters/lookml/fetch.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/lookml/fetch.test.ts rename to packages/cli/src/context/ingest/adapters/lookml/fetch.test.ts diff --git a/packages/context/src/ingest/adapters/lookml/fetch.ts b/packages/cli/src/context/ingest/adapters/lookml/fetch.ts similarity index 100% rename from packages/context/src/ingest/adapters/lookml/fetch.ts rename to packages/cli/src/context/ingest/adapters/lookml/fetch.ts diff --git a/packages/context/src/ingest/adapters/lookml/graph.test.ts b/packages/cli/src/context/ingest/adapters/lookml/graph.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/lookml/graph.test.ts rename to packages/cli/src/context/ingest/adapters/lookml/graph.test.ts diff --git a/packages/context/src/ingest/adapters/lookml/graph.ts b/packages/cli/src/context/ingest/adapters/lookml/graph.ts similarity index 100% rename from packages/context/src/ingest/adapters/lookml/graph.ts rename to packages/cli/src/context/ingest/adapters/lookml/graph.ts diff --git a/packages/context/src/ingest/adapters/lookml/lookml-parser.d.ts b/packages/cli/src/context/ingest/adapters/lookml/lookml-parser.d.ts similarity index 100% rename from packages/context/src/ingest/adapters/lookml/lookml-parser.d.ts rename to packages/cli/src/context/ingest/adapters/lookml/lookml-parser.d.ts diff --git a/packages/context/src/ingest/adapters/lookml/lookml.adapter.test.ts b/packages/cli/src/context/ingest/adapters/lookml/lookml.adapter.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/lookml/lookml.adapter.test.ts rename to packages/cli/src/context/ingest/adapters/lookml/lookml.adapter.test.ts diff --git a/packages/context/src/ingest/adapters/lookml/lookml.adapter.ts b/packages/cli/src/context/ingest/adapters/lookml/lookml.adapter.ts similarity index 100% rename from packages/context/src/ingest/adapters/lookml/lookml.adapter.ts rename to packages/cli/src/context/ingest/adapters/lookml/lookml.adapter.ts diff --git a/packages/context/src/ingest/adapters/lookml/parse.test.ts b/packages/cli/src/context/ingest/adapters/lookml/parse.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/lookml/parse.test.ts rename to packages/cli/src/context/ingest/adapters/lookml/parse.test.ts diff --git a/packages/context/src/ingest/adapters/lookml/parse.ts b/packages/cli/src/context/ingest/adapters/lookml/parse.ts similarity index 100% rename from packages/context/src/ingest/adapters/lookml/parse.ts rename to packages/cli/src/context/ingest/adapters/lookml/parse.ts diff --git a/packages/context/src/ingest/adapters/lookml/pull-config.test.ts b/packages/cli/src/context/ingest/adapters/lookml/pull-config.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/lookml/pull-config.test.ts rename to packages/cli/src/context/ingest/adapters/lookml/pull-config.test.ts diff --git a/packages/context/src/ingest/adapters/lookml/pull-config.ts b/packages/cli/src/context/ingest/adapters/lookml/pull-config.ts similarity index 96% rename from packages/context/src/ingest/adapters/lookml/pull-config.ts rename to packages/cli/src/context/ingest/adapters/lookml/pull-config.ts index a6eefc9f..1e555c64 100644 --- a/packages/context/src/ingest/adapters/lookml/pull-config.ts +++ b/packages/cli/src/context/ingest/adapters/lookml/pull-config.ts @@ -1,7 +1,7 @@ import * as z from 'zod'; import { parsedTargetTableSchema } from '../../parsed-target-table.js'; -export const lookmlPullConfigSchema = z.object({ +const lookmlPullConfigSchema = z.object({ repoUrl: z.string().url(), branch: z.string().default('main'), path: z.string().nullable().default(null), diff --git a/packages/context/src/ingest/adapters/metabase/card-references.test.ts b/packages/cli/src/context/ingest/adapters/metabase/card-references.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/card-references.test.ts rename to packages/cli/src/context/ingest/adapters/metabase/card-references.test.ts diff --git a/packages/context/src/ingest/adapters/metabase/card-references.ts b/packages/cli/src/context/ingest/adapters/metabase/card-references.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/card-references.ts rename to packages/cli/src/context/ingest/adapters/metabase/card-references.ts diff --git a/packages/context/src/ingest/adapters/metabase/chunk.test.ts b/packages/cli/src/context/ingest/adapters/metabase/chunk.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/chunk.test.ts rename to packages/cli/src/context/ingest/adapters/metabase/chunk.test.ts diff --git a/packages/context/src/ingest/adapters/metabase/chunk.ts b/packages/cli/src/context/ingest/adapters/metabase/chunk.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/chunk.ts rename to packages/cli/src/context/ingest/adapters/metabase/chunk.ts diff --git a/packages/context/src/ingest/adapters/metabase/client-boundary.test.ts b/packages/cli/src/context/ingest/adapters/metabase/client-boundary.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/client-boundary.test.ts rename to packages/cli/src/context/ingest/adapters/metabase/client-boundary.test.ts diff --git a/packages/context/src/ingest/adapters/metabase/client-port.test.ts b/packages/cli/src/context/ingest/adapters/metabase/client-port.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/client-port.test.ts rename to packages/cli/src/context/ingest/adapters/metabase/client-port.test.ts diff --git a/packages/context/src/ingest/adapters/metabase/client-port.ts b/packages/cli/src/context/ingest/adapters/metabase/client-port.ts similarity index 97% rename from packages/context/src/ingest/adapters/metabase/client-port.ts rename to packages/cli/src/context/ingest/adapters/metabase/client-port.ts index a5fdb6ce..47d78283 100644 --- a/packages/context/src/ingest/adapters/metabase/client-port.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/client-port.ts @@ -69,7 +69,7 @@ export interface MetabaseCardSummary { collection_id?: number | 'root' | null; } -export interface MetabaseResultMetadataColumn { +interface MetabaseResultMetadataColumn { name: string; base_type: string; semantic_type?: string | null; @@ -79,7 +79,7 @@ export interface MetabaseResultMetadataColumn { field_ref?: unknown[] | null; } -export interface MetabaseParameter { +interface MetabaseParameter { id: string; name: string; type: string; @@ -103,7 +103,7 @@ export interface MetabaseTemplateTag { 'widget-type'?: string; } -export interface MetabaseResolvedTemplateTag { +interface MetabaseResolvedTemplateTag { name: string; type: string; cardReference?: number | null; diff --git a/packages/context/src/ingest/adapters/metabase/client.test.ts b/packages/cli/src/context/ingest/adapters/metabase/client.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/client.test.ts rename to packages/cli/src/context/ingest/adapters/metabase/client.test.ts diff --git a/packages/context/src/ingest/adapters/metabase/client.ts b/packages/cli/src/context/ingest/adapters/metabase/client.ts similarity index 99% rename from packages/context/src/ingest/adapters/metabase/client.ts rename to packages/cli/src/context/ingest/adapters/metabase/client.ts index 1962bfe0..7b075991 100644 --- a/packages/context/src/ingest/adapters/metabase/client.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/client.ts @@ -91,6 +91,7 @@ class MetabaseApiError extends Error { * `[[`/`]]` literals in string values or regex predicates are preserved. Metabase's * grammar disallows nested optional blocks (per docs), so non-greedy matching is safe. */ +/** @internal */ export function stripOptionalClauses(sql: string): string { return sql.replace(/\[\[[\s\S]*?\]\]/g, (match) => (match.includes('{{') ? '' : match)); } @@ -163,6 +164,7 @@ function injectNativeSql(datasetQuery: MetabaseDatasetQuery, sql: string): Metab * format, number widgets need a string scalar, identifier/enum widgets accept `[string]`. * Sending `['placeholder']` for a date widget triggers a ClassCastException → HTTP 500. */ +/** @internal */ export function getDummyValueForWidgetType(widgetType: string | undefined): string | string[] { switch (widgetType) { case 'date/range': diff --git a/packages/context/src/ingest/adapters/metabase/detect.test.ts b/packages/cli/src/context/ingest/adapters/metabase/detect.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/detect.test.ts rename to packages/cli/src/context/ingest/adapters/metabase/detect.test.ts diff --git a/packages/context/src/ingest/adapters/metabase/detect.ts b/packages/cli/src/context/ingest/adapters/metabase/detect.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/detect.ts rename to packages/cli/src/context/ingest/adapters/metabase/detect.ts diff --git a/packages/context/src/ingest/adapters/metabase/fanout-planner.test.ts b/packages/cli/src/context/ingest/adapters/metabase/fanout-planner.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/fanout-planner.test.ts rename to packages/cli/src/context/ingest/adapters/metabase/fanout-planner.test.ts diff --git a/packages/context/src/ingest/adapters/metabase/fanout-planner.ts b/packages/cli/src/context/ingest/adapters/metabase/fanout-planner.ts similarity index 96% rename from packages/context/src/ingest/adapters/metabase/fanout-planner.ts rename to packages/cli/src/context/ingest/adapters/metabase/fanout-planner.ts index 660e8100..b0916a4c 100644 --- a/packages/context/src/ingest/adapters/metabase/fanout-planner.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/fanout-planner.ts @@ -1,6 +1,6 @@ import { parseMetabasePullConfig, type MetabasePullConfig } from './types.js'; -export interface MetabaseFanoutMappingInput { +interface MetabaseFanoutMappingInput { metabaseDatabaseId: number; targetConnectionId: string | null; syncEnabled: boolean; diff --git a/packages/context/src/ingest/adapters/metabase/fetch-scope.test.ts b/packages/cli/src/context/ingest/adapters/metabase/fetch-scope.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/fetch-scope.test.ts rename to packages/cli/src/context/ingest/adapters/metabase/fetch-scope.test.ts diff --git a/packages/context/src/ingest/adapters/metabase/fetch-scope.ts b/packages/cli/src/context/ingest/adapters/metabase/fetch-scope.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/fetch-scope.ts rename to packages/cli/src/context/ingest/adapters/metabase/fetch-scope.ts diff --git a/packages/context/src/ingest/adapters/metabase/fetch.test.ts b/packages/cli/src/context/ingest/adapters/metabase/fetch.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/fetch.test.ts rename to packages/cli/src/context/ingest/adapters/metabase/fetch.test.ts diff --git a/packages/context/src/ingest/adapters/metabase/fetch.ts b/packages/cli/src/context/ingest/adapters/metabase/fetch.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/fetch.ts rename to packages/cli/src/context/ingest/adapters/metabase/fetch.ts diff --git a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts b/packages/cli/src/context/ingest/adapters/metabase/local-metabase.adapter.test.ts similarity index 96% rename from packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts rename to packages/cli/src/context/ingest/adapters/metabase/local-metabase.adapter.test.ts index b25ea18b..c20a65ac 100644 --- a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/local-metabase.adapter.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import type { KtxProjectConnectionConfig } from '../../../project/index.js'; +import type { KtxProjectConnectionConfig } from '../../../../context/project/config.js'; import { metabaseRuntimeConfigFromLocalConnection } from './local-metabase.adapter.js'; describe('metabaseRuntimeConfigFromLocalConnection', () => { diff --git a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.ts b/packages/cli/src/context/ingest/adapters/metabase/local-metabase.adapter.ts similarity index 92% rename from packages/context/src/ingest/adapters/metabase/local-metabase.adapter.ts rename to packages/cli/src/context/ingest/adapters/metabase/local-metabase.adapter.ts index 8d8d5f06..7cb3d843 100644 --- a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/local-metabase.adapter.ts @@ -1,5 +1,6 @@ -import type { KtxLocalProject, KtxProjectConnectionConfig } from '../../../project/index.js'; -import { ktxLocalStateDbPath } from '../../../project/index.js'; +import type { KtxLocalProject } from '../../../../context/project/project.js'; +import type { KtxProjectConnectionConfig } from '../../../../context/project/config.js'; +import { ktxLocalStateDbPath } from '../../../../context/project/local-state-db.js'; import { resolveKtxConfigReference } from '../../../core/config-reference.js'; import { DEFAULT_METABASE_CLIENT_CONFIG, diff --git a/packages/context/src/ingest/adapters/metabase/local-source-state-store.test.ts b/packages/cli/src/context/ingest/adapters/metabase/local-source-state-store.test.ts similarity index 97% rename from packages/context/src/ingest/adapters/metabase/local-source-state-store.test.ts rename to packages/cli/src/context/ingest/adapters/metabase/local-source-state-store.test.ts index 4fdafba5..0225f398 100644 --- a/packages/context/src/ingest/adapters/metabase/local-source-state-store.test.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/local-source-state-store.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { buildDefaultKtxProjectConfig } from '../../../project/index.js'; +import { buildDefaultKtxProjectConfig } from '../../../../context/project/config.js'; import { connectionConfigSchema } from '../../../project/driver-schemas.js'; import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './local-source-state-store.js'; diff --git a/packages/context/src/ingest/adapters/metabase/local-source-state-store.ts b/packages/cli/src/context/ingest/adapters/metabase/local-source-state-store.ts similarity index 96% rename from packages/context/src/ingest/adapters/metabase/local-source-state-store.ts rename to packages/cli/src/context/ingest/adapters/metabase/local-source-state-store.ts index f8026db6..3393b9f8 100644 --- a/packages/context/src/ingest/adapters/metabase/local-source-state-store.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/local-source-state-store.ts @@ -1,15 +1,12 @@ import { mkdirSync } from 'node:fs'; import { dirname } from 'node:path'; import Database from 'better-sqlite3'; -import { - parseMetabaseMappingBootstrap, - type KtxLocalProject, - type MetabaseMappingBootstrap, -} from '../../../project/index.js'; +import { parseMetabaseMappingBootstrap, type MetabaseMappingBootstrap } from '../../../../context/project/mappings-yaml-schema.js'; +import type { KtxLocalProject } from '../../../../context/project/project.js'; import type { DiscoveredMetabaseDatabase } from './mapping.js'; import type { MetabaseSourceState, MetabaseSourceStateReader, MetabaseSourceStateSelection } from './source-state-port.js'; -export type LocalMetabaseMappingSource = 'ktx.yaml' | 'refresh'; +type LocalMetabaseMappingSource = 'ktx.yaml' | 'refresh'; interface LocalMetabaseDiscoveryCacheOptions { dbPath: string; diff --git a/packages/context/src/ingest/adapters/metabase/mapping.test.ts b/packages/cli/src/context/ingest/adapters/metabase/mapping.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/mapping.test.ts rename to packages/cli/src/context/ingest/adapters/metabase/mapping.test.ts diff --git a/packages/context/src/ingest/adapters/metabase/mapping.ts b/packages/cli/src/context/ingest/adapters/metabase/mapping.ts similarity index 98% rename from packages/context/src/ingest/adapters/metabase/mapping.ts rename to packages/cli/src/context/ingest/adapters/metabase/mapping.ts index 9dc75006..788ae43b 100644 --- a/packages/context/src/ingest/adapters/metabase/mapping.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/mapping.ts @@ -1,5 +1,6 @@ import type { MetabaseDatabase, MetabaseRuntimeClient } from './client-port.js'; +/** @internal */ export const METABASE_ENGINE_TO_CONNECTION_TYPE = { postgres: 'POSTGRESQL', bigquery: 'BIGQUERY', @@ -9,8 +10,6 @@ export const METABASE_ENGINE_TO_CONNECTION_TYPE = { mysql: 'MYSQL', } as const; -export type MetabaseMappedConnectionType = - (typeof METABASE_ENGINE_TO_CONNECTION_TYPE)[keyof typeof METABASE_ENGINE_TO_CONNECTION_TYPE]; export interface DiscoveredMetabaseDatabase { id: number; @@ -42,26 +41,31 @@ export interface KtxConnectionPhysicalInfo { [key: string]: unknown; } +/** @internal */ export interface PhysicalMismatchInput { mappingId: string; metabase: MappingPhysicalInfo; target: KtxConnectionPhysicalInfo; } +/** @internal */ export interface PhysicalMismatch { mappingId: string; reason: string; } +/** @internal */ export interface MappingRefreshReport { drift: MetabaseMappingDrift; physicalMismatches: PhysicalMismatch[]; } +/** @internal */ export type MetabaseMappingValidationResult = | { ok: true } | { ok: false; errors: Array<{ key: string; reason: string }> }; +/** @internal */ export interface AutoMatchCandidate { id: string; name: string; @@ -69,6 +73,7 @@ export interface AutoMatchCandidate { connection_params: unknown; } +/** @internal */ export interface AutoMatchResult { connectionId: string; connectionName: string; @@ -170,6 +175,7 @@ export function computeMetabaseMappingDrift(args: { return { unmappedDiscovered, staleMappings, inSync }; } +/** @internal */ export function validateMetabaseMappings(args: { mappings: Record; knownKtxConnectionIds: Set; @@ -236,6 +242,7 @@ export function validateMappingPhysicalMatch( return null; } +/** @internal */ export function computeMetabaseMappingPhysicalMismatches(inputs: PhysicalMismatchInput[]): PhysicalMismatch[] { const mismatches: PhysicalMismatch[] = []; for (const input of inputs) { @@ -247,6 +254,7 @@ export function computeMetabaseMappingPhysicalMismatches(inputs: PhysicalMismatc return mismatches; } +/** @internal */ export async function refreshMetabaseMapping(args: { client: Pick; currentMappings: Record; @@ -286,6 +294,7 @@ export async function refreshMetabaseMapping(args: { return { drift, physicalMismatches }; } +/** @internal */ export function findBestMatch(mapping: MappingPhysicalInfo, candidates: AutoMatchCandidate[]): AutoMatchResult | null { const engine = mapping.metabaseEngine?.toLowerCase(); if (!engine) { diff --git a/packages/context/src/ingest/adapters/metabase/metabase.adapter.test.ts b/packages/cli/src/context/ingest/adapters/metabase/metabase.adapter.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/metabase.adapter.test.ts rename to packages/cli/src/context/ingest/adapters/metabase/metabase.adapter.test.ts diff --git a/packages/context/src/ingest/adapters/metabase/metabase.adapter.ts b/packages/cli/src/context/ingest/adapters/metabase/metabase.adapter.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/metabase.adapter.ts rename to packages/cli/src/context/ingest/adapters/metabase/metabase.adapter.ts diff --git a/packages/context/src/ingest/adapters/metabase/serialize-card.test.ts b/packages/cli/src/context/ingest/adapters/metabase/serialize-card.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/serialize-card.test.ts rename to packages/cli/src/context/ingest/adapters/metabase/serialize-card.test.ts diff --git a/packages/context/src/ingest/adapters/metabase/serialize-card.ts b/packages/cli/src/context/ingest/adapters/metabase/serialize-card.ts similarity index 98% rename from packages/context/src/ingest/adapters/metabase/serialize-card.ts rename to packages/cli/src/context/ingest/adapters/metabase/serialize-card.ts index 57f1acf9..4cb844f7 100644 --- a/packages/context/src/ingest/adapters/metabase/serialize-card.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/serialize-card.ts @@ -6,6 +6,7 @@ const CARD_REF_RE = /\{\{#(\d+)\}\}/g; * Input TemplateTag shape mirrors `MetabaseClient.getTemplateTags` output. We keep the * shape loose — only `name`, `type`, and optional `cardReference`/`default` are needed here. */ +/** @internal */ export interface InputTemplateTag { name: string; type: string; @@ -13,6 +14,7 @@ export interface InputTemplateTag { defaultValue?: string | null; } +/** @internal */ export function extractReferencedCardIds(templateTags: InputTemplateTag[], sql: string): number[] { const ids = new Set(); for (const tag of templateTags) { @@ -34,7 +36,7 @@ export function extractReferencedCardIds(templateTags: InputTemplateTag[], sql: * care about. The adapter reads whatever the client returns; this helper stays * duck-typed so the client's type can evolve without churn here. */ -export interface InputCard { +interface InputCard { id: number; name: string; description?: string | null; diff --git a/packages/context/src/ingest/adapters/metabase/source-state-port.ts b/packages/cli/src/context/ingest/adapters/metabase/source-state-port.ts similarity index 93% rename from packages/context/src/ingest/adapters/metabase/source-state-port.ts rename to packages/cli/src/context/ingest/adapters/metabase/source-state-port.ts index 16de9369..8b50aae5 100644 --- a/packages/context/src/ingest/adapters/metabase/source-state-port.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/source-state-port.ts @@ -5,7 +5,7 @@ export interface MetabaseSourceStateSelection { metabaseObjectId: number; } -export interface MetabaseSourceStateMapping { +interface MetabaseSourceStateMapping { metabaseDatabaseId: number; metabaseDatabaseName: string | null; metabaseEngine: string | null; diff --git a/packages/context/src/ingest/adapters/metabase/types.test.ts b/packages/cli/src/context/ingest/adapters/metabase/types.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metabase/types.test.ts rename to packages/cli/src/context/ingest/adapters/metabase/types.test.ts diff --git a/packages/context/src/ingest/adapters/metabase/types.ts b/packages/cli/src/context/ingest/adapters/metabase/types.ts similarity index 92% rename from packages/context/src/ingest/adapters/metabase/types.ts rename to packages/cli/src/context/ingest/adapters/metabase/types.ts index fc2ecb87..da84de0c 100644 --- a/packages/context/src/ingest/adapters/metabase/types.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/types.ts @@ -1,9 +1,9 @@ import { z } from 'zod'; -export const metabaseSyncModeSchema = z.enum(['ALL', 'ONLY', 'EXCEPT']); +const metabaseSyncModeSchema = z.enum(['ALL', 'ONLY', 'EXCEPT']); export type MetabaseSyncMode = z.infer; -export const metabaseLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/); +const metabaseLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/); /** * The lean config the adapter needs at `fetch()` time. Lives in the BullMQ payload's @@ -11,6 +11,7 @@ export const metabaseLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a- * job — the persisted state (enabled/disabled, auth, scheduling) lives on the * Metabase connection's `connections.config` JSONB. */ +/** @internal */ export const metabasePullConfigSchema = z.object({ /** The Metabase connection (source) — the thing being swept. */ metabaseConnectionId: metabaseLocalConnectionIdSchema, @@ -25,7 +26,7 @@ export function parseMetabasePullConfig(raw: unknown): MetabasePullConfig { } /** A Metabase column from `card.result_metadata`. Mirrors what the LLM consumes today. */ -export const stagedResultColumnSchema = z.object({ +const stagedResultColumnSchema = z.object({ name: z.string(), display_name: z.string().optional().nullable(), base_type: z.string(), @@ -37,7 +38,7 @@ export const stagedResultColumnSchema = z.object({ export type StagedResultColumn = z.infer; -export const stagedParameterSchema = z.object({ +const stagedParameterSchema = z.object({ id: z.string(), name: z.string(), type: z.string(), @@ -49,7 +50,7 @@ export const stagedParameterSchema = z.object({ export type StagedParameter = z.infer; /** A template tag pulled from an MBQL card's `dataset_query.stages[0].template-tags`. */ -export const stagedTemplateTagSchema = z.object({ +const stagedTemplateTagSchema = z.object({ name: z.string(), type: z.string(), defaultValue: z.string().optional().nullable(), @@ -87,7 +88,7 @@ export const stagedCardFileSchema = z.object({ export type StagedCardFile = z.infer; /** A serialized collection file, `collections/.json`. Minimal — path lives on the card. */ -export const stagedCollectionFileSchema = z.object({ +const stagedCollectionFileSchema = z.object({ metabaseId: z.union([z.number().int(), z.literal('root')]), name: z.string(), parentId: z.union([z.number().int(), z.literal('root')]).nullable(), @@ -96,7 +97,7 @@ export const stagedCollectionFileSchema = z.object({ export type StagedCollectionFile = z.infer; /** A serialized database-mapping snapshot, `databases/.json`. */ -export const stagedDatabaseFileSchema = z.object({ +const stagedDatabaseFileSchema = z.object({ metabaseDatabaseId: z.number().int().positive(), metabaseDatabaseName: z.string(), metabaseEngine: z.string().nullable(), diff --git a/packages/context/src/ingest/adapters/metricflow/chunk.test.ts b/packages/cli/src/context/ingest/adapters/metricflow/chunk.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metricflow/chunk.test.ts rename to packages/cli/src/context/ingest/adapters/metricflow/chunk.test.ts diff --git a/packages/context/src/ingest/adapters/metricflow/chunk.ts b/packages/cli/src/context/ingest/adapters/metricflow/chunk.ts similarity index 100% rename from packages/context/src/ingest/adapters/metricflow/chunk.ts rename to packages/cli/src/context/ingest/adapters/metricflow/chunk.ts diff --git a/packages/context/src/ingest/adapters/metricflow/deep-parse.test.ts b/packages/cli/src/context/ingest/adapters/metricflow/deep-parse.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metricflow/deep-parse.test.ts rename to packages/cli/src/context/ingest/adapters/metricflow/deep-parse.test.ts diff --git a/packages/context/src/ingest/adapters/metricflow/deep-parse.ts b/packages/cli/src/context/ingest/adapters/metricflow/deep-parse.ts similarity index 99% rename from packages/context/src/ingest/adapters/metricflow/deep-parse.ts rename to packages/cli/src/context/ingest/adapters/metricflow/deep-parse.ts index a76e94f7..8c10d638 100644 --- a/packages/context/src/ingest/adapters/metricflow/deep-parse.ts +++ b/packages/cli/src/context/ingest/adapters/metricflow/deep-parse.ts @@ -1,7 +1,7 @@ import { parse as parseYaml } from 'yaml'; -import { noopLogger, type KtxLogger } from '../../../core/index.js'; +import { noopLogger, type KtxLogger } from '../../../../context/core/config.js'; -export interface DimensionDefinition { +interface DimensionDefinition { name: string; column: string; type: string; @@ -9,7 +9,7 @@ export interface DimensionDefinition { description?: string; } -export interface SimpleMeasureDefinition { +interface SimpleMeasureDefinition { type: 'simple'; name: string; column: string; @@ -20,7 +20,7 @@ export interface SimpleMeasureDefinition { cumulative?: boolean; } -export type MeasureDefinition = +type MeasureDefinition = | SimpleMeasureDefinition | { type: 'derived'; @@ -186,6 +186,7 @@ export function parseMetricflowFiles( return parser.parseFiles(files); } +/** @internal */ export function translateMetricflowJinjaFilter(filter: string): string { return new MetricflowDeepParser(noopLogger).translateJinjaFilter(filter); } diff --git a/packages/context/src/ingest/adapters/metricflow/detect.test.ts b/packages/cli/src/context/ingest/adapters/metricflow/detect.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metricflow/detect.test.ts rename to packages/cli/src/context/ingest/adapters/metricflow/detect.test.ts diff --git a/packages/context/src/ingest/adapters/metricflow/detect.ts b/packages/cli/src/context/ingest/adapters/metricflow/detect.ts similarity index 100% rename from packages/context/src/ingest/adapters/metricflow/detect.ts rename to packages/cli/src/context/ingest/adapters/metricflow/detect.ts diff --git a/packages/context/src/ingest/adapters/metricflow/fetch.test.ts b/packages/cli/src/context/ingest/adapters/metricflow/fetch.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metricflow/fetch.test.ts rename to packages/cli/src/context/ingest/adapters/metricflow/fetch.test.ts diff --git a/packages/context/src/ingest/adapters/metricflow/fetch.ts b/packages/cli/src/context/ingest/adapters/metricflow/fetch.ts similarity index 100% rename from packages/context/src/ingest/adapters/metricflow/fetch.ts rename to packages/cli/src/context/ingest/adapters/metricflow/fetch.ts diff --git a/packages/context/src/ingest/adapters/metricflow/graph.test.ts b/packages/cli/src/context/ingest/adapters/metricflow/graph.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metricflow/graph.test.ts rename to packages/cli/src/context/ingest/adapters/metricflow/graph.test.ts diff --git a/packages/context/src/ingest/adapters/metricflow/graph.ts b/packages/cli/src/context/ingest/adapters/metricflow/graph.ts similarity index 100% rename from packages/context/src/ingest/adapters/metricflow/graph.ts rename to packages/cli/src/context/ingest/adapters/metricflow/graph.ts diff --git a/packages/context/src/ingest/adapters/metricflow/import-semantic-models.test.ts b/packages/cli/src/context/ingest/adapters/metricflow/import-semantic-models.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metricflow/import-semantic-models.test.ts rename to packages/cli/src/context/ingest/adapters/metricflow/import-semantic-models.test.ts diff --git a/packages/context/src/ingest/adapters/metricflow/import-semantic-models.ts b/packages/cli/src/context/ingest/adapters/metricflow/import-semantic-models.ts similarity index 95% rename from packages/context/src/ingest/adapters/metricflow/import-semantic-models.ts rename to packages/cli/src/context/ingest/adapters/metricflow/import-semantic-models.ts index 7d80a9c0..9aa6181d 100644 --- a/packages/context/src/ingest/adapters/metricflow/import-semantic-models.ts +++ b/packages/cli/src/context/ingest/adapters/metricflow/import-semantic-models.ts @@ -1,10 +1,6 @@ -import type { SemanticLayerService, SemanticLayerSource } from '../../../sl/index.js'; -import { - addTouchedSlSource, - createTouchedSlSources, - listTouchedSlSources, - type TouchedSlSource, -} from '../../../tools/index.js'; +import type { SemanticLayerService } from '../../../../context/sl/semantic-layer.service.js'; +import type { SemanticLayerSource } from '../../../../context/sl/types.js'; +import { addTouchedSlSource, createTouchedSlSources, listTouchedSlSources, type TouchedSlSource } from '../../../../context/tools/touched-sl-sources.js'; import type { MetricFlowParseResult } from './deep-parse.js'; import { buildMetricflowJoinsForModel, @@ -29,12 +25,12 @@ export interface MetricFlowImportResult { touchedSources: TouchedSlSource[]; } -export type MetricflowSemanticLayerWriter = Pick< +type MetricflowSemanticLayerWriter = Pick< SemanticLayerService, 'getManifestEntry' | 'isManifestBacked' | 'loadAllSources' | 'loadSource' | 'writeSource' >; -export type MetricflowSemanticLayerService = MetricflowSemanticLayerWriter & { +type MetricflowSemanticLayerService = MetricflowSemanticLayerWriter & { forWorktree(workdir: string): MetricflowSemanticLayerWriter; }; diff --git a/packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts b/packages/cli/src/context/ingest/adapters/metricflow/metricflow.adapter.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metricflow/metricflow.adapter.test.ts rename to packages/cli/src/context/ingest/adapters/metricflow/metricflow.adapter.test.ts diff --git a/packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts b/packages/cli/src/context/ingest/adapters/metricflow/metricflow.adapter.ts similarity index 100% rename from packages/context/src/ingest/adapters/metricflow/metricflow.adapter.ts rename to packages/cli/src/context/ingest/adapters/metricflow/metricflow.adapter.ts diff --git a/packages/context/src/ingest/adapters/metricflow/parse.test.ts b/packages/cli/src/context/ingest/adapters/metricflow/parse.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metricflow/parse.test.ts rename to packages/cli/src/context/ingest/adapters/metricflow/parse.test.ts diff --git a/packages/context/src/ingest/adapters/metricflow/parse.ts b/packages/cli/src/context/ingest/adapters/metricflow/parse.ts similarity index 97% rename from packages/context/src/ingest/adapters/metricflow/parse.ts rename to packages/cli/src/context/ingest/adapters/metricflow/parse.ts index 935858d0..6232e548 100644 --- a/packages/context/src/ingest/adapters/metricflow/parse.ts +++ b/packages/cli/src/context/ingest/adapters/metricflow/parse.ts @@ -2,7 +2,7 @@ import { readdir, readFile } from 'node:fs/promises'; import { join, relative } from 'node:path'; import { parse as parseYaml } from 'yaml'; -export interface ParsedMetricFlowSemanticModel { +interface ParsedMetricFlowSemanticModel { /** Path relative to stagedDir, e.g. "models/orders.yml". */ path: string; /** `name:` on the semantic_model. */ @@ -24,9 +24,9 @@ export interface ParsedMetricFlowSemanticModel { defaultTimeDimension: string | null; } -export type MetricFlowMetricType = 'simple' | 'derived' | 'cumulative' | 'ratio' | 'conversion'; +type MetricFlowMetricType = 'simple' | 'derived' | 'cumulative' | 'ratio' | 'conversion'; -export interface ParsedMetricFlowMetric { +interface ParsedMetricFlowMetric { path: string; name: string; type: MetricFlowMetricType; diff --git a/packages/context/src/ingest/adapters/metricflow/projection-config.ts b/packages/cli/src/context/ingest/adapters/metricflow/projection-config.ts similarity index 100% rename from packages/context/src/ingest/adapters/metricflow/projection-config.ts rename to packages/cli/src/context/ingest/adapters/metricflow/projection-config.ts diff --git a/packages/context/src/ingest/adapters/metricflow/pull-config.test.ts b/packages/cli/src/context/ingest/adapters/metricflow/pull-config.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/metricflow/pull-config.test.ts rename to packages/cli/src/context/ingest/adapters/metricflow/pull-config.test.ts diff --git a/packages/context/src/ingest/adapters/metricflow/pull-config.ts b/packages/cli/src/context/ingest/adapters/metricflow/pull-config.ts similarity index 96% rename from packages/context/src/ingest/adapters/metricflow/pull-config.ts rename to packages/cli/src/context/ingest/adapters/metricflow/pull-config.ts index ad38b033..8fccbeee 100644 --- a/packages/context/src/ingest/adapters/metricflow/pull-config.ts +++ b/packages/cli/src/context/ingest/adapters/metricflow/pull-config.ts @@ -1,7 +1,7 @@ import { z } from 'zod'; import { parsedTargetTableSchema } from '../../parsed-target-table.js'; -export const metricflowPullConfigSchema = z.object({ +const metricflowPullConfigSchema = z.object({ repoUrl: z.string().url(), branch: z.string().default('main'), path: z.string().nullable().default(null), diff --git a/packages/context/src/ingest/adapters/metricflow/semantic-models.test.ts b/packages/cli/src/context/ingest/adapters/metricflow/semantic-models.test.ts similarity index 98% rename from packages/context/src/ingest/adapters/metricflow/semantic-models.test.ts rename to packages/cli/src/context/ingest/adapters/metricflow/semantic-models.test.ts index 9ad88563..c22ac97d 100644 --- a/packages/context/src/ingest/adapters/metricflow/semantic-models.test.ts +++ b/packages/cli/src/context/ingest/adapters/metricflow/semantic-models.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { composeOverlay, type SemanticLayerSource } from '../../../sl/index.js'; +import { composeOverlay } from '../../../../context/sl/semantic-layer.service.js'; +import type { SemanticLayerSource } from '../../../../context/sl/types.js'; import type { ParsedCrossModelMetric, ParsedMetricflowRelationship, ParsedSemanticModel } from './deep-parse.js'; import { buildMetricflowColumns, diff --git a/packages/context/src/ingest/adapters/metricflow/semantic-models.ts b/packages/cli/src/context/ingest/adapters/metricflow/semantic-models.ts similarity index 95% rename from packages/context/src/ingest/adapters/metricflow/semantic-models.ts rename to packages/cli/src/context/ingest/adapters/metricflow/semantic-models.ts index f2f84c61..7137bf0d 100644 --- a/packages/context/src/ingest/adapters/metricflow/semantic-models.ts +++ b/packages/cli/src/context/ingest/adapters/metricflow/semantic-models.ts @@ -1,4 +1,4 @@ -import type { SemanticLayerSource } from '../../../sl/index.js'; +import type { SemanticLayerSource } from '../../../../context/sl/types.js'; import type { ParsedCrossModelMetric, ParsedMetricflowRelationship, @@ -25,6 +25,7 @@ export type MetricflowSemanticModelJoin = SemanticLayerSource['joins'][number]; export type MetricflowWritableSemanticLayerSource = Pick & Partial>; +/** @internal */ export function toKebabCaseMetricflowName(str: string): string { return str .toLowerCase() @@ -32,6 +33,7 @@ export function toKebabCaseMetricflowName(str: string): string { .replace(/^-|-$/g, ''); } +/** @internal */ export function mapSemanticModelToSource(model: ParsedSemanticModel, tableRef?: string): SemanticLayerSource { return { name: toKebabCaseMetricflowName(model.modelRef), @@ -177,7 +179,7 @@ export function buildMetricflowSemanticModelSource( return mapMetricflowSemanticModelToStandalone(model, sourceName, matchedTable?.name ?? model.modelRef, joins); } -export function buildMetricflowMeasures(model: ParsedSemanticModel): SemanticLayerSource['measures'] { +function buildMetricflowMeasures(model: ParsedSemanticModel): SemanticLayerSource['measures'] { return model.measures.map((measure) => { if (measure.type === 'simple') { return { @@ -195,6 +197,7 @@ export function buildMetricflowMeasures(model: ParsedSemanticModel): SemanticLay }); } +/** @internal */ export function buildMetricflowColumns(model: ParsedSemanticModel): SemanticLayerSource['columns'] { const columns: SemanticLayerSource['columns'] = model.dimensions.map((dimension) => ({ name: dimension.column, @@ -243,6 +246,7 @@ export function getMetricflowAvailableColumnNames(context: MetricflowSemanticMod return new Set(columns.map((column) => column.name.toLowerCase())); } +/** @internal */ export function countImportableMetricflowRelationships( relationships: ParsedMetricflowRelationship[], hostTables: MetricflowHostTable[], @@ -336,10 +340,11 @@ function mergeMetricflowJoins( return [...baseJoins, ...newJoins]; } -export function normalizeMetricflowJoinOn(on: string): string { +function normalizeMetricflowJoinOn(on: string): string { return on.replace(/\s+/g, ' ').trim(); } +/** @internal */ export function rewriteMetricflowManifestJoins( joins: SemanticLayerSource['joins'], sourceNameByManifestName: Map, @@ -351,7 +356,7 @@ export function rewriteMetricflowManifestJoins( })); } -export function rewriteMetricflowJoinOn(on: string, sourceNameByManifestName: Map): string { +function rewriteMetricflowJoinOn(on: string, sourceNameByManifestName: Map): string { const parts = on.split('='); if (parts.length !== 2) { return on; @@ -366,7 +371,7 @@ export function rewriteMetricflowJoinOn(on: string, sourceNameByManifestName: Ma return `${leftTable}.${left.column} = ${rightTable}.${right.column}`; } -export function parseMetricflowJoinReference(ref: string): { table: string; column: string } | null { +function parseMetricflowJoinReference(ref: string): { table: string; column: string } | null { const lastDot = ref.lastIndexOf('.'); if (lastDot <= 0 || lastDot === ref.length - 1) { return null; diff --git a/packages/context/src/ingest/adapters/notion/chunk.ts b/packages/cli/src/context/ingest/adapters/notion/chunk.ts similarity index 99% rename from packages/context/src/ingest/adapters/notion/chunk.ts rename to packages/cli/src/context/ingest/adapters/notion/chunk.ts index bdae8622..10f5d98d 100644 --- a/packages/context/src/ingest/adapters/notion/chunk.ts +++ b/packages/cli/src/context/ingest/adapters/notion/chunk.ts @@ -5,6 +5,7 @@ import type { ChunkResult, DiffSet, ScopeDescriptor, WorkUnit } from '../../type import { notionManifestSchema, notionMetadataSchema } from './types.js'; const MAX_NOTION_WORK_UNIT_CHARS = 40_000; +/** @internal */ export const NOTION_ORG_KNOWLEDGE_WARNING = 'Anything accessible to this Notion integration can become organization knowledge.'; const NOTION_SL_WRITE_GUIDANCE = diff --git a/packages/context/src/ingest/adapters/notion/cluster.test.ts b/packages/cli/src/context/ingest/adapters/notion/cluster.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/notion/cluster.test.ts rename to packages/cli/src/context/ingest/adapters/notion/cluster.test.ts diff --git a/packages/context/src/ingest/adapters/notion/cluster.ts b/packages/cli/src/context/ingest/adapters/notion/cluster.ts similarity index 99% rename from packages/context/src/ingest/adapters/notion/cluster.ts rename to packages/cli/src/context/ingest/adapters/notion/cluster.ts index 090ff4f7..db5fbcb5 100644 --- a/packages/context/src/ingest/adapters/notion/cluster.ts +++ b/packages/cli/src/context/ingest/adapters/notion/cluster.ts @@ -5,6 +5,7 @@ import { kmeans, pickK } from '../../clustering/kmeans.js'; import type { WorkUnit } from '../../types.js'; import { notionMetadataSchema } from './types.js'; +/** @internal */ export const MIN_PAGES_TO_CLUSTER = 5; const CLUSTER_TEXT_BODY_CHARS = 1024; const CLUSTER_SEED = 42; diff --git a/packages/context/src/ingest/adapters/notion/detect.ts b/packages/cli/src/context/ingest/adapters/notion/detect.ts similarity index 100% rename from packages/context/src/ingest/adapters/notion/detect.ts rename to packages/cli/src/context/ingest/adapters/notion/detect.ts diff --git a/packages/context/src/ingest/adapters/notion/fetch.test.ts b/packages/cli/src/context/ingest/adapters/notion/fetch.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/notion/fetch.test.ts rename to packages/cli/src/context/ingest/adapters/notion/fetch.test.ts diff --git a/packages/context/src/ingest/adapters/notion/fetch.ts b/packages/cli/src/context/ingest/adapters/notion/fetch.ts similarity index 100% rename from packages/context/src/ingest/adapters/notion/fetch.ts rename to packages/cli/src/context/ingest/adapters/notion/fetch.ts diff --git a/packages/context/src/ingest/adapters/notion/local-state-store.test.ts b/packages/cli/src/context/ingest/adapters/notion/local-state-store.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/notion/local-state-store.test.ts rename to packages/cli/src/context/ingest/adapters/notion/local-state-store.test.ts diff --git a/packages/context/src/ingest/adapters/notion/local-state-store.ts b/packages/cli/src/context/ingest/adapters/notion/local-state-store.ts similarity index 100% rename from packages/context/src/ingest/adapters/notion/local-state-store.ts rename to packages/cli/src/context/ingest/adapters/notion/local-state-store.ts diff --git a/packages/context/src/ingest/adapters/notion/normalize.test.ts b/packages/cli/src/context/ingest/adapters/notion/normalize.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/notion/normalize.test.ts rename to packages/cli/src/context/ingest/adapters/notion/normalize.test.ts diff --git a/packages/context/src/ingest/adapters/notion/normalize.ts b/packages/cli/src/context/ingest/adapters/notion/normalize.ts similarity index 99% rename from packages/context/src/ingest/adapters/notion/normalize.ts rename to packages/cli/src/context/ingest/adapters/notion/normalize.ts index 0e608697..fad1ff95 100644 --- a/packages/context/src/ingest/adapters/notion/normalize.ts +++ b/packages/cli/src/context/ingest/adapters/notion/normalize.ts @@ -14,6 +14,7 @@ function richTextToMarkdown(value: unknown): string { .trim(); } +/** @internal */ export function propertyValueToText(value: unknown): string { if (!value || typeof value !== 'object' || !('type' in value)) { return ''; diff --git a/packages/context/src/ingest/adapters/notion/notion-client.test.ts b/packages/cli/src/context/ingest/adapters/notion/notion-client.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/notion/notion-client.test.ts rename to packages/cli/src/context/ingest/adapters/notion/notion-client.test.ts diff --git a/packages/context/src/ingest/adapters/notion/notion-client.ts b/packages/cli/src/context/ingest/adapters/notion/notion-client.ts similarity index 99% rename from packages/context/src/ingest/adapters/notion/notion-client.ts rename to packages/cli/src/context/ingest/adapters/notion/notion-client.ts index c7cb007c..0bbde862 100644 --- a/packages/context/src/ingest/adapters/notion/notion-client.ts +++ b/packages/cli/src/context/ingest/adapters/notion/notion-client.ts @@ -89,6 +89,7 @@ function shouldRetryNotionError(error: unknown): boolean { return code === 'rate_limited' || transientErrorCodes.has(code ?? '') || transientStatusCodes.has(status ?? 0); } +/** @internal */ export async function retryNotionRequest(operation: () => Promise, options: RetryOptions = {}): Promise { const maxAttempts = options.maxAttempts ?? 4; const sleep = options.sleep ?? defaultSleep; diff --git a/packages/context/src/ingest/adapters/notion/notion.adapter.test.ts b/packages/cli/src/context/ingest/adapters/notion/notion.adapter.test.ts similarity index 100% rename from packages/context/src/ingest/adapters/notion/notion.adapter.test.ts rename to packages/cli/src/context/ingest/adapters/notion/notion.adapter.test.ts diff --git a/packages/context/src/ingest/adapters/notion/notion.adapter.ts b/packages/cli/src/context/ingest/adapters/notion/notion.adapter.ts similarity index 100% rename from packages/context/src/ingest/adapters/notion/notion.adapter.ts rename to packages/cli/src/context/ingest/adapters/notion/notion.adapter.ts diff --git a/packages/context/src/ingest/adapters/notion/pull-config.ts b/packages/cli/src/context/ingest/adapters/notion/pull-config.ts similarity index 100% rename from packages/context/src/ingest/adapters/notion/pull-config.ts rename to packages/cli/src/context/ingest/adapters/notion/pull-config.ts diff --git a/packages/context/src/ingest/adapters/notion/types.ts b/packages/cli/src/context/ingest/adapters/notion/types.ts similarity index 100% rename from packages/context/src/ingest/adapters/notion/types.ts rename to packages/cli/src/context/ingest/adapters/notion/types.ts diff --git a/packages/context/src/ingest/artifact-gates.test.ts b/packages/cli/src/context/ingest/artifact-gates.test.ts similarity index 100% rename from packages/context/src/ingest/artifact-gates.test.ts rename to packages/cli/src/context/ingest/artifact-gates.test.ts diff --git a/packages/context/src/ingest/artifact-gates.ts b/packages/cli/src/context/ingest/artifact-gates.ts similarity index 95% rename from packages/context/src/ingest/artifact-gates.ts rename to packages/cli/src/context/ingest/artifact-gates.ts index 44f7a66f..35735575 100644 --- a/packages/context/src/ingest/artifact-gates.ts +++ b/packages/cli/src/context/ingest/artifact-gates.ts @@ -1,10 +1,10 @@ -import type { SemanticLayerService } from '../sl/index.js'; -import type { TouchedSlSource } from '../tools/index.js'; -import type { KnowledgeWikiService } from '../wiki/index.js'; +import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js'; +import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; import { findMissingWikiRefs } from '../wiki/wiki-ref-validation.js'; import { findInvalidWikiBodyRefs } from './wiki-body-refs.js'; -export interface TouchedValidationResult { +interface TouchedValidationResult { invalidSources: string[]; validSources: string[]; } diff --git a/packages/context/src/ingest/canonical-pins.test.ts b/packages/cli/src/context/ingest/canonical-pins.test.ts similarity index 100% rename from packages/context/src/ingest/canonical-pins.test.ts rename to packages/cli/src/context/ingest/canonical-pins.test.ts diff --git a/packages/context/src/ingest/canonical-pins.ts b/packages/cli/src/context/ingest/canonical-pins.ts similarity index 100% rename from packages/context/src/ingest/canonical-pins.ts rename to packages/cli/src/context/ingest/canonical-pins.ts diff --git a/packages/context/src/ingest/clustering/kmeans.test.ts b/packages/cli/src/context/ingest/clustering/kmeans.test.ts similarity index 100% rename from packages/context/src/ingest/clustering/kmeans.test.ts rename to packages/cli/src/context/ingest/clustering/kmeans.test.ts diff --git a/packages/context/src/ingest/clustering/kmeans.ts b/packages/cli/src/context/ingest/clustering/kmeans.ts similarity index 100% rename from packages/context/src/ingest/clustering/kmeans.ts rename to packages/cli/src/context/ingest/clustering/kmeans.ts diff --git a/packages/context/src/ingest/context-candidates/candidate-dedup.service.test.ts b/packages/cli/src/context/ingest/context-candidates/candidate-dedup.service.test.ts similarity index 100% rename from packages/context/src/ingest/context-candidates/candidate-dedup.service.test.ts rename to packages/cli/src/context/ingest/context-candidates/candidate-dedup.service.test.ts diff --git a/packages/context/src/ingest/context-candidates/candidate-dedup.service.ts b/packages/cli/src/context/ingest/context-candidates/candidate-dedup.service.ts similarity index 99% rename from packages/context/src/ingest/context-candidates/candidate-dedup.service.ts rename to packages/cli/src/context/ingest/context-candidates/candidate-dedup.service.ts index e9c0e878..d7f1a07b 100644 --- a/packages/context/src/ingest/context-candidates/candidate-dedup.service.ts +++ b/packages/cli/src/context/ingest/context-candidates/candidate-dedup.service.ts @@ -1,4 +1,4 @@ -import { type KtxLogger, noopLogger } from '../../core/index.js'; +import { type KtxLogger, noopLogger } from '../../../context/core/config.js'; import type { CandidateDedupResult, ContextCandidateForDedup, JsonValue } from '../ports.js'; import { buildContextCandidateEmbeddingText } from './embedding-text.js'; import type { ContextCandidateStorePort } from './store.js'; diff --git a/packages/context/src/ingest/context-candidates/context-candidate-carryforward.service.test.ts b/packages/cli/src/context/ingest/context-candidates/context-candidate-carryforward.service.test.ts similarity index 100% rename from packages/context/src/ingest/context-candidates/context-candidate-carryforward.service.test.ts rename to packages/cli/src/context/ingest/context-candidates/context-candidate-carryforward.service.test.ts diff --git a/packages/context/src/ingest/context-candidates/context-candidate-carryforward.service.ts b/packages/cli/src/context/ingest/context-candidates/context-candidate-carryforward.service.ts similarity index 98% rename from packages/context/src/ingest/context-candidates/context-candidate-carryforward.service.ts rename to packages/cli/src/context/ingest/context-candidates/context-candidate-carryforward.service.ts index 74151e7e..19ec9f39 100644 --- a/packages/context/src/ingest/context-candidates/context-candidate-carryforward.service.ts +++ b/packages/cli/src/context/ingest/context-candidates/context-candidate-carryforward.service.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { type KtxLogger, noopLogger } from '../../core/index.js'; +import { type KtxLogger, noopLogger } from '../../../context/core/config.js'; import type { JsonValue } from '../ports.js'; import type { ContextCandidateStorePort } from './store.js'; import type { diff --git a/packages/context/src/ingest/context-candidates/curator-pagination.service.test.ts b/packages/cli/src/context/ingest/context-candidates/curator-pagination.service.test.ts similarity index 100% rename from packages/context/src/ingest/context-candidates/curator-pagination.service.test.ts rename to packages/cli/src/context/ingest/context-candidates/curator-pagination.service.test.ts diff --git a/packages/context/src/ingest/context-candidates/curator-pagination.service.ts b/packages/cli/src/context/ingest/context-candidates/curator-pagination.service.ts similarity index 97% rename from packages/context/src/ingest/context-candidates/curator-pagination.service.ts rename to packages/cli/src/context/ingest/context-candidates/curator-pagination.service.ts index 5130b20b..348544ca 100644 --- a/packages/context/src/ingest/context-candidates/curator-pagination.service.ts +++ b/packages/cli/src/context/ingest/context-candidates/curator-pagination.service.ts @@ -1,7 +1,7 @@ -import type { KtxModelRole } from '@ktx/llm'; -import { type KtxLogger, noopLogger } from '../../core/index.js'; -import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../llm/index.js'; -import type { MemoryAction } from '../../memory/index.js'; +import type { KtxModelRole } from '../../../llm/types.js'; +import { type KtxLogger, noopLogger } from '../../../context/core/config.js'; +import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; +import type { MemoryAction } from '../../../context/memory/types.js'; import type { ContextCandidateForDedup, CuratorPaginationPort, CuratorPaginationReport } from '../ports.js'; import type { ReconcileCandidateForPrompt, diff --git a/packages/context/src/ingest/context-candidates/embedding-text.test.ts b/packages/cli/src/context/ingest/context-candidates/embedding-text.test.ts similarity index 100% rename from packages/context/src/ingest/context-candidates/embedding-text.test.ts rename to packages/cli/src/context/ingest/context-candidates/embedding-text.test.ts diff --git a/packages/context/src/ingest/context-candidates/embedding-text.ts b/packages/cli/src/context/ingest/context-candidates/embedding-text.ts similarity index 100% rename from packages/context/src/ingest/context-candidates/embedding-text.ts rename to packages/cli/src/context/ingest/context-candidates/embedding-text.ts diff --git a/packages/context/src/ingest/context-candidates/store.test.ts b/packages/cli/src/context/ingest/context-candidates/store.test.ts similarity index 100% rename from packages/context/src/ingest/context-candidates/store.test.ts rename to packages/cli/src/context/ingest/context-candidates/store.test.ts diff --git a/packages/context/src/ingest/context-candidates/store.ts b/packages/cli/src/context/ingest/context-candidates/store.ts similarity index 100% rename from packages/context/src/ingest/context-candidates/store.ts rename to packages/cli/src/context/ingest/context-candidates/store.ts diff --git a/packages/context/src/ingest/context-candidates/types.ts b/packages/cli/src/context/ingest/context-candidates/types.ts similarity index 81% rename from packages/context/src/ingest/context-candidates/types.ts rename to packages/cli/src/context/ingest/context-candidates/types.ts index 2f8b7db2..c2e3c3c4 100644 --- a/packages/context/src/ingest/context-candidates/types.ts +++ b/packages/cli/src/context/ingest/context-candidates/types.ts @@ -1,7 +1,7 @@ import type { JsonValue } from '../ports.js'; -export type ContextCandidateActionHint = 'create' | 'update' | 'merge' | 'conflict' | 'skip'; -export type ContextCandidateStatus = 'pending' | 'promoted' | 'merged' | 'rejected' | 'conflict'; +type ContextCandidateActionHint = 'create' | 'update' | 'merge' | 'conflict' | 'skip'; +type ContextCandidateStatus = 'pending' | 'promoted' | 'merged' | 'rejected' | 'conflict'; export type ContextCandidateRejectionReason = | 'low_score' | 'duplicates_existing_wiki' @@ -10,20 +10,9 @@ export type ContextCandidateRejectionReason = | 'exceeded_run_budget' | 'exceeded_curator_passes' | 'curator_pass_error'; -export type ContextCandidateLane = 'light' | 'full' | null; -export type ContextCandidateScoreAggregation = 'max' | 'mean' | 'sum'; +type ContextCandidateLane = 'light' | 'full' | null; +type ContextCandidateScoreAggregation = 'max' | 'mean' | 'sum'; -export interface ContextCandidateForPrompt { - candidateKey: string; - topic: string; - assertion: string; - rationale: string; - actionHint: string; - status: string; - promotionScore: number; - suggestedPageKey: string | null; - evidenceRefs: JsonValue; -} export interface ContextCandidateVerdictSummary { pending: number; diff --git a/packages/context/src/ingest/context-evidence/context-evidence-index.service.test.ts b/packages/cli/src/context/ingest/context-evidence/context-evidence-index.service.test.ts similarity index 100% rename from packages/context/src/ingest/context-evidence/context-evidence-index.service.test.ts rename to packages/cli/src/context/ingest/context-evidence/context-evidence-index.service.test.ts diff --git a/packages/context/src/ingest/context-evidence/context-evidence-index.service.ts b/packages/cli/src/context/ingest/context-evidence/context-evidence-index.service.ts similarity index 99% rename from packages/context/src/ingest/context-evidence/context-evidence-index.service.ts rename to packages/cli/src/context/ingest/context-evidence/context-evidence-index.service.ts index 7b54e80f..fe79fef7 100644 --- a/packages/context/src/ingest/context-evidence/context-evidence-index.service.ts +++ b/packages/cli/src/context/ingest/context-evidence/context-evidence-index.service.ts @@ -1,7 +1,7 @@ import { createHash } from 'node:crypto'; import { readdir, readFile } from 'node:fs/promises'; import { basename, dirname, join, relative } from 'node:path'; -import { noopLogger, type KtxLogger } from '../../core/index.js'; +import { noopLogger, type KtxLogger } from '../../../context/core/config.js'; import type { JsonValue } from '../ports.js'; import type { DiffSet } from '../types.js'; import type { ContextEvidenceIndexStorePort } from './store.js'; diff --git a/packages/context/src/ingest/context-evidence/sqlite-context-evidence-store.test.ts b/packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.test.ts similarity index 99% rename from packages/context/src/ingest/context-evidence/sqlite-context-evidence-store.test.ts rename to packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.test.ts index 0767a6be..6b575c00 100644 --- a/packages/context/src/ingest/context-evidence/sqlite-context-evidence-store.test.ts +++ b/packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import type { InsertContextCandidateInput } from '../context-candidates/index.js'; +import type { InsertContextCandidateInput } from '../../../context/ingest/context-candidates/types.js'; import type { JsonValue } from '../ports.js'; import { SqliteContextEvidenceStore } from './sqlite-context-evidence-store.js'; diff --git a/packages/context/src/ingest/context-evidence/sqlite-context-evidence-store.ts b/packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.ts similarity index 98% rename from packages/context/src/ingest/context-evidence/sqlite-context-evidence-store.ts rename to packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.ts index 5d18394f..adba683c 100644 --- a/packages/context/src/ingest/context-evidence/sqlite-context-evidence-store.ts +++ b/packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.ts @@ -2,7 +2,8 @@ import { randomUUID } from 'node:crypto'; import { mkdirSync } from 'node:fs'; import { dirname } from 'node:path'; import Database from 'better-sqlite3'; -import { HybridSearchCore, type SearchCandidateGenerator, type SearchLaneBreakdown } from '../../search/index.js'; +import { HybridSearchCore } from '../../../context/search/hybrid-search-core.js'; +import type { SearchCandidateGenerator, SearchLaneBreakdown } from '../../../context/search/types.js'; import type { ContextCandidateStatusResult, ContextEvidenceChunkForCandidate, @@ -14,16 +15,9 @@ import type { ContextEvidenceSearchResult, ContextEvidenceToolStorePort, } from '../../tools/context-evidence-tool-store.js'; -import type { - BudgetExhaustedCandidateForCarryForward, - ContextCandidateRejectionReason, - ContextCandidateStorePort, - ContextCandidateVerdictSummary, - CurrentRunEvidenceChunkForCarryForward, - InsertContextCandidateInput, - MarkContextCandidateClusterInput, -} from '../context-candidates/index.js'; -import type { PageTriageEvidenceChunk, PageTriageStorePort } from '../page-triage/index.js'; +import type { BudgetExhaustedCandidateForCarryForward, ContextCandidateRejectionReason, ContextCandidateVerdictSummary, CurrentRunEvidenceChunkForCarryForward, InsertContextCandidateInput, MarkContextCandidateClusterInput } from '../../../context/ingest/context-candidates/types.js'; +import type { ContextCandidateStorePort } from '../../../context/ingest/context-candidates/store.js'; +import type { PageTriageEvidenceChunk, PageTriageStorePort } from '../../../context/ingest/page-triage/page-triage.service.js'; import type { ContextCandidateForDedup, ContextCandidateSummary, JsonValue } from '../ports.js'; import type { ContextEvidenceIndexStorePort } from './store.js'; import type { diff --git a/packages/context/src/ingest/context-evidence/store.test.ts b/packages/cli/src/context/ingest/context-evidence/store.test.ts similarity index 100% rename from packages/context/src/ingest/context-evidence/store.test.ts rename to packages/cli/src/context/ingest/context-evidence/store.test.ts diff --git a/packages/context/src/ingest/context-evidence/store.ts b/packages/cli/src/context/ingest/context-evidence/store.ts similarity index 100% rename from packages/context/src/ingest/context-evidence/store.ts rename to packages/cli/src/context/ingest/context-evidence/store.ts diff --git a/packages/context/src/ingest/context-evidence/types.ts b/packages/cli/src/context/ingest/context-evidence/types.ts similarity index 100% rename from packages/context/src/ingest/context-evidence/types.ts rename to packages/cli/src/context/ingest/context-evidence/types.ts diff --git a/packages/context/src/ingest/dbt-shared/project-vars.test.ts b/packages/cli/src/context/ingest/dbt-shared/project-vars.test.ts similarity index 100% rename from packages/context/src/ingest/dbt-shared/project-vars.test.ts rename to packages/cli/src/context/ingest/dbt-shared/project-vars.test.ts diff --git a/packages/context/src/ingest/dbt-shared/project-vars.ts b/packages/cli/src/context/ingest/dbt-shared/project-vars.ts similarity index 98% rename from packages/context/src/ingest/dbt-shared/project-vars.ts rename to packages/cli/src/context/ingest/dbt-shared/project-vars.ts index 2900e08f..3750e246 100644 --- a/packages/context/src/ingest/dbt-shared/project-vars.ts +++ b/packages/cli/src/context/ingest/dbt-shared/project-vars.ts @@ -18,6 +18,7 @@ export interface ResolveJinjaVariablesResult { unresolvedVars: string[]; } +/** @internal */ export function parseProjectVars(yamlContent: string): Map { const variables = new Map(); const project = parseProjectYaml(yamlContent); @@ -30,6 +31,7 @@ export function parseProjectVars(yamlContent: string): Map { return variables; } +/** @internal */ export function parseProjectName(yamlContent: string): string | null { const project = parseProjectYaml(yamlContent); diff --git a/packages/context/src/ingest/dbt-shared/schema-files.test.ts b/packages/cli/src/context/ingest/dbt-shared/schema-files.test.ts similarity index 100% rename from packages/context/src/ingest/dbt-shared/schema-files.test.ts rename to packages/cli/src/context/ingest/dbt-shared/schema-files.test.ts diff --git a/packages/context/src/ingest/dbt-shared/schema-files.ts b/packages/cli/src/context/ingest/dbt-shared/schema-files.ts similarity index 99% rename from packages/context/src/ingest/dbt-shared/schema-files.ts rename to packages/cli/src/context/ingest/dbt-shared/schema-files.ts index 6c223b83..946d8226 100644 --- a/packages/context/src/ingest/dbt-shared/schema-files.ts +++ b/packages/cli/src/context/ingest/dbt-shared/schema-files.ts @@ -22,6 +22,7 @@ export async function loadDbtSchemaFiles(projectDir: string): Promise { const schemaFiles: string[] = []; diff --git a/packages/context/src/ingest/diff-set.service.test.ts b/packages/cli/src/context/ingest/diff-set.service.test.ts similarity index 100% rename from packages/context/src/ingest/diff-set.service.test.ts rename to packages/cli/src/context/ingest/diff-set.service.test.ts diff --git a/packages/context/src/ingest/diff-set.service.ts b/packages/cli/src/context/ingest/diff-set.service.ts similarity index 100% rename from packages/context/src/ingest/diff-set.service.ts rename to packages/cli/src/context/ingest/diff-set.service.ts diff --git a/packages/context/src/ingest/final-gate-repair.test.ts b/packages/cli/src/context/ingest/final-gate-repair.test.ts similarity index 100% rename from packages/context/src/ingest/final-gate-repair.test.ts rename to packages/cli/src/context/ingest/final-gate-repair.test.ts diff --git a/packages/context/src/ingest/final-gate-repair.ts b/packages/cli/src/context/ingest/final-gate-repair.ts similarity index 97% rename from packages/context/src/ingest/final-gate-repair.ts rename to packages/cli/src/context/ingest/final-gate-repair.ts index 57ff1619..1c373aa6 100644 --- a/packages/context/src/ingest/final-gate-repair.ts +++ b/packages/cli/src/context/ingest/final-gate-repair.ts @@ -1,8 +1,8 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { z } from 'zod'; -import type { AgentRunnerPort, KtxRuntimeToolSet } from '../llm/index.js'; -import type { TouchedSlSource } from '../tools/index.js'; +import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; +import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js'; import type { IngestTraceWriter } from './ingest-trace.js'; import { traceTimed } from './ingest-trace.js'; diff --git a/packages/context/src/ingest/finalization-scope.test.ts b/packages/cli/src/context/ingest/finalization-scope.test.ts similarity index 100% rename from packages/context/src/ingest/finalization-scope.test.ts rename to packages/cli/src/context/ingest/finalization-scope.test.ts diff --git a/packages/context/src/ingest/finalization-scope.ts b/packages/cli/src/context/ingest/finalization-scope.ts similarity index 96% rename from packages/context/src/ingest/finalization-scope.ts rename to packages/cli/src/context/ingest/finalization-scope.ts index 6ecdc83e..b5ace2b9 100644 --- a/packages/context/src/ingest/finalization-scope.ts +++ b/packages/cli/src/context/ingest/finalization-scope.ts @@ -1,5 +1,5 @@ -import type { SemanticLayerSource } from '../sl/index.js'; -import type { TouchedSlSource } from '../tools/index.js'; +import type { SemanticLayerSource } from '../../context/sl/types.js'; +import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js'; import type { IngestReportFinalizationMismatch } from './reports.js'; interface DeriveTouchedSourcesInput { diff --git a/packages/context/src/ingest/git-env.ts b/packages/cli/src/context/ingest/git-env.ts similarity index 100% rename from packages/context/src/ingest/git-env.ts rename to packages/cli/src/context/ingest/git-env.ts diff --git a/packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts b/packages/cli/src/context/ingest/ingest-bundle.runner.isolated-diff.test.ts similarity index 99% rename from packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts rename to packages/cli/src/context/ingest/ingest-bundle.runner.isolated-diff.test.ts index fee60600..0201b881 100644 --- a/packages/context/src/ingest/ingest-bundle.runner.isolated-diff.test.ts +++ b/packages/cli/src/context/ingest/ingest-bundle.runner.isolated-diff.test.ts @@ -2,9 +2,10 @@ import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promis import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it, vi } from 'vitest'; -import { GitService, SessionWorktreeService } from '../core/index.js'; +import { GitService } from '../../context/core/git.service.js'; +import { SessionWorktreeService } from '../../context/core/session-worktree.service.js'; import { LocalGitFileStore } from '../project/local-git-file-store.js'; -import { addTouchedSlSource } from '../tools/index.js'; +import { addTouchedSlSource } from '../../context/tools/touched-sl-sources.js'; import { IngestBundleRunner } from './ingest-bundle.runner.js'; import type { IngestBundleRunnerDeps } from './ports.js'; diff --git a/packages/context/src/ingest/ingest-bundle.runner.test.ts b/packages/cli/src/context/ingest/ingest-bundle.runner.test.ts similarity index 99% rename from packages/context/src/ingest/ingest-bundle.runner.test.ts rename to packages/cli/src/context/ingest/ingest-bundle.runner.test.ts index ae6f4c14..85f45049 100644 --- a/packages/context/src/ingest/ingest-bundle.runner.test.ts +++ b/packages/cli/src/context/ingest/ingest-bundle.runner.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { addTouchedSlSource } from '../tools/index.js'; +import { addTouchedSlSource } from '../../context/tools/touched-sl-sources.js'; import { IngestBundleRunner } from './ingest-bundle.runner.js'; import { createMemoryFlowLiveBuffer } from './memory-flow/live-buffer.js'; import type { MemoryFlowReplayInput } from './memory-flow/types.js'; diff --git a/packages/context/src/ingest/ingest-bundle.runner.ts b/packages/cli/src/context/ingest/ingest-bundle.runner.ts similarity index 99% rename from packages/context/src/ingest/ingest-bundle.runner.ts rename to packages/cli/src/context/ingest/ingest-bundle.runner.ts index 43a2b251..510e88d0 100644 --- a/packages/context/src/ingest/ingest-bundle.runner.ts +++ b/packages/cli/src/context/ingest/ingest-bundle.runner.ts @@ -2,12 +2,17 @@ import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import pLimit from 'p-limit'; import { z } from 'zod'; -import { type KtxLogger, noopLogger } from '../core/index.js'; -import { createRuntimeToolDescriptorFromAiTool, type KtxRuntimeToolSet } from '../llm/index.js'; -import type { CaptureSession, MemoryAction } from '../memory/index.js'; -import type { SemanticLayerService, SemanticLayerSource, SlValidationDeps } from '../sl/index.js'; -import { createTouchedSlSources, type ToolContext, type ToolSession, type TouchedSlSource } from '../tools/index.js'; -import type { KnowledgeWikiService } from '../wiki/index.js'; +import { type KtxLogger, noopLogger } from '../../context/core/config.js'; +import { createRuntimeToolDescriptorFromAiTool } from '../../context/llm/runtime-tools.js'; +import type { KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; +import type { CaptureSession, MemoryAction } from '../../context/memory/types.js'; +import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import type { SemanticLayerSource } from '../../context/sl/types.js'; +import type { SlValidationDeps } from '../../context/sl/tools/sl-warehouse-validation.js'; +import { createTouchedSlSources, type TouchedSlSource } from '../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../context/tools/base-tool.js'; +import type { ToolSession } from '../../context/tools/tool-session.js'; +import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; import { findDanglingWikiRefsForActions } from '../wiki/wiki-ref-validation.js'; import { actionTargetConnectionId } from './action-identity.js'; import { NOTION_DEFAULT_MAX_KNOWLEDGE_CREATES_PER_RUN } from './adapters/notion/types.js'; diff --git a/packages/context/src/ingest/ingest-prompts.test.ts b/packages/cli/src/context/ingest/ingest-prompts.test.ts similarity index 100% rename from packages/context/src/ingest/ingest-prompts.test.ts rename to packages/cli/src/context/ingest/ingest-prompts.test.ts diff --git a/packages/context/src/ingest/ingest-runtime-assets.test.ts b/packages/cli/src/context/ingest/ingest-runtime-assets.test.ts similarity index 96% rename from packages/context/src/ingest/ingest-runtime-assets.test.ts rename to packages/cli/src/context/ingest/ingest-runtime-assets.test.ts index 6b9d83ba..f6a46111 100644 --- a/packages/context/src/ingest/ingest-runtime-assets.test.ts +++ b/packages/cli/src/context/ingest/ingest-runtime-assets.test.ts @@ -2,8 +2,8 @@ import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; -import { PromptService } from '../prompts/index.js'; -import { SkillsRegistryService } from '../skills/index.js'; +import { PromptService } from '../../context/prompts/prompt.service.js'; +import { SkillsRegistryService } from '../../context/skills/skills-registry.service.js'; const promptsDir = fileURLToPath(new URL('../../prompts', import.meta.url)); const skillsDir = fileURLToPath(new URL('../../skills', import.meta.url)); diff --git a/packages/context/src/ingest/ingest-trace.test.ts b/packages/cli/src/context/ingest/ingest-trace.test.ts similarity index 100% rename from packages/context/src/ingest/ingest-trace.test.ts rename to packages/cli/src/context/ingest/ingest-trace.test.ts diff --git a/packages/context/src/ingest/ingest-trace.ts b/packages/cli/src/context/ingest/ingest-trace.ts similarity index 91% rename from packages/context/src/ingest/ingest-trace.ts rename to packages/cli/src/context/ingest/ingest-trace.ts index eed0cfd5..bac114be 100644 --- a/packages/context/src/ingest/ingest-trace.ts +++ b/packages/cli/src/context/ingest/ingest-trace.ts @@ -20,7 +20,7 @@ export interface IngestTraceContext { level?: IngestTraceLevel; } -export interface IngestTraceEvent { +interface IngestTraceEvent { schemaVersion: 1; at: string; level: IngestTraceLevel; @@ -121,22 +121,6 @@ export class FileIngestTraceWriter implements IngestTraceWriter { } } -export class NoopIngestTraceWriter implements IngestTraceWriter { - readonly tracePath = ''; - readonly context: IngestTraceContext = { - tracePath: '', - jobId: '', - connectionId: '', - sourceKey: '', - level: 'error', - }; - - withContext(): IngestTraceWriter { - return this; - } - - async event(): Promise {} -} export async function traceTimed( trace: IngestTraceWriter, diff --git a/packages/context/src/ingest/isolated-diff/git-patch.test.ts b/packages/cli/src/context/ingest/isolated-diff/git-patch.test.ts similarity index 100% rename from packages/context/src/ingest/isolated-diff/git-patch.test.ts rename to packages/cli/src/context/ingest/isolated-diff/git-patch.test.ts diff --git a/packages/context/src/ingest/isolated-diff/git-patch.ts b/packages/cli/src/context/ingest/isolated-diff/git-patch.ts similarity index 99% rename from packages/context/src/ingest/isolated-diff/git-patch.ts rename to packages/cli/src/context/ingest/isolated-diff/git-patch.ts index ee7f0020..e0250b11 100644 --- a/packages/context/src/ingest/isolated-diff/git-patch.ts +++ b/packages/cli/src/context/ingest/isolated-diff/git-patch.ts @@ -1,5 +1,6 @@ import { assertSemanticLayerTargetPathsAllowed } from '../semantic-layer-target-policy.js'; +/** @internal */ export const textArtifactRoots = ['wiki/', 'semantic-layer/'] as const; export interface PatchTouchedPath { diff --git a/packages/context/src/ingest/isolated-diff/patch-integrator.test.ts b/packages/cli/src/context/ingest/isolated-diff/patch-integrator.test.ts similarity index 99% rename from packages/context/src/ingest/isolated-diff/patch-integrator.test.ts rename to packages/cli/src/context/ingest/isolated-diff/patch-integrator.test.ts index d55cfc5b..e547e22e 100644 --- a/packages/context/src/ingest/isolated-diff/patch-integrator.test.ts +++ b/packages/cli/src/context/ingest/isolated-diff/patch-integrator.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it, vi } from 'vitest'; -import { GitService } from '../../core/index.js'; +import { GitService } from '../../../context/core/git.service.js'; import { FileIngestTraceWriter } from '../ingest-trace.js'; import { integrateWorkUnitPatch } from './patch-integrator.js'; diff --git a/packages/context/src/ingest/isolated-diff/patch-integrator.ts b/packages/cli/src/context/ingest/isolated-diff/patch-integrator.ts similarity index 98% rename from packages/context/src/ingest/isolated-diff/patch-integrator.ts rename to packages/cli/src/context/ingest/isolated-diff/patch-integrator.ts index a4542576..869c019e 100644 --- a/packages/context/src/ingest/isolated-diff/patch-integrator.ts +++ b/packages/cli/src/context/ingest/isolated-diff/patch-integrator.ts @@ -1,12 +1,12 @@ import { readFile } from 'node:fs/promises'; -import type { GitService } from '../../core/index.js'; +import type { GitService } from '../../../context/core/git.service.js'; import type { FinalGateRepairResult } from '../final-gate-repair.js'; import type { IngestTraceWriter } from '../ingest-trace.js'; import { traceTimed } from '../ingest-trace.js'; import { assertPatchAllowedForWorkUnit, parsePatchTouchedPaths } from './git-patch.js'; import type { TextualConflictResolutionResult } from './textual-conflict-resolver.js'; -export type PatchIntegrationTextualResolution = +type PatchIntegrationTextualResolution = | { status: 'repaired'; attempts: number; changedPaths: string[] } | { status: 'failed'; attempts: number; reason: string }; diff --git a/packages/context/src/ingest/isolated-diff/textual-conflict-resolver.test.ts b/packages/cli/src/context/ingest/isolated-diff/textual-conflict-resolver.test.ts similarity index 100% rename from packages/context/src/ingest/isolated-diff/textual-conflict-resolver.test.ts rename to packages/cli/src/context/ingest/isolated-diff/textual-conflict-resolver.test.ts diff --git a/packages/context/src/ingest/isolated-diff/textual-conflict-resolver.ts b/packages/cli/src/context/ingest/isolated-diff/textual-conflict-resolver.ts similarity index 98% rename from packages/context/src/ingest/isolated-diff/textual-conflict-resolver.ts rename to packages/cli/src/context/ingest/isolated-diff/textual-conflict-resolver.ts index c5128291..5ae551d1 100644 --- a/packages/context/src/ingest/isolated-diff/textual-conflict-resolver.ts +++ b/packages/cli/src/context/ingest/isolated-diff/textual-conflict-resolver.ts @@ -1,7 +1,7 @@ import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { z } from 'zod'; -import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../llm/index.js'; +import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; import type { IngestTraceWriter } from '../ingest-trace.js'; import { traceTimed } from '../ingest-trace.js'; diff --git a/packages/context/src/ingest/isolated-diff/work-unit-executor.test.ts b/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.test.ts similarity index 98% rename from packages/context/src/ingest/isolated-diff/work-unit-executor.test.ts rename to packages/cli/src/context/ingest/isolated-diff/work-unit-executor.test.ts index 6f8a7599..5975dee8 100644 --- a/packages/context/src/ingest/isolated-diff/work-unit-executor.test.ts +++ b/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it, vi } from 'vitest'; -import { GitService } from '../../core/index.js'; +import { GitService } from '../../../context/core/git.service.js'; import { FileIngestTraceWriter } from '../ingest-trace.js'; import { runIsolatedWorkUnit } from './work-unit-executor.js'; diff --git a/packages/context/src/ingest/isolated-diff/work-unit-executor.ts b/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.ts similarity index 97% rename from packages/context/src/ingest/isolated-diff/work-unit-executor.ts rename to packages/cli/src/context/ingest/isolated-diff/work-unit-executor.ts index ac013d5a..81e6edfa 100644 --- a/packages/context/src/ingest/isolated-diff/work-unit-executor.ts +++ b/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.ts @@ -1,6 +1,6 @@ import { mkdir, readFile } from 'node:fs/promises'; import { join } from 'node:path'; -import type { SessionOutcome } from '../../core/index.js'; +import type { SessionOutcome } from '../../../context/core/session-worktree.service.js'; import type { IngestSessionWorktree, IngestSessionWorktreePort } from '../ports.js'; import type { WorkUnit } from '../types.js'; import type { IngestTraceWriter } from '../ingest-trace.js'; diff --git a/packages/context/src/ingest/local-adapters.test.ts b/packages/cli/src/context/ingest/local-adapters.test.ts similarity index 99% rename from packages/context/src/ingest/local-adapters.test.ts rename to packages/cli/src/context/ingest/local-adapters.test.ts index a4e9eea6..5f61c739 100644 --- a/packages/context/src/ingest/local-adapters.test.ts +++ b/packages/cli/src/context/ingest/local-adapters.test.ts @@ -2,8 +2,8 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js'; -import type { SqlAnalysisPort } from '../sql-analysis/index.js'; +import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js'; +import type { SqlAnalysisPort } from '../../context/sql-analysis/ports.js'; import type { HistoricSqlReader } from './adapters/historic-sql/types.js'; import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js'; import { LocalNotionRuntimeStore } from './adapters/notion/local-state-store.js'; diff --git a/packages/context/src/ingest/local-adapters.ts b/packages/cli/src/context/ingest/local-adapters.ts similarity index 96% rename from packages/context/src/ingest/local-adapters.ts rename to packages/cli/src/context/ingest/local-adapters.ts index 2b13327a..ea7556e7 100644 --- a/packages/context/src/ingest/local-adapters.ts +++ b/packages/cli/src/context/ingest/local-adapters.ts @@ -1,8 +1,10 @@ import { join } from 'node:path'; -import { localConnectionToWarehouseDescriptor, notionConnectionToPullConfig, parseNotionConnectionConfig } from '../connections/index.js'; +import { localConnectionToWarehouseDescriptor } from '../../context/connections/local-warehouse-descriptor.js'; +import { notionConnectionToPullConfig, parseNotionConnectionConfig } from '../../context/connections/notion-config.js'; import { resolveKtxConfigReference } from '../core/config-reference.js'; -import { ktxLocalStateDbPath, type KtxLocalProject } from '../project/index.js'; -import type { SqlAnalysisPort } from '../sql-analysis/index.js'; +import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import type { SqlAnalysisPort } from '../../context/sql-analysis/ports.js'; import { DbtSourceAdapter } from './adapters/dbt/dbt.adapter.js'; import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js'; import { HistoricSqlSourceAdapter } from './adapters/historic-sql/historic-sql.adapter.js'; diff --git a/packages/context/src/ingest/local-bundle-ingest.test.ts b/packages/cli/src/context/ingest/local-bundle-ingest.test.ts similarity index 99% rename from packages/context/src/ingest/local-bundle-ingest.test.ts rename to packages/cli/src/context/ingest/local-bundle-ingest.test.ts index 5a336f35..4b4b834c 100644 --- a/packages/context/src/ingest/local-bundle-ingest.test.ts +++ b/packages/cli/src/context/ingest/local-bundle-ingest.test.ts @@ -3,8 +3,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import Database from 'better-sqlite3'; import YAML from 'yaml'; -import type { AgentRunnerPort, RunLoopParams } from '../llm/index.js'; -import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js'; +import type { AgentRunnerPort, RunLoopParams } from '../../context/llm/runtime-port.js'; +import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js'; import { makeLocalGitRepo } from '../test/make-local-git-repo.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js'; diff --git a/packages/context/src/ingest/local-bundle-runtime.test.ts b/packages/cli/src/context/ingest/local-bundle-runtime.test.ts similarity index 98% rename from packages/context/src/ingest/local-bundle-runtime.test.ts rename to packages/cli/src/context/ingest/local-bundle-runtime.test.ts index df906adf..3c87c351 100644 --- a/packages/context/src/ingest/local-bundle-runtime.test.ts +++ b/packages/cli/src/context/ingest/local-bundle-runtime.test.ts @@ -1,8 +1,8 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import type { AgentRunnerPort } from '../llm/index.js'; -import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js'; +import type { AgentRunnerPort } from '../../context/llm/runtime-port.js'; +import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js'; import { createLocalBundleIngestRuntime } from './local-bundle-runtime.js'; diff --git a/packages/context/src/ingest/local-bundle-runtime.ts b/packages/cli/src/context/ingest/local-bundle-runtime.ts similarity index 83% rename from packages/context/src/ingest/local-bundle-runtime.ts rename to packages/cli/src/context/ingest/local-bundle-runtime.ts index 2749b4c7..8b87d7be 100644 --- a/packages/context/src/ingest/local-bundle-runtime.ts +++ b/packages/cli/src/context/ingest/local-bundle-runtime.ts @@ -2,82 +2,64 @@ import { mkdirSync } from 'node:fs'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import YAML from 'yaml'; -import { localConnectionInfoFromConfig, type KtxSqlQueryExecutorPort } from '../connections/index.js'; -import type { KtxEmbeddingPort, KtxLogger } from '../core/index.js'; -import { noopLogger, SessionWorktreeService } from '../core/index.js'; -import type { KtxSemanticLayerComputePort } from '../daemon/index.js'; -import { - createRuntimeToolDescriptorFromAiTool, - createLocalKtxLlmRuntimeFromConfig, - KtxIngestEmbeddingPortAdapter, - RuntimeAgentRunner, - type AgentRunnerPort, - type KtxLlmRuntimePort, - type KtxRuntimeToolSet, -} from '../llm/index.js'; -import type { KtxEmbeddingProvider } from '@ktx/llm'; -import type { KtxLocalProject } from '../project/index.js'; -import { ktxLocalStateDbPath } from '../project/index.js'; -import { PromptService } from '../prompts/index.js'; -import { SkillsRegistryService } from '../skills/index.js'; -import { - type KtxConnectionInfo, - type KtxQueryResult, - SemanticLayerService, - type SlConnectionCatalogPort, - SlDiscoverTool, - SlEditSourceTool, - type SlPythonPort, - SlReadSourceTool, - SlRollbackTool, - SlSearchService, - type SlSourcesIndexPort, - SlValidateTool, - type SlValidationDeps, - type SlValidatorPort, - SlWriteSourceTool, - SqliteSlSourcesIndex, - sourceDefinitionSchema, - sourceOverlaySchema, -} from '../sl/index.js'; -import { - BaseTool, - ContextCandidateMarkTool, - ContextCandidateWriteTool, - ContextEvidenceNeighborsTool, - ContextEvidenceReadTool, - ContextEvidenceSearchTool, - type GitAuthorResolverPort, - type ToolContext, - type ToolSession, -} from '../tools/index.js'; -import { - buildKnowledgeSearchText, - type KnowledgeEventPort, - type KnowledgeIndexPort, - type KnowledgeIndexPageListing, - KnowledgeWikiService, - searchLocalKnowledgePages, - SqliteKnowledgeIndex, - type SqliteKnowledgeIndexPage, - WikiListTagsTool, - WikiReadTool, - WikiRemoveTool, - WikiSearchTool, - WikiWriteTool, -} from '../wiki/index.js'; -import { - CandidateDedupService, - ContextCandidateCarryforwardService, - CuratorPaginationService, -} from './context-candidates/index.js'; +import { localConnectionInfoFromConfig } from '../../context/connections/local-warehouse-descriptor.js'; +import type { KtxSqlQueryExecutorPort } from '../../context/connections/query-executor.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxLogger } from '../../context/core/config.js'; +import { noopLogger } from '../../context/core/config.js'; +import { SessionWorktreeService } from '../../context/core/session-worktree.service.js'; +import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js'; +import { createRuntimeToolDescriptorFromAiTool } from '../../context/llm/runtime-tools.js'; +import { createLocalKtxLlmRuntimeFromConfig } from '../../context/llm/local-config.js'; +import { KtxIngestEmbeddingPortAdapter } from '../../context/llm/embedding-port.js'; +import { RuntimeAgentRunner, type AgentRunnerPort, type KtxLlmRuntimePort, type KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; +import type { KtxEmbeddingProvider } from '../../llm/types.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js'; +import { PromptService } from '../../context/prompts/prompt.service.js'; +import { SkillsRegistryService } from '../../context/skills/skills-registry.service.js'; +import type { KtxConnectionInfo, KtxQueryResult, SlConnectionCatalogPort, SlPythonPort, SlSourcesIndexPort } from '../../context/sl/ports.js'; +import { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import { SlDiscoverTool } from '../../context/sl/tools/sl-discover.tool.js'; +import { SlEditSourceTool } from '../../context/sl/tools/sl-edit-source.tool.js'; +import { SlReadSourceTool } from '../../context/sl/tools/sl-read-source.tool.js'; +import { SlRollbackTool } from '../../context/sl/tools/sl-rollback.tool.js'; +import { SlSearchService } from '../../context/sl/sl-search.service.js'; +import { SlValidateTool } from '../../context/sl/tools/sl-validate.tool.js'; +import type { SlValidationDeps } from '../../context/sl/tools/sl-warehouse-validation.js'; +import type { SlValidatorPort } from '../../context/sl/sl-validator.port.js'; +import { SlWriteSourceTool } from '../../context/sl/tools/sl-write-source.tool.js'; +import { SqliteSlSourcesIndex } from '../../context/sl/sqlite-sl-sources-index.js'; +import { sourceDefinitionSchema, sourceOverlaySchema } from '../../context/sl/schemas.js'; +import { BaseTool, type ToolContext } from '../../context/tools/base-tool.js'; +import { ContextCandidateMarkTool } from '../../context/tools/context-candidate-mark.tool.js'; +import { ContextCandidateWriteTool } from '../../context/tools/context-candidate-write.tool.js'; +import { ContextEvidenceNeighborsTool } from '../../context/tools/context-evidence-neighbors.tool.js'; +import { ContextEvidenceReadTool } from '../../context/tools/context-evidence-read.tool.js'; +import { ContextEvidenceSearchTool } from '../../context/tools/context-evidence-search.tool.js'; +import type { GitAuthorResolverPort } from '../../context/tools/authors.js'; +import type { ToolSession } from '../../context/tools/tool-session.js'; +import { buildKnowledgeSearchText } from '../../context/wiki/knowledge-search-text.js'; +import type { KnowledgeEventPort, KnowledgeIndexPort, KnowledgeIndexPageListing } from '../../context/wiki/ports.js'; +import { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; +import { searchLocalKnowledgePages } from '../../context/wiki/local-knowledge.js'; +import { SqliteKnowledgeIndex, type SqliteKnowledgeIndexPage } from '../../context/wiki/sqlite-knowledge-index.js'; +import { WikiListTagsTool } from '../../context/wiki/tools/wiki-list-tags.tool.js'; +import { WikiReadTool } from '../../context/wiki/tools/wiki-read.tool.js'; +import { WikiRemoveTool } from '../../context/wiki/tools/wiki-remove.tool.js'; +import { WikiSearchTool } from '../../context/wiki/tools/wiki-search.tool.js'; +import { WikiWriteTool } from '../../context/wiki/tools/wiki-write.tool.js'; +import { CandidateDedupService } from '../../context/ingest/context-candidates/candidate-dedup.service.js'; +import { ContextCandidateCarryforwardService } from '../../context/ingest/context-candidates/context-candidate-carryforward.service.js'; +import { CuratorPaginationService } from '../../context/ingest/context-candidates/curator-pagination.service.js'; import { createEmitHistoricSqlEvidenceTool } from './adapters/historic-sql/evidence-tool.js'; -import { ContextEvidenceIndexService, SqliteContextEvidenceStore } from './context-evidence/index.js'; +import { ContextEvidenceIndexService } from '../../context/ingest/context-evidence/context-evidence-index.service.js'; +import { SqliteContextEvidenceStore } from '../../context/ingest/context-evidence/sqlite-context-evidence-store.js'; import { DiffSetService } from './diff-set.service.js'; import { ingestTracePathForJob, type IngestTraceLevel } from './ingest-trace.js'; import { IngestBundleRunner } from './ingest-bundle.runner.js'; -import { PageTriageService } from './page-triage/index.js'; -import { createWarehouseVerificationTools } from './tools/warehouse-verification/index.js'; +import { PageTriageService } from '../../context/ingest/page-triage/page-triage.service.js'; +import { createWarehouseVerificationTools } from '../../context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.js'; import type { IngestBundleRunnerDeps, IngestCommitMessagePort, diff --git a/packages/context/src/ingest/local-embedding-provider.integration.test.ts b/packages/cli/src/context/ingest/local-embedding-provider.integration.test.ts similarity index 100% rename from packages/context/src/ingest/local-embedding-provider.integration.test.ts rename to packages/cli/src/context/ingest/local-embedding-provider.integration.test.ts diff --git a/packages/context/src/ingest/local-ingest.ts b/packages/cli/src/context/ingest/local-ingest.ts similarity index 92% rename from packages/context/src/ingest/local-ingest.ts rename to packages/cli/src/context/ingest/local-ingest.ts index 794ccfc4..2832d9ff 100644 --- a/packages/context/src/ingest/local-ingest.ts +++ b/packages/cli/src/context/ingest/local-ingest.ts @@ -1,12 +1,12 @@ import { randomUUID } from 'node:crypto'; import { cp, mkdir, rm } from 'node:fs/promises'; import { isAbsolute, resolve } from 'node:path'; -import type { KtxSqlQueryExecutorPort } from '../connections/index.js'; -import type { KtxLogger } from '../core/index.js'; -import type { KtxSemanticLayerComputePort } from '../daemon/index.js'; -import type { AgentRunnerPort, KtxLlmRuntimePort } from '../llm/index.js'; -import type { KtxLocalProject } from '../project/index.js'; -import { ktxLocalStateDbPath } from '../project/index.js'; +import type { KtxSqlQueryExecutorPort } from '../../context/connections/query-executor.js'; +import type { KtxLogger } from '../../context/core/config.js'; +import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js'; +import type { AgentRunnerPort, KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js'; import { planMetabaseFanoutChildren } from './adapters/metabase/fanout-planner.js'; import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './adapters/metabase/local-source-state-store.js'; import { localPullConfigForAdapter, type DefaultLocalIngestAdaptersOptions } from './local-adapters.js'; @@ -34,24 +34,7 @@ export interface RunLocalIngestOptions { semanticLayerCompute?: KtxSemanticLayerComputePort; queryExecutor?: KtxSqlQueryExecutorPort; logger?: KtxLogger; - embeddingProvider?: import('@ktx/llm').KtxEmbeddingProvider | null; -} - -export interface LocalIngestMcpOptions - extends Pick< - RunLocalIngestOptions, - | 'agentRunner' - | 'llmRuntime' - | 'memoryModel' - | 'semanticLayerCompute' - | 'queryExecutor' - | 'logger' - | 'pullConfigOptions' - > { - adapters?: SourceAdapter[]; - jobIdFactory?: () => string; - runLocalIngest?: (options: RunLocalIngestOptions) => Promise; - runLocalMetabaseIngest?: (options: RunLocalMetabaseIngestOptions) => Promise; + embeddingProvider?: import('../../llm/types.js').KtxEmbeddingProvider | null; } export interface LocalIngestResult { @@ -59,7 +42,7 @@ export interface LocalIngestResult { report: IngestReportSnapshot; } -export interface LocalMetabaseFanoutChild { +interface LocalMetabaseFanoutChild { jobId: string; metabaseConnectionId: string; metabaseDatabaseId: number; @@ -75,7 +58,7 @@ export interface LocalMetabaseFanoutResult { totals?: { workUnits: number; failedWorkUnits: number }; } -export interface LocalMetabaseFanoutProgressChild { +interface LocalMetabaseFanoutProgressChild { metabaseDatabaseId: number; targetConnectionId: string; } @@ -173,7 +156,7 @@ async function runScheduledPullJob(options: { semanticLayerCompute?: KtxSemanticLayerComputePort; queryExecutor?: KtxSqlQueryExecutorPort; logger?: KtxLogger; - embeddingProvider?: import('@ktx/llm').KtxEmbeddingProvider | null; + embeddingProvider?: import('../../llm/types.js').KtxEmbeddingProvider | null; }): Promise { const runtime = createLocalBundleIngestRuntime(options); const jobId = options.jobId ?? runtime.nextJobId(); diff --git a/packages/context/src/ingest/local-mapping-reconcile.test.ts b/packages/cli/src/context/ingest/local-mapping-reconcile.test.ts similarity index 94% rename from packages/context/src/ingest/local-mapping-reconcile.test.ts rename to packages/cli/src/context/ingest/local-mapping-reconcile.test.ts index 903ef43f..3eed9d53 100644 --- a/packages/context/src/ingest/local-mapping-reconcile.test.ts +++ b/packages/cli/src/context/ingest/local-mapping-reconcile.test.ts @@ -2,7 +2,8 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; -import { ktxLocalStateDbPath, type KtxLocalProject } from '../project/index.js'; +import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js'; import { seedLocalMappingStateFromKtxYaml } from './local-mapping-reconcile.js'; diff --git a/packages/context/src/ingest/local-mapping-reconcile.ts b/packages/cli/src/context/ingest/local-mapping-reconcile.ts similarity index 77% rename from packages/context/src/ingest/local-mapping-reconcile.ts rename to packages/cli/src/context/ingest/local-mapping-reconcile.ts index 1a58af95..3b9d5d8f 100644 --- a/packages/context/src/ingest/local-mapping-reconcile.ts +++ b/packages/cli/src/context/ingest/local-mapping-reconcile.ts @@ -1,9 +1,6 @@ -import { - ktxLocalStateDbPath, - parseConnectionMappingBootstrap, - type KtxLocalProject, - type LookerMappingBootstrap, -} from '../project/index.js'; +import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js'; +import { parseConnectionMappingBootstrap, type LookerMappingBootstrap } from '../../context/project/mappings-yaml-schema.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js'; function lookerMappings(bootstrap: LookerMappingBootstrap) { diff --git a/packages/context/src/ingest/local-metabase-ingest.test.ts b/packages/cli/src/context/ingest/local-metabase-ingest.test.ts similarity index 98% rename from packages/context/src/ingest/local-metabase-ingest.test.ts rename to packages/cli/src/context/ingest/local-metabase-ingest.test.ts index 04f18923..ff91d827 100644 --- a/packages/context/src/ingest/local-metabase-ingest.test.ts +++ b/packages/cli/src/context/ingest/local-metabase-ingest.test.ts @@ -1,9 +1,9 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import type { AgentRunnerPort, RunLoopParams } from '../llm/index.js'; +import type { AgentRunnerPort, RunLoopParams } from '../../context/llm/runtime-port.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { LocalMetabaseDiscoveryCache } from './adapters/metabase/local-source-state-store.js'; import { getLocalIngestStatus, runLocalMetabaseIngest } from './local-ingest.js'; import type { ChunkResult, FetchContext, SourceAdapter } from './types.js'; diff --git a/packages/context/src/ingest/local-stage-ingest.test.ts b/packages/cli/src/context/ingest/local-stage-ingest.test.ts similarity index 99% rename from packages/context/src/ingest/local-stage-ingest.test.ts rename to packages/cli/src/context/ingest/local-stage-ingest.test.ts index 66fea320..7a2c5a6a 100644 --- a/packages/context/src/ingest/local-stage-ingest.test.ts +++ b/packages/cli/src/context/ingest/local-stage-ingest.test.ts @@ -2,7 +2,7 @@ import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promise import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js'; import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js'; import { createDefaultLocalIngestAdapters } from './local-adapters.js'; import { diff --git a/packages/context/src/ingest/local-stage-ingest.ts b/packages/cli/src/context/ingest/local-stage-ingest.ts similarity index 98% rename from packages/context/src/ingest/local-stage-ingest.ts rename to packages/cli/src/context/ingest/local-stage-ingest.ts index 0365b071..5897281f 100644 --- a/packages/context/src/ingest/local-stage-ingest.ts +++ b/packages/cli/src/context/ingest/local-stage-ingest.ts @@ -1,7 +1,7 @@ import { createHash } from 'node:crypto'; import { cp, mkdir, readdir, readFile, rm } from 'node:fs/promises'; import { isAbsolute, join, relative, resolve, sep } from 'node:path'; -import type { KtxLocalProject } from '../project/index.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import { ktxLocalStateDbPath } from '../project/local-state-db.js'; import { computeDiffSetFromHashes } from './diff-set.service.js'; import { localPullConfigForAdapter } from './local-adapters.js'; @@ -11,9 +11,9 @@ import { buildSyncId } from './raw-sources-paths.js'; import { SqliteLocalIngestStore } from './sqlite-local-ingest-store.js'; import type { IngestTrigger, SourceAdapter, WorkUnit } from './types.js'; -export type LocalIngestStatus = 'running' | 'done' | 'error'; +type LocalIngestStatus = 'running' | 'done' | 'error'; -export interface LocalIngestDiffPaths { +interface LocalIngestDiffPaths { added: string[]; modified: string[]; deleted: string[]; diff --git a/packages/context/src/ingest/memory-flow/acceptance-fixtures.ts b/packages/cli/src/context/ingest/memory-flow/acceptance-fixtures.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/acceptance-fixtures.ts rename to packages/cli/src/context/ingest/memory-flow/acceptance-fixtures.ts diff --git a/packages/context/src/ingest/memory-flow/acceptance.test.ts b/packages/cli/src/context/ingest/memory-flow/acceptance.test.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/acceptance.test.ts rename to packages/cli/src/context/ingest/memory-flow/acceptance.test.ts diff --git a/packages/context/src/ingest/memory-flow/events.test.ts b/packages/cli/src/context/ingest/memory-flow/events.test.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/events.test.ts rename to packages/cli/src/context/ingest/memory-flow/events.test.ts diff --git a/packages/context/src/ingest/memory-flow/events.ts b/packages/cli/src/context/ingest/memory-flow/events.ts similarity index 98% rename from packages/context/src/ingest/memory-flow/events.ts rename to packages/cli/src/context/ingest/memory-flow/events.ts index 7692e710..020ce5ae 100644 --- a/packages/context/src/ingest/memory-flow/events.ts +++ b/packages/cli/src/context/ingest/memory-flow/events.ts @@ -1,4 +1,4 @@ -import type { MemoryAction } from '../../memory/index.js'; +import type { MemoryAction } from '../../../context/memory/types.js'; import type { LocalIngestRunRecord } from '../local-stage-ingest.js'; import type { IngestReportSnapshot } from '../reports.js'; import type { @@ -132,6 +132,7 @@ function detailSectionsFromReport(report: IngestReportSnapshot): MemoryFlowDetai }; } +/** @internal */ export function localIngestRunToMemoryFlowReplay(record: LocalIngestRunRecord): MemoryFlowReplayInput { const events: MemoryFlowEvent[] = [ { type: 'source_acquired', adapter: record.adapter, trigger: 'manual_resync', fileCount: record.rawFileCount }, diff --git a/packages/context/src/ingest/memory-flow/interaction.test.ts b/packages/cli/src/context/ingest/memory-flow/interaction.test.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/interaction.test.ts rename to packages/cli/src/context/ingest/memory-flow/interaction.test.ts diff --git a/packages/context/src/ingest/memory-flow/interaction.ts b/packages/cli/src/context/ingest/memory-flow/interaction.ts similarity index 99% rename from packages/context/src/ingest/memory-flow/interaction.ts rename to packages/cli/src/context/ingest/memory-flow/interaction.ts index a7703c36..6d1591af 100644 --- a/packages/context/src/ingest/memory-flow/interaction.ts +++ b/packages/cli/src/context/ingest/memory-flow/interaction.ts @@ -233,6 +233,7 @@ export function createInitialMemoryFlowInteractionState(view: MemoryFlowViewMode }; } +/** @internal */ export function selectMemoryFlowColumn( view: MemoryFlowViewModel, state: MemoryFlowInteractionState, @@ -253,6 +254,7 @@ export function selectMemoryFlowColumn( return { ...nextState, selectedChipIndex: clampChipIndex(column, nextState, view) }; } +/** @internal */ export function selectMemoryFlowChip( view: MemoryFlowViewModel, state: MemoryFlowInteractionState, diff --git a/packages/context/src/ingest/memory-flow/interactive-render.test.ts b/packages/cli/src/context/ingest/memory-flow/interactive-render.test.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/interactive-render.test.ts rename to packages/cli/src/context/ingest/memory-flow/interactive-render.test.ts diff --git a/packages/context/src/ingest/memory-flow/interactive-render.ts b/packages/cli/src/context/ingest/memory-flow/interactive-render.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/interactive-render.ts rename to packages/cli/src/context/ingest/memory-flow/interactive-render.ts diff --git a/packages/context/src/ingest/memory-flow/known-errors.ts b/packages/cli/src/context/ingest/memory-flow/known-errors.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/known-errors.ts rename to packages/cli/src/context/ingest/memory-flow/known-errors.ts diff --git a/packages/context/src/ingest/memory-flow/live-buffer.test.ts b/packages/cli/src/context/ingest/memory-flow/live-buffer.test.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/live-buffer.test.ts rename to packages/cli/src/context/ingest/memory-flow/live-buffer.test.ts diff --git a/packages/context/src/ingest/memory-flow/live-buffer.ts b/packages/cli/src/context/ingest/memory-flow/live-buffer.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/live-buffer.ts rename to packages/cli/src/context/ingest/memory-flow/live-buffer.ts diff --git a/packages/context/src/ingest/memory-flow/render.test.ts b/packages/cli/src/context/ingest/memory-flow/render.test.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/render.test.ts rename to packages/cli/src/context/ingest/memory-flow/render.test.ts diff --git a/packages/context/src/ingest/memory-flow/render.ts b/packages/cli/src/context/ingest/memory-flow/render.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/render.ts rename to packages/cli/src/context/ingest/memory-flow/render.ts diff --git a/packages/context/src/ingest/memory-flow/schema.test.ts b/packages/cli/src/context/ingest/memory-flow/schema.test.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/schema.test.ts rename to packages/cli/src/context/ingest/memory-flow/schema.test.ts diff --git a/packages/context/src/ingest/memory-flow/schema.ts b/packages/cli/src/context/ingest/memory-flow/schema.ts similarity index 93% rename from packages/context/src/ingest/memory-flow/schema.ts rename to packages/cli/src/context/ingest/memory-flow/schema.ts index 09cba418..0268a53f 100644 --- a/packages/context/src/ingest/memory-flow/schema.ts +++ b/packages/cli/src/context/ingest/memory-flow/schema.ts @@ -1,7 +1,7 @@ import * as z from 'zod'; import type { MemoryFlowReplayInput } from './types.js'; -export const memoryFlowRunStatusSchema = z.enum(['running', 'done', 'error']); +const memoryFlowRunStatusSchema = z.enum(['running', 'done', 'error']); const memoryFlowEventTimestampShape = { emittedAt: z.string().datetime().optional(), @@ -22,7 +22,7 @@ const memoryFlowReplayMetadataSchema = z.object({ fallbackReason: z.string().min(1).nullable(), }); -export const memoryFlowEventSchema = z.discriminatedUnion('type', [ +const memoryFlowEventSchema = z.discriminatedUnion('type', [ eventSchema({ type: z.literal('source_acquired'), adapter: z.string().min(1), @@ -114,14 +114,14 @@ export const memoryFlowEventSchema = z.discriminatedUnion('type', [ }), ]); -export const memoryFlowPlannedWorkUnitSchema = z.object({ +const memoryFlowPlannedWorkUnitSchema = z.object({ unitKey: z.string().min(1), rawFiles: z.array(z.string()), peerFileCount: z.number().int().min(0), dependencyCount: z.number().int().min(0), }); -export const memoryFlowActionDetailSchema = z.object({ +const memoryFlowActionDetailSchema = z.object({ unitKey: z.string().min(1), target: z.enum(['wiki', 'sl']), action: z.enum(['created', 'updated', 'removed']), @@ -146,7 +146,7 @@ const memoryFlowTranscriptDetailSchema = z.object({ toolNames: z.array(z.string()), }); -export const memoryFlowDetailSectionsSchema = z.object({ +const memoryFlowDetailSectionsSchema = z.object({ actions: z.array(memoryFlowActionDetailSchema), provenance: z.array(memoryFlowProvenanceDetailSchema), transcripts: z.array(memoryFlowTranscriptDetailSchema), @@ -168,6 +168,7 @@ export const memoryFlowReplayInputSchema: z.ZodType = z.o details: memoryFlowDetailSectionsSchema, }); +/** @internal */ export const memoryFlowStreamEventSchema = z.discriminatedUnion('type', [ z.object({ type: z.literal('snapshot'), snapshot: memoryFlowReplayInputSchema }), z.object({ @@ -177,8 +178,8 @@ export const memoryFlowStreamEventSchema = z.discriminatedUnion('type', [ }), ]); -export type MemoryFlowStreamEvent = z.infer; +/** @internal */ export function parseMemoryFlowReplayInput(value: unknown): MemoryFlowReplayInput { const result = memoryFlowReplayInputSchema.safeParse(value); if (!result.success) { diff --git a/packages/context/src/ingest/memory-flow/summary.test.ts b/packages/cli/src/context/ingest/memory-flow/summary.test.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/summary.test.ts rename to packages/cli/src/context/ingest/memory-flow/summary.test.ts diff --git a/packages/context/src/ingest/memory-flow/summary.ts b/packages/cli/src/context/ingest/memory-flow/summary.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/summary.ts rename to packages/cli/src/context/ingest/memory-flow/summary.ts diff --git a/packages/context/src/ingest/memory-flow/types.ts b/packages/cli/src/context/ingest/memory-flow/types.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/types.ts rename to packages/cli/src/context/ingest/memory-flow/types.ts diff --git a/packages/context/src/ingest/memory-flow/view-model.test.ts b/packages/cli/src/context/ingest/memory-flow/view-model.test.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/view-model.test.ts rename to packages/cli/src/context/ingest/memory-flow/view-model.test.ts diff --git a/packages/context/src/ingest/memory-flow/view-model.ts b/packages/cli/src/context/ingest/memory-flow/view-model.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/view-model.ts rename to packages/cli/src/context/ingest/memory-flow/view-model.ts diff --git a/packages/context/src/ingest/memory-flow/visuals.test.ts b/packages/cli/src/context/ingest/memory-flow/visuals.test.ts similarity index 100% rename from packages/context/src/ingest/memory-flow/visuals.test.ts rename to packages/cli/src/context/ingest/memory-flow/visuals.test.ts diff --git a/packages/context/src/ingest/memory-flow/visuals.ts b/packages/cli/src/context/ingest/memory-flow/visuals.ts similarity index 95% rename from packages/context/src/ingest/memory-flow/visuals.ts rename to packages/cli/src/context/ingest/memory-flow/visuals.ts index 84eb6113..e3221383 100644 --- a/packages/context/src/ingest/memory-flow/visuals.ts +++ b/packages/cli/src/context/ingest/memory-flow/visuals.ts @@ -5,12 +5,13 @@ import type { MemoryFlowViewModel, } from './types.js'; +/** @internal */ export interface MemoryFlowStatusBadge { label: '..' | '>>' | 'OK' | '!!' | 'XX'; text: 'waiting' | 'active' | 'complete' | 'warning' | 'failed'; } -export interface MemoryFlowVisualColumn { +interface MemoryFlowVisualColumn { id: MemoryFlowColumnId; title: string; status: MemoryFlowDisplayStatus; @@ -18,12 +19,14 @@ export interface MemoryFlowVisualColumn { pulse: boolean; } +/** @internal */ export interface MemoryFlowVisualModel { columns: MemoryFlowVisualColumn[]; connectorLine: string; pulseColumnId: MemoryFlowColumnId; } +/** @internal */ export function memoryFlowStatusBadge(status: MemoryFlowDisplayStatus): MemoryFlowStatusBadge { if (status === 'active') return { label: '>>', text: 'active' }; if (status === 'complete') return { label: 'OK', text: 'complete' }; @@ -56,6 +59,7 @@ function renderColumn(column: MemoryFlowVisualColumn): string { return `${column.badge.label} ${column.title}`; } +/** @internal */ export function buildMemoryFlowVisualModel(view: MemoryFlowViewModel): MemoryFlowVisualModel { const pulseColumn = selectPulseColumn(view.columns); const columns = view.columns.map((column) => ({ diff --git a/packages/context/src/ingest/page-triage/page-triage.service.test.ts b/packages/cli/src/context/ingest/page-triage/page-triage.service.test.ts similarity index 100% rename from packages/context/src/ingest/page-triage/page-triage.service.test.ts rename to packages/cli/src/context/ingest/page-triage/page-triage.service.test.ts diff --git a/packages/context/src/ingest/page-triage/page-triage.service.ts b/packages/cli/src/context/ingest/page-triage/page-triage.service.ts similarity index 97% rename from packages/context/src/ingest/page-triage/page-triage.service.ts rename to packages/cli/src/context/ingest/page-triage/page-triage.service.ts index cb9ea471..289a6aeb 100644 --- a/packages/context/src/ingest/page-triage/page-triage.service.ts +++ b/packages/cli/src/context/ingest/page-triage/page-triage.service.ts @@ -3,10 +3,10 @@ import { readdir, readFile } from 'node:fs/promises'; import { dirname, join, relative } from 'node:path'; import pLimit from 'p-limit'; import { z } from 'zod'; -import { type KtxLogger, noopLogger } from '../../core/index.js'; -import type { KtxLlmRuntimePort } from '../../llm/index.js'; -import type { PromptService } from '../../prompts/index.js'; -import type { InsertContextCandidateInput } from '../context-candidates/index.js'; +import { type KtxLogger, noopLogger } from '../../../context/core/config.js'; +import type { KtxLlmRuntimePort } from '../../../context/llm/runtime-port.js'; +import type { PromptService } from '../../../context/prompts/prompt.service.js'; +import type { InsertContextCandidateInput } from '../../../context/ingest/context-candidates/types.js'; import type { JsonValue } from '../ports.js'; import type { DiffSet, SourceAdapter, TriageLane, TriageSignals } from '../types.js'; @@ -42,7 +42,7 @@ interface StagedTriageDocument { markdown: string; } -export interface PageTriageReport { +interface PageTriageReport { pageCount: number; skip: number; light: number; @@ -89,7 +89,7 @@ export interface PageTriageStorePort { insertCandidate(input: InsertContextCandidateInput): Promise; } -export interface PageTriageSettings { +interface PageTriageSettings { enabled: boolean; maxConcurrency: number; lightExtractionEnabled: boolean; diff --git a/packages/context/src/ingest/parsed-target-table.ts b/packages/cli/src/context/ingest/parsed-target-table.ts similarity index 100% rename from packages/context/src/ingest/parsed-target-table.ts rename to packages/cli/src/context/ingest/parsed-target-table.ts diff --git a/packages/context/src/ingest/ports.ts b/packages/cli/src/context/ingest/ports.ts similarity index 83% rename from packages/context/src/ingest/ports.ts rename to packages/cli/src/context/ingest/ports.ts index 15123089..76e9d765 100644 --- a/packages/context/src/ingest/ports.ts +++ b/packages/cli/src/context/ingest/ports.ts @@ -1,20 +1,22 @@ -import type { KtxModelRole } from '@ktx/llm'; +import type { KtxModelRole } from '../../llm/types.js'; import type { KtxEmbeddingPort } from '../core/embedding.js'; -import type { GitService, KtxFileStorePort, KtxLogger, SessionOutcome } from '../core/index.js'; -import type { AgentRunnerPort, KtxLlmRuntimePort, KtxRuntimeToolSet } from '../llm/index.js'; -import type { CaptureSession, MemoryAction, MemoryKnowledgeSlRefsPort } from '../memory/index.js'; -import type { PromptService } from '../prompts/index.js'; -import type { SkillsRegistryService } from '../skills/index.js'; -import type { - SemanticLayerService, - SlConnectionCatalogPort, - SlSearchService, - SlSourcesIndexPort, - SlValidationDeps, - SlValidatorPort, -} from '../sl/index.js'; -import type { ToolContext, ToolSession } from '../tools/index.js'; -import type { KnowledgeIndexPort, KnowledgeWikiService } from '../wiki/index.js'; +import type { GitService } from '../../context/core/git.service.js'; +import type { KtxFileStorePort } from '../../context/core/file-store.js'; +import type { KtxLogger } from '../../context/core/config.js'; +import type { SessionOutcome } from '../../context/core/session-worktree.service.js'; +import type { AgentRunnerPort, KtxLlmRuntimePort, KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; +import type { MemoryAction, MemoryKnowledgeSlRefsPort } from '../../context/memory/types.js'; +import type { PromptService } from '../../context/prompts/prompt.service.js'; +import type { SkillsRegistryService } from '../../context/skills/skills-registry.service.js'; +import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import type { SlConnectionCatalogPort, SlSourcesIndexPort } from '../../context/sl/ports.js'; +import type { SlSearchService } from '../../context/sl/sl-search.service.js'; +import type { SlValidationDeps } from '../../context/sl/tools/sl-warehouse-validation.js'; +import type { SlValidatorPort } from '../../context/sl/sl-validator.port.js'; +import type { ToolContext } from '../../context/tools/base-tool.js'; +import type { ToolSession } from '../../context/tools/tool-session.js'; +import type { KnowledgeIndexPort } from '../../context/wiki/ports.js'; +import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; import type { CanonicalPin } from './canonical-pins.js'; import type { IngestTraceLevel } from './ingest-trace.js'; import type { IngestReportSnapshot } from './reports.js'; @@ -28,13 +30,12 @@ import type { StageIndex } from './stages/stage-index.types.js'; import type { DiffSet, EvictionUnit, - IngestBundleJob, IngestDiffSummary, IngestTrigger, SourceAdapter, } from './types.js'; -export type JsonPrimitive = string | number | boolean | null; +type JsonPrimitive = string | number | boolean | null; export type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue | undefined }; export interface IngestRunRecord { @@ -120,7 +121,7 @@ export interface IngestLockPort { withLock(key: string, fn: () => Promise): Promise; } -export interface IngestFileStorePort extends KtxFileStorePort {} +interface IngestFileStorePort extends KtxFileStorePort {} export interface IngestSessionWorktree { chatId: string; @@ -137,7 +138,7 @@ export interface IngestSessionWorktreePort { cleanup(session: IngestSessionWorktree, outcome: SessionOutcome): Promise; } -export interface IngestSettingsPort { +interface IngestSettingsPort { memoryIngestionModel: string; probeRowCount: number; workUnitMaxConcurrency?: number; @@ -146,7 +147,7 @@ export interface IngestSettingsPort { ingestTraceLevel?: IngestTraceLevel; } -export interface IngestGitAuthor { +interface IngestGitAuthor { name: string; email: string; } @@ -172,7 +173,7 @@ export interface IngestToolsetFactoryPort { createIngestWuToolset(session: ToolSession, options?: { includeContextEvidenceTools?: boolean }): IngestToolsetLike; } -export type IngestKnowledgeIndexPort = Pick; +type IngestKnowledgeIndexPort = Pick; export interface SourceAdapterRegistryPort { register(adapter: SourceAdapter): void; @@ -181,7 +182,7 @@ export interface SourceAdapterRegistryPort { list(): string[]; } -export interface DiffSetComputerPort { +interface DiffSetComputerPort { compute( connectionId: string, sourceKey: string, @@ -203,7 +204,7 @@ export interface ContextEvidenceIndexSummary { warnings: string[]; } -export interface ContextEvidenceIndexPort { +interface ContextEvidenceIndexPort { indexStagedDir(args: { stagedDir: string; runId: string; @@ -230,7 +231,7 @@ export interface PageTriageRunResult { warnings: string[]; } -export interface PageTriagePort { +interface PageTriagePort { triageRun(args: { stagedDir: string; runId: string; @@ -243,7 +244,7 @@ export interface PageTriagePort { }): Promise; } -export interface ContextCandidateCarryforwardPort { +interface ContextCandidateCarryforwardPort { carryForward(args: { runId: string; connectionId: string; sourceKey: string }): Promise<{ warnings: string[] }>; } @@ -271,7 +272,7 @@ export interface CandidateDedupResult { warnings: string[]; } -export interface CandidateDedupPort { +interface CandidateDedupPort { deduplicateRun(runId: string): Promise; } @@ -284,7 +285,7 @@ export interface ContextCandidateSummary { conflict: number; } -export interface ContextEvidenceCandidatesPort { +interface ContextEvidenceCandidatesPort { getCandidateSummary(runId: string): Promise; } @@ -358,10 +359,3 @@ export interface IngestBundleRunnerDeps { curatorPagination?: CuratorPaginationPort; logger?: KtxLogger; } - -export interface IngestCaptureState { - session: CaptureSession; - actions: MemoryAction[]; -} - -export type IngestRunnerJob = IngestBundleJob; diff --git a/packages/context/src/ingest/raw-sources-paths.test.ts b/packages/cli/src/context/ingest/raw-sources-paths.test.ts similarity index 100% rename from packages/context/src/ingest/raw-sources-paths.test.ts rename to packages/cli/src/context/ingest/raw-sources-paths.test.ts diff --git a/packages/context/src/ingest/raw-sources-paths.ts b/packages/cli/src/context/ingest/raw-sources-paths.ts similarity index 96% rename from packages/context/src/ingest/raw-sources-paths.ts rename to packages/cli/src/context/ingest/raw-sources-paths.ts index 654ce8ba..9e9f847f 100644 --- a/packages/context/src/ingest/raw-sources-paths.ts +++ b/packages/cli/src/context/ingest/raw-sources-paths.ts @@ -1,3 +1,4 @@ +/** @internal */ export const rawSourcesRoot = 'raw-sources'; export function buildSyncId(now: Date, jobId: string): string { @@ -14,6 +15,7 @@ export function rawSourcesDirForSync(connectionId: string, sourceKey: string, sy return `${rawSourcesRoot}/${connectionId}/${sourceKey}/${syncId}`; } +/** @internal */ export function provenanceMarker(rawPath: string, startLine: number, endLine: number): string { return ``; } diff --git a/packages/context/src/ingest/repo-fetch.test.ts b/packages/cli/src/context/ingest/repo-fetch.test.ts similarity index 100% rename from packages/context/src/ingest/repo-fetch.test.ts rename to packages/cli/src/context/ingest/repo-fetch.test.ts diff --git a/packages/context/src/ingest/repo-fetch.ts b/packages/cli/src/context/ingest/repo-fetch.ts similarity index 97% rename from packages/context/src/ingest/repo-fetch.ts rename to packages/cli/src/context/ingest/repo-fetch.ts index cf394675..1fecb422 100644 --- a/packages/context/src/ingest/repo-fetch.ts +++ b/packages/cli/src/context/ingest/repo-fetch.ts @@ -3,12 +3,14 @@ import { join } from 'node:path'; import type { CloneOptions } from 'simple-git'; import { createSimpleGit } from './git-env.js'; +/** @internal */ export interface RepoFetchConfig { repoUrl: string; branch?: string; authToken?: string | null; } +/** @internal */ export class RepoConfigError extends Error { constructor(message: string) { super(message); @@ -16,6 +18,7 @@ export class RepoConfigError extends Error { } } +/** @internal */ export class RepoFetchError extends Error { constructor(message: string) { super(message); @@ -23,6 +26,7 @@ export class RepoFetchError extends Error { } } +/** @internal */ export function validateRepoConfig(config: RepoFetchConfig): void { if (!config.repoUrl) { throw new RepoConfigError('Repository URL is required'); @@ -35,6 +39,7 @@ export function validateRepoConfig(config: RepoFetchConfig): void { } } +/** @internal */ export function buildAuthenticatedUrl(repoUrl: string, authToken: string | null | undefined): string { if (!authToken) { return repoUrl; @@ -70,6 +75,7 @@ export function sanitizeRepoError(err: unknown, authToken: string | null | undef return sanitized; } +/** @internal */ export async function repoDirExists(dir: string): Promise { try { await access(join(dir, '.git')); @@ -130,6 +136,7 @@ export async function testRepoConnection(args: { } } +/** @internal */ export async function cleanupRepoDir(dir: string): Promise { await rm(dir, { recursive: true, force: true }); } diff --git a/packages/context/src/ingest/report-snapshot.test.ts b/packages/cli/src/context/ingest/report-snapshot.test.ts similarity index 100% rename from packages/context/src/ingest/report-snapshot.test.ts rename to packages/cli/src/context/ingest/report-snapshot.test.ts diff --git a/packages/context/src/ingest/report-snapshot.ts b/packages/cli/src/context/ingest/report-snapshot.ts similarity index 99% rename from packages/context/src/ingest/report-snapshot.ts rename to packages/cli/src/context/ingest/report-snapshot.ts index 3cde6a4e..d91f374b 100644 --- a/packages/context/src/ingest/report-snapshot.ts +++ b/packages/cli/src/context/ingest/report-snapshot.ts @@ -158,7 +158,7 @@ const finalizationOutcomeSchema = z.object({ provenanceExclusions: z.array(finalizationProvenanceExclusionSchema).default([]), }); -export const ingestReportSnapshotSchema = z +const ingestReportSnapshotSchema = z .object({ id: z.string().min(1), runId: z.string().min(1), diff --git a/packages/context/src/ingest/reports.ts b/packages/cli/src/context/ingest/reports.ts similarity index 90% rename from packages/context/src/ingest/reports.ts rename to packages/cli/src/context/ingest/reports.ts index 280b224f..ea02a31a 100644 --- a/packages/context/src/ingest/reports.ts +++ b/packages/cli/src/context/ingest/reports.ts @@ -1,5 +1,5 @@ -import type { MemoryAction } from '../memory/index.js'; -import type { TouchedSlSource } from '../tools/index.js'; +import type { MemoryAction } from '../../context/memory/types.js'; +import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js'; import type { MemoryFlowReplayInput } from './memory-flow/types.js'; import type { IngestProvenanceInsert } from './ports.js'; import type { @@ -31,7 +31,7 @@ export interface IngestReportProvenanceDetail { actionType: IngestProvenanceInsert['actionType']; } -export interface IngestReportToolTranscriptSummary { +interface IngestReportToolTranscriptSummary { unitKey: string; path: string; toolCallCount: number; @@ -68,7 +68,7 @@ export interface IngestReportFinalizationOutcome { provenanceExclusions: IngestReportFinalizationProvenanceExclusion[]; } -export interface IngestReportFailure { +interface IngestReportFailure { phase: string; message: string; details?: Record; @@ -135,16 +135,6 @@ export interface IngestSavedMemoryCounts { slCount: number; } -export function finalizationSavedMemoryCounts( - finalization: IngestReportFinalizationOutcome | undefined, -): IngestSavedMemoryCounts { - const actions = finalization?.actions ?? []; - return { - wikiCount: actions.filter((action) => action.target === 'wiki').length, - slCount: actions.filter((action) => action.target === 'sl').length, - }; -} - export function savedMemoryCountsForReport(report: IngestReportSnapshot): IngestSavedMemoryCounts { const workUnitActions = report.body.workUnits.flatMap((workUnit) => workUnit.actions); const reconciliationActions = report.body.reconciliationActions ?? []; diff --git a/packages/context/src/ingest/semantic-layer-target-policy.test.ts b/packages/cli/src/context/ingest/semantic-layer-target-policy.test.ts similarity index 100% rename from packages/context/src/ingest/semantic-layer-target-policy.test.ts rename to packages/cli/src/context/ingest/semantic-layer-target-policy.test.ts diff --git a/packages/context/src/ingest/semantic-layer-target-policy.ts b/packages/cli/src/context/ingest/semantic-layer-target-policy.ts similarity index 96% rename from packages/context/src/ingest/semantic-layer-target-policy.ts rename to packages/cli/src/context/ingest/semantic-layer-target-policy.ts index adf63b3b..58cfc19e 100644 --- a/packages/context/src/ingest/semantic-layer-target-policy.ts +++ b/packages/cli/src/context/ingest/semantic-layer-target-policy.ts @@ -3,17 +3,20 @@ export interface SemanticLayerTargetPolicyInput { allowedConnectionIds: ReadonlySet; } +/** @internal */ export interface SemanticLayerTargetPolicyViolation { path: string; connectionId: string; } +/** @internal */ export function semanticLayerConnectionIdFromPath(path: string): string | null { const normalized = path.replace(/^[ab]\//, ''); const match = /^semantic-layer\/([^/]+)\//.exec(normalized); return match?.[1] ?? null; } +/** @internal */ export function findDisallowedSemanticLayerTargetPaths( input: SemanticLayerTargetPolicyInput, ): SemanticLayerTargetPolicyViolation[] { diff --git a/packages/context/src/ingest/source-adapter-registry.test.ts b/packages/cli/src/context/ingest/source-adapter-registry.test.ts similarity index 100% rename from packages/context/src/ingest/source-adapter-registry.test.ts rename to packages/cli/src/context/ingest/source-adapter-registry.test.ts diff --git a/packages/context/src/ingest/source-adapter-registry.ts b/packages/cli/src/context/ingest/source-adapter-registry.ts similarity index 100% rename from packages/context/src/ingest/source-adapter-registry.ts rename to packages/cli/src/context/ingest/source-adapter-registry.ts diff --git a/packages/context/src/ingest/sqlite-bundle-ingest-store.test.ts b/packages/cli/src/context/ingest/sqlite-bundle-ingest-store.test.ts similarity index 99% rename from packages/context/src/ingest/sqlite-bundle-ingest-store.test.ts rename to packages/cli/src/context/ingest/sqlite-bundle-ingest-store.test.ts index cd6d2385..0cee47d0 100644 --- a/packages/context/src/ingest/sqlite-bundle-ingest-store.test.ts +++ b/packages/cli/src/context/ingest/sqlite-bundle-ingest-store.test.ts @@ -3,7 +3,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { DiffSetService } from './diff-set.service.js'; -import type { IngestDiffSummary, IngestReportBody, IngestTrigger } from './index.js'; +import type { IngestDiffSummary, IngestTrigger } from '../../context/ingest/types.js'; +import type { IngestReportBody } from '../../context/ingest/reports.js'; import { SqliteBundleIngestStore } from './sqlite-bundle-ingest-store.js'; function idFactory(ids: string[]): () => string { diff --git a/packages/context/src/ingest/sqlite-bundle-ingest-store.ts b/packages/cli/src/context/ingest/sqlite-bundle-ingest-store.ts similarity index 100% rename from packages/context/src/ingest/sqlite-bundle-ingest-store.ts rename to packages/cli/src/context/ingest/sqlite-bundle-ingest-store.ts diff --git a/packages/context/src/ingest/sqlite-local-ingest-store.test.ts b/packages/cli/src/context/ingest/sqlite-local-ingest-store.test.ts similarity index 100% rename from packages/context/src/ingest/sqlite-local-ingest-store.test.ts rename to packages/cli/src/context/ingest/sqlite-local-ingest-store.test.ts diff --git a/packages/context/src/ingest/sqlite-local-ingest-store.ts b/packages/cli/src/context/ingest/sqlite-local-ingest-store.ts similarity index 100% rename from packages/context/src/ingest/sqlite-local-ingest-store.ts rename to packages/cli/src/context/ingest/sqlite-local-ingest-store.ts diff --git a/packages/context/src/ingest/stages/build-reconcile-context.context-candidates.test.ts b/packages/cli/src/context/ingest/stages/build-reconcile-context.context-candidates.test.ts similarity index 100% rename from packages/context/src/ingest/stages/build-reconcile-context.context-candidates.test.ts rename to packages/cli/src/context/ingest/stages/build-reconcile-context.context-candidates.test.ts diff --git a/packages/context/src/ingest/stages/build-reconcile-context.test.ts b/packages/cli/src/context/ingest/stages/build-reconcile-context.test.ts similarity index 100% rename from packages/context/src/ingest/stages/build-reconcile-context.test.ts rename to packages/cli/src/context/ingest/stages/build-reconcile-context.test.ts diff --git a/packages/context/src/ingest/stages/build-reconcile-context.ts b/packages/cli/src/context/ingest/stages/build-reconcile-context.ts similarity index 98% rename from packages/context/src/ingest/stages/build-reconcile-context.ts rename to packages/cli/src/context/ingest/stages/build-reconcile-context.ts index 366eeb4f..0f033616 100644 --- a/packages/context/src/ingest/stages/build-reconcile-context.ts +++ b/packages/cli/src/context/ingest/stages/build-reconcile-context.ts @@ -1,5 +1,5 @@ import { buildCanonicalPinsPromptBlock, type CanonicalPin } from '../canonical-pins.js'; -import type { KtxRuntimeToolSet } from '../../llm/index.js'; +import type { KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; import { createVerificationLedgerState, VERIFICATION_LEDGER_PROMPT, diff --git a/packages/context/src/ingest/stages/build-wu-context.test.ts b/packages/cli/src/context/ingest/stages/build-wu-context.test.ts similarity index 100% rename from packages/context/src/ingest/stages/build-wu-context.test.ts rename to packages/cli/src/context/ingest/stages/build-wu-context.test.ts diff --git a/packages/context/src/ingest/stages/build-wu-context.ts b/packages/cli/src/context/ingest/stages/build-wu-context.ts similarity index 96% rename from packages/context/src/ingest/stages/build-wu-context.ts rename to packages/cli/src/context/ingest/stages/build-wu-context.ts index 14c2912b..f8fb4af4 100644 --- a/packages/context/src/ingest/stages/build-wu-context.ts +++ b/packages/cli/src/context/ingest/stages/build-wu-context.ts @@ -1,6 +1,7 @@ import { buildCanonicalPinsPromptBlock, type CanonicalPin } from '../canonical-pins.js'; import { createLookerQueryToSlTool } from '../adapters/looker/tools/looker-query-to-sl.tool.js'; -import { createRuntimeToolDescriptorFromAiTool, type KtxRuntimeToolSet } from '../../llm/index.js'; +import { createRuntimeToolDescriptorFromAiTool } from '../../../context/llm/runtime-tools.js'; +import type { KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; import type { IngestProvenanceRow } from '../ports.js'; import { createReadRawFileTool } from '../tools/read-raw-file.tool.js'; import { createReadRawSpanTool } from '../tools/read-raw-span.tool.js'; diff --git a/packages/context/src/ingest/stages/stage-1-stage-raw-files.test.ts b/packages/cli/src/context/ingest/stages/stage-1-stage-raw-files.test.ts similarity index 100% rename from packages/context/src/ingest/stages/stage-1-stage-raw-files.test.ts rename to packages/cli/src/context/ingest/stages/stage-1-stage-raw-files.test.ts diff --git a/packages/context/src/ingest/stages/stage-1-stage-raw-files.ts b/packages/cli/src/context/ingest/stages/stage-1-stage-raw-files.ts similarity index 100% rename from packages/context/src/ingest/stages/stage-1-stage-raw-files.ts rename to packages/cli/src/context/ingest/stages/stage-1-stage-raw-files.ts diff --git a/packages/context/src/ingest/stages/stage-3-work-units.test.ts b/packages/cli/src/context/ingest/stages/stage-3-work-units.test.ts similarity index 98% rename from packages/context/src/ingest/stages/stage-3-work-units.test.ts rename to packages/cli/src/context/ingest/stages/stage-3-work-units.test.ts index 2cb0b432..fc39fd9b 100644 --- a/packages/context/src/ingest/stages/stage-3-work-units.test.ts +++ b/packages/cli/src/context/ingest/stages/stage-3-work-units.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import type { CaptureSession, MemoryAction } from '../../memory/index.js'; -import { addTouchedSlSource, createTouchedSlSources } from '../../tools/index.js'; +import type { CaptureSession, MemoryAction } from '../../../context/memory/types.js'; +import { addTouchedSlSource, createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; import type { WorkUnit } from '../types.js'; import { executeWorkUnit, type WorkUnitExecutionDeps } from './stage-3-work-units.js'; diff --git a/packages/context/src/ingest/stages/stage-3-work-units.ts b/packages/cli/src/context/ingest/stages/stage-3-work-units.ts similarity index 94% rename from packages/context/src/ingest/stages/stage-3-work-units.ts rename to packages/cli/src/context/ingest/stages/stage-3-work-units.ts index caa78ebf..96c0e65c 100644 --- a/packages/context/src/ingest/stages/stage-3-work-units.ts +++ b/packages/cli/src/context/ingest/stages/stage-3-work-units.ts @@ -1,12 +1,12 @@ -import type { KtxModelRole } from '@ktx/llm'; -import type { AgentRunnerPort, KtxRuntimeToolSet } from '@ktx/context'; -import type { CaptureSession, MemoryAction } from '../../memory/index.js'; -import { listTouchedSlSources, type TouchedSlSource } from '../../tools/index.js'; +import type { KtxModelRole } from '../../../llm/types.js'; +import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; +import type { CaptureSession, MemoryAction } from '../../../context/memory/types.js'; +import { listTouchedSlSources, type TouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; import type { WorkUnit } from '../types.js'; const MAX_WORK_UNIT_PROMPT_CHARS = 240_000; -export interface TouchedValidationResult { +interface TouchedValidationResult { invalidSources: string[]; validSources: string[]; } diff --git a/packages/context/src/ingest/stages/stage-4-reconciliation.test.ts b/packages/cli/src/context/ingest/stages/stage-4-reconciliation.test.ts similarity index 100% rename from packages/context/src/ingest/stages/stage-4-reconciliation.test.ts rename to packages/cli/src/context/ingest/stages/stage-4-reconciliation.test.ts diff --git a/packages/context/src/ingest/stages/stage-4-reconciliation.ts b/packages/cli/src/context/ingest/stages/stage-4-reconciliation.ts similarity index 91% rename from packages/context/src/ingest/stages/stage-4-reconciliation.ts rename to packages/cli/src/context/ingest/stages/stage-4-reconciliation.ts index 59f3235e..61fc1efe 100644 --- a/packages/context/src/ingest/stages/stage-4-reconciliation.ts +++ b/packages/cli/src/context/ingest/stages/stage-4-reconciliation.ts @@ -1,5 +1,5 @@ -import type { AgentRunnerPort, KtxRuntimeToolSet } from '@ktx/context'; -import type { KtxModelRole } from '@ktx/llm'; +import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; +import type { KtxModelRole } from '../../../llm/types.js'; import type { EvictionUnit } from '../types.js'; import type { StageIndex } from './stage-index.types.js'; diff --git a/packages/context/src/ingest/stages/stage-index.types.ts b/packages/cli/src/context/ingest/stages/stage-index.types.ts similarity index 89% rename from packages/context/src/ingest/stages/stage-index.types.ts rename to packages/cli/src/context/ingest/stages/stage-index.types.ts index fa1a652e..fd6fd59c 100644 --- a/packages/context/src/ingest/stages/stage-index.types.ts +++ b/packages/cli/src/context/ingest/stages/stage-index.types.ts @@ -1,7 +1,7 @@ -import type { MemoryAction } from '../../memory/index.js'; -import type { TouchedSlSource } from '../../tools/index.js'; +import type { MemoryAction } from '../../../context/memory/types.js'; +import type { TouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; -export interface StageIndexWorkUnit { +interface StageIndexWorkUnit { unitKey: string; rawFiles: string[]; status: 'success' | 'failed'; diff --git a/packages/context/src/ingest/stages/validate-wu-sources.test.ts b/packages/cli/src/context/ingest/stages/validate-wu-sources.test.ts similarity index 100% rename from packages/context/src/ingest/stages/validate-wu-sources.test.ts rename to packages/cli/src/context/ingest/stages/validate-wu-sources.test.ts diff --git a/packages/context/src/ingest/stages/validate-wu-sources.ts b/packages/cli/src/context/ingest/stages/validate-wu-sources.ts similarity index 74% rename from packages/context/src/ingest/stages/validate-wu-sources.ts rename to packages/cli/src/context/ingest/stages/validate-wu-sources.ts index dd72c8d0..4bc3aaa0 100644 --- a/packages/context/src/ingest/stages/validate-wu-sources.ts +++ b/packages/cli/src/context/ingest/stages/validate-wu-sources.ts @@ -1,5 +1,6 @@ -import type { SlValidationDeps, SlValidatorPort } from '@ktx/context/sl'; -import type { TouchedSlSource } from '../../tools/index.js'; +import type { SlValidationDeps } from '../../../context/sl/tools/sl-warehouse-validation.js'; +import type { SlValidatorPort } from '../../../context/sl/sl-validator.port.js'; +import type { TouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; export interface WuValidationResult { validSources: string[]; diff --git a/packages/context/src/ingest/tools/emit-artifact-resolution.tool.ts b/packages/cli/src/context/ingest/tools/emit-artifact-resolution.tool.ts similarity index 100% rename from packages/context/src/ingest/tools/emit-artifact-resolution.tool.ts rename to packages/cli/src/context/ingest/tools/emit-artifact-resolution.tool.ts diff --git a/packages/context/src/ingest/tools/emit-conflict-resolution.tool.ts b/packages/cli/src/context/ingest/tools/emit-conflict-resolution.tool.ts similarity index 100% rename from packages/context/src/ingest/tools/emit-conflict-resolution.tool.ts rename to packages/cli/src/context/ingest/tools/emit-conflict-resolution.tool.ts diff --git a/packages/context/src/ingest/tools/emit-eviction-decision.tool.ts b/packages/cli/src/context/ingest/tools/emit-eviction-decision.tool.ts similarity index 100% rename from packages/context/src/ingest/tools/emit-eviction-decision.tool.ts rename to packages/cli/src/context/ingest/tools/emit-eviction-decision.tool.ts diff --git a/packages/context/src/ingest/tools/emit-reconciliation-records.tool.test.ts b/packages/cli/src/context/ingest/tools/emit-reconciliation-records.tool.test.ts similarity index 100% rename from packages/context/src/ingest/tools/emit-reconciliation-records.tool.test.ts rename to packages/cli/src/context/ingest/tools/emit-reconciliation-records.tool.test.ts diff --git a/packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts b/packages/cli/src/context/ingest/tools/emit-unmapped-fallback.tool.ts similarity index 100% rename from packages/context/src/ingest/tools/emit-unmapped-fallback.tool.ts rename to packages/cli/src/context/ingest/tools/emit-unmapped-fallback.tool.ts diff --git a/packages/context/src/ingest/tools/eviction-list.tool.test.ts b/packages/cli/src/context/ingest/tools/eviction-list.tool.test.ts similarity index 100% rename from packages/context/src/ingest/tools/eviction-list.tool.test.ts rename to packages/cli/src/context/ingest/tools/eviction-list.tool.test.ts diff --git a/packages/context/src/ingest/tools/eviction-list.tool.ts b/packages/cli/src/context/ingest/tools/eviction-list.tool.ts similarity index 100% rename from packages/context/src/ingest/tools/eviction-list.tool.ts rename to packages/cli/src/context/ingest/tools/eviction-list.tool.ts diff --git a/packages/context/src/ingest/tools/read-raw-file.tool.test.ts b/packages/cli/src/context/ingest/tools/read-raw-file.tool.test.ts similarity index 100% rename from packages/context/src/ingest/tools/read-raw-file.tool.test.ts rename to packages/cli/src/context/ingest/tools/read-raw-file.tool.test.ts diff --git a/packages/context/src/ingest/tools/read-raw-file.tool.ts b/packages/cli/src/context/ingest/tools/read-raw-file.tool.ts similarity index 100% rename from packages/context/src/ingest/tools/read-raw-file.tool.ts rename to packages/cli/src/context/ingest/tools/read-raw-file.tool.ts diff --git a/packages/context/src/ingest/tools/read-raw-span.tool.test.ts b/packages/cli/src/context/ingest/tools/read-raw-span.tool.test.ts similarity index 100% rename from packages/context/src/ingest/tools/read-raw-span.tool.test.ts rename to packages/cli/src/context/ingest/tools/read-raw-span.tool.test.ts diff --git a/packages/context/src/ingest/tools/read-raw-span.tool.ts b/packages/cli/src/context/ingest/tools/read-raw-span.tool.ts similarity index 100% rename from packages/context/src/ingest/tools/read-raw-span.tool.ts rename to packages/cli/src/context/ingest/tools/read-raw-span.tool.ts diff --git a/packages/context/src/ingest/tools/stage-diff.tool.test.ts b/packages/cli/src/context/ingest/tools/stage-diff.tool.test.ts similarity index 100% rename from packages/context/src/ingest/tools/stage-diff.tool.test.ts rename to packages/cli/src/context/ingest/tools/stage-diff.tool.test.ts diff --git a/packages/context/src/ingest/tools/stage-diff.tool.ts b/packages/cli/src/context/ingest/tools/stage-diff.tool.ts similarity index 100% rename from packages/context/src/ingest/tools/stage-diff.tool.ts rename to packages/cli/src/context/ingest/tools/stage-diff.tool.ts diff --git a/packages/context/src/ingest/tools/stage-list.tool.test.ts b/packages/cli/src/context/ingest/tools/stage-list.tool.test.ts similarity index 100% rename from packages/context/src/ingest/tools/stage-list.tool.test.ts rename to packages/cli/src/context/ingest/tools/stage-list.tool.test.ts diff --git a/packages/context/src/ingest/tools/stage-list.tool.ts b/packages/cli/src/context/ingest/tools/stage-list.tool.ts similarity index 100% rename from packages/context/src/ingest/tools/stage-list.tool.ts rename to packages/cli/src/context/ingest/tools/stage-list.tool.ts diff --git a/packages/context/src/ingest/tools/tool-call-logger.ts b/packages/cli/src/context/ingest/tools/tool-call-logger.ts similarity index 97% rename from packages/context/src/ingest/tools/tool-call-logger.ts rename to packages/cli/src/context/ingest/tools/tool-call-logger.ts index c91a74c7..61d020c6 100644 --- a/packages/context/src/ingest/tools/tool-call-logger.ts +++ b/packages/cli/src/context/ingest/tools/tool-call-logger.ts @@ -1,6 +1,6 @@ import { appendFile, mkdir } from 'node:fs/promises'; import { dirname } from 'node:path'; -import type { KtxRuntimeToolSet } from '../../llm/index.js'; +import type { KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; export interface ToolCallLogEntry { ts: string; diff --git a/packages/context/src/ingest/tools/tool-transcript-summary.test.ts b/packages/cli/src/context/ingest/tools/tool-transcript-summary.test.ts similarity index 100% rename from packages/context/src/ingest/tools/tool-transcript-summary.test.ts rename to packages/cli/src/context/ingest/tools/tool-transcript-summary.test.ts diff --git a/packages/context/src/ingest/tools/tool-transcript-summary.ts b/packages/cli/src/context/ingest/tools/tool-transcript-summary.ts similarity index 100% rename from packages/context/src/ingest/tools/tool-transcript-summary.ts rename to packages/cli/src/context/ingest/tools/tool-transcript-summary.ts diff --git a/packages/context/src/ingest/tools/verification-ledger.tool.ts b/packages/cli/src/context/ingest/tools/verification-ledger.tool.ts similarity index 98% rename from packages/context/src/ingest/tools/verification-ledger.tool.ts rename to packages/cli/src/context/ingest/tools/verification-ledger.tool.ts index 7dd3b56c..4ab87131 100644 --- a/packages/context/src/ingest/tools/verification-ledger.tool.ts +++ b/packages/cli/src/context/ingest/tools/verification-ledger.tool.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import type { KtxRuntimeToolDescriptor, KtxRuntimeToolSet } from '../../llm/index.js'; +import type { KtxRuntimeToolDescriptor, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; const verificationLedgerInputSchema = z.object({ summary: z.string().min(1).max(2000), diff --git a/packages/context/src/ingest/tools/warehouse-verification/index.ts b/packages/cli/src/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.ts similarity index 81% rename from packages/context/src/ingest/tools/warehouse-verification/index.ts rename to packages/cli/src/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.ts index 0478305c..166713b9 100644 --- a/packages/context/src/ingest/tools/warehouse-verification/index.ts +++ b/packages/cli/src/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.ts @@ -1,7 +1,7 @@ -import type { KtxFileStorePort } from '../../../core/index.js'; -import type { SlConnectionCatalogPort } from '../../../sl/index.js'; +import type { KtxFileStorePort } from '../../../core/file-store.js'; +import type { SlConnectionCatalogPort } from '../../../sl/ports.js'; import { WarehouseCatalogService } from '../../../scan/warehouse-catalog.js'; -import type { BaseTool, ToolContext } from '../../../tools/index.js'; +import type { BaseTool, ToolContext } from '../../../tools/base-tool.js'; import { DiscoverDataTool } from './discover-data.tool.js'; import { EntityDetailsTool } from './entity-details.tool.js'; import { SqlExecutionTool } from './sql-execution.tool.js'; diff --git a/packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts b/packages/cli/src/context/ingest/tools/warehouse-verification/discover-data.tool.test.ts similarity index 98% rename from packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts rename to packages/cli/src/context/ingest/tools/warehouse-verification/discover-data.tool.test.ts index 8982e300..7aebc101 100644 --- a/packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.test.ts +++ b/packages/cli/src/context/ingest/tools/warehouse-verification/discover-data.tool.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { WarehouseCatalogService } from '../../../scan/warehouse-catalog.js'; -import type { BaseTool, ToolContext } from '../../../tools/index.js'; +import type { BaseTool, ToolContext } from '../../../../context/tools/base-tool.js'; import { DiscoverDataTool } from './discover-data.tool.js'; describe('DiscoverDataTool', () => { diff --git a/packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts b/packages/cli/src/context/ingest/tools/warehouse-verification/discover-data.tool.ts similarity index 99% rename from packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts rename to packages/cli/src/context/ingest/tools/warehouse-verification/discover-data.tool.ts index 4d13ea6b..e358e970 100644 --- a/packages/context/src/ingest/tools/warehouse-verification/discover-data.tool.ts +++ b/packages/cli/src/context/ingest/tools/warehouse-verification/discover-data.tool.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; import { WarehouseCatalogService, type RawSchemaHit } from '../../../scan/warehouse-catalog.js'; -import { BaseTool, type ToolContext, type ToolOutput } from '../../../tools/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../../context/tools/base-tool.js'; const discoverDataInputSchema = z.object({ query: z.string().optional(), diff --git a/packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts b/packages/cli/src/context/ingest/tools/warehouse-verification/entity-details.tool.test.ts similarity index 97% rename from packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts rename to packages/cli/src/context/ingest/tools/warehouse-verification/entity-details.tool.test.ts index e6cdbdc8..fcef38df 100644 --- a/packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.test.ts +++ b/packages/cli/src/context/ingest/tools/warehouse-verification/entity-details.tool.test.ts @@ -2,9 +2,9 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../../../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../../../context/project/project.js'; import { WarehouseCatalogService } from '../../../scan/warehouse-catalog.js'; -import type { ToolContext } from '../../../tools/index.js'; +import type { ToolContext } from '../../../../context/tools/base-tool.js'; import { EntityDetailsTool } from './entity-details.tool.js'; describe('EntityDetailsTool', () => { diff --git a/packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts b/packages/cli/src/context/ingest/tools/warehouse-verification/entity-details.tool.ts similarity index 99% rename from packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts rename to packages/cli/src/context/ingest/tools/warehouse-verification/entity-details.tool.ts index 79ce92b2..45ecba2b 100644 --- a/packages/context/src/ingest/tools/warehouse-verification/entity-details.tool.ts +++ b/packages/cli/src/context/ingest/tools/warehouse-verification/entity-details.tool.ts @@ -1,7 +1,7 @@ import { z } from 'zod'; import type { KtxTableRef } from '../../../scan/types.js'; import { WarehouseCatalogService, type TableDetail } from '../../../scan/warehouse-catalog.js'; -import { BaseTool, type ToolContext, type ToolOutput } from '../../../tools/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../../context/tools/base-tool.js'; const targetSchema = z.union([ z.object({ display: z.string().min(1) }), diff --git a/packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts b/packages/cli/src/context/ingest/tools/warehouse-verification/sql-execution.tool.test.ts similarity index 94% rename from packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts rename to packages/cli/src/context/ingest/tools/warehouse-verification/sql-execution.tool.test.ts index ec7ef0ba..4458471a 100644 --- a/packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.test.ts +++ b/packages/cli/src/context/ingest/tools/warehouse-verification/sql-execution.tool.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import type { SlConnectionCatalogPort } from '../../../sl/index.js'; -import type { ToolContext } from '../../../tools/index.js'; +import type { SlConnectionCatalogPort } from '../../../../context/sl/ports.js'; +import type { ToolContext } from '../../../../context/tools/base-tool.js'; import { SqlExecutionTool } from './sql-execution.tool.js'; describe('SqlExecutionTool', () => { diff --git a/packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts b/packages/cli/src/context/ingest/tools/warehouse-verification/sql-execution.tool.ts similarity index 95% rename from packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts rename to packages/cli/src/context/ingest/tools/warehouse-verification/sql-execution.tool.ts index 8b2e3b5c..76911ce7 100644 --- a/packages/context/src/ingest/tools/warehouse-verification/sql-execution.tool.ts +++ b/packages/cli/src/context/ingest/tools/warehouse-verification/sql-execution.tool.ts @@ -1,7 +1,7 @@ import { z } from 'zod'; -import { assertReadOnlySql, limitSqlForExecution } from '../../../connections/index.js'; -import type { SlConnectionCatalogPort } from '../../../sl/index.js'; -import { BaseTool, type ToolContext, type ToolOutput } from '../../../tools/index.js'; +import { assertReadOnlySql, limitSqlForExecution } from '../../../../context/connections/read-only-sql.js'; +import type { SlConnectionCatalogPort } from '../../../../context/sl/ports.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../../context/tools/base-tool.js'; const sqlExecutionInputSchema = z.object({ connectionId: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), diff --git a/packages/context/src/ingest/types.ts b/packages/cli/src/context/ingest/types.ts similarity index 95% rename from packages/context/src/ingest/types.ts rename to packages/cli/src/context/ingest/types.ts index e0317141..991670f6 100644 --- a/packages/context/src/ingest/types.ts +++ b/packages/cli/src/context/ingest/types.ts @@ -1,7 +1,7 @@ import type { KtxEmbeddingPort } from '../core/embedding.js'; -import type { MemoryAction } from '../memory/index.js'; -import type { SemanticLayerService } from '../sl/index.js'; -import type { TouchedSlSource } from '../tools/index.js'; +import type { MemoryAction } from '../../context/memory/types.js'; +import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js'; import type { MemoryFlowEventSink } from './memory-flow/types.js'; import type { StageIndex } from './stages/stage-index.types.js'; import type { WorkUnitOutcome } from './stages/stage-3-work-units.js'; @@ -62,7 +62,7 @@ type SourceFetchIssueKind = | 'derived_table_not_supported' | 'lookml_connection_mismatch'; -export interface SourceFetchIssue { +interface SourceFetchIssue { rawPath: string; entityType: string; entityId: string | null; @@ -179,7 +179,7 @@ export interface SourceAdapter { }): Promise; } -export type IngestBundleRef = +type IngestBundleRef = | { kind: 'upload'; uploadId: string } | { kind: 'scheduled_pull'; config: unknown } | { kind: 'override'; priorJobId: string }; diff --git a/packages/context/src/ingest/wiki-body-refs.test.ts b/packages/cli/src/context/ingest/wiki-body-refs.test.ts similarity index 100% rename from packages/context/src/ingest/wiki-body-refs.test.ts rename to packages/cli/src/context/ingest/wiki-body-refs.test.ts diff --git a/packages/context/src/ingest/wiki-body-refs.ts b/packages/cli/src/context/ingest/wiki-body-refs.ts similarity index 97% rename from packages/context/src/ingest/wiki-body-refs.ts rename to packages/cli/src/context/ingest/wiki-body-refs.ts index 25f25eb3..fa62aefa 100644 --- a/packages/context/src/ingest/wiki-body-refs.ts +++ b/packages/cli/src/context/ingest/wiki-body-refs.ts @@ -1,5 +1,6 @@ -import type { SemanticLayerSource } from '../sl/index.js'; +import type { SemanticLayerSource } from '../../context/sl/types.js'; +/** @internal */ export type WikiBodyRef = | { kind: 'sl_entity'; connectionId: string | null; sourceName: string; entityName: string } | { kind: 'sl_source'; connectionId: string | null; sourceName: string } @@ -42,6 +43,7 @@ function isIdentifierToken(value: string): boolean { return /^[A-Za-z_][A-Za-z0-9_]*$/.test(value); } +/** @internal */ export function parseWikiBodyRefs(body: string): WikiBodyRef[] { const refs: WikiBodyRef[] = []; for (const line of visibleLinesOutsideFences(body)) { diff --git a/packages/context/src/ingest/wiki-sl-ref-repair.test.ts b/packages/cli/src/context/ingest/wiki-sl-ref-repair.test.ts similarity index 100% rename from packages/context/src/ingest/wiki-sl-ref-repair.test.ts rename to packages/cli/src/context/ingest/wiki-sl-ref-repair.test.ts diff --git a/packages/context/src/ingest/wiki-sl-ref-repair.ts b/packages/cli/src/context/ingest/wiki-sl-ref-repair.ts similarity index 91% rename from packages/context/src/ingest/wiki-sl-ref-repair.ts rename to packages/cli/src/context/ingest/wiki-sl-ref-repair.ts index 3205ebbd..7dcd1bd1 100644 --- a/packages/context/src/ingest/wiki-sl-ref-repair.ts +++ b/packages/cli/src/context/ingest/wiki-sl-ref-repair.ts @@ -1,7 +1,9 @@ -import type { KtxFileStorePort } from '../core/index.js'; -import type { SemanticLayerService, SemanticLayerSource } from '../sl/index.js'; +import type { KtxFileStorePort } from '../../context/core/file-store.js'; +import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import type { SemanticLayerSource } from '../../context/sl/types.js'; import { isFlatWikiKey } from '../wiki/keys.js'; -import type { KnowledgeWikiService, WikiFrontmatter } from '../wiki/index.js'; +import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; +import type { WikiFrontmatter } from '../../context/wiki/types.js'; const SYSTEM_AUTHOR = 'System User'; const SYSTEM_EMAIL = 'system@example.com'; diff --git a/packages/context/src/llm/ai-sdk-runtime.test.ts b/packages/cli/src/context/llm/ai-sdk-runtime.test.ts similarity index 100% rename from packages/context/src/llm/ai-sdk-runtime.test.ts rename to packages/cli/src/context/llm/ai-sdk-runtime.test.ts diff --git a/packages/context/src/llm/ai-sdk-runtime.ts b/packages/cli/src/context/llm/ai-sdk-runtime.ts similarity index 95% rename from packages/context/src/llm/ai-sdk-runtime.ts rename to packages/cli/src/context/llm/ai-sdk-runtime.ts index f6201813..33a55c11 100644 --- a/packages/context/src/llm/ai-sdk-runtime.ts +++ b/packages/cli/src/context/llm/ai-sdk-runtime.ts @@ -1,7 +1,8 @@ -import { KtxMessageBuilder, splitKtxSystemMessages, type KtxLlmProvider } from '@ktx/llm'; +import { KtxMessageBuilder, splitKtxSystemMessages } from '../../llm/message-builder.js'; +import type { KtxLlmProvider } from '../../llm/types.js'; import { generateText, Output, stepCountIs, type FlexibleSchema, type TelemetrySettings, type ToolSet } from 'ai'; import type { z } from 'zod'; -import { noopLogger, type KtxLogger } from '../core/index.js'; +import { noopLogger, type KtxLogger } from '../../context/core/config.js'; import { summarizeKtxLlmDebugRequest, type KtxLlmDebugRequestRecorder } from './debug-request-recorder.js'; import { createAiSdkToolSet } from './runtime-tools.js'; import type { @@ -12,7 +13,7 @@ import type { RunLoopResult, } from './runtime-port.js'; -export interface AgentTelemetryPort { +interface AgentTelemetryPort { createTelemetry(tags: Record): TelemetrySettings; } diff --git a/packages/context/src/llm/claude-code-env.test.ts b/packages/cli/src/context/llm/claude-code-env.test.ts similarity index 100% rename from packages/context/src/llm/claude-code-env.test.ts rename to packages/cli/src/context/llm/claude-code-env.test.ts diff --git a/packages/context/src/llm/claude-code-env.ts b/packages/cli/src/context/llm/claude-code-env.ts similarity index 97% rename from packages/context/src/llm/claude-code-env.ts rename to packages/cli/src/context/llm/claude-code-env.ts index 285113e4..a80f633f 100644 --- a/packages/context/src/llm/claude-code-env.ts +++ b/packages/cli/src/context/llm/claude-code-env.ts @@ -1,3 +1,4 @@ +/** @internal */ export const CLAUDE_CODE_PROVIDER_ENV_DENYLIST = [ 'ANTHROPIC_API_KEY', 'ANTHROPIC_AUTH_TOKEN', diff --git a/packages/context/src/llm/claude-code-models.test.ts b/packages/cli/src/context/llm/claude-code-models.test.ts similarity index 100% rename from packages/context/src/llm/claude-code-models.test.ts rename to packages/cli/src/context/llm/claude-code-models.test.ts diff --git a/packages/context/src/llm/claude-code-models.ts b/packages/cli/src/context/llm/claude-code-models.ts similarity index 100% rename from packages/context/src/llm/claude-code-models.ts rename to packages/cli/src/context/llm/claude-code-models.ts diff --git a/packages/context/src/llm/claude-code-runtime.test.ts b/packages/cli/src/context/llm/claude-code-runtime.test.ts similarity index 100% rename from packages/context/src/llm/claude-code-runtime.test.ts rename to packages/cli/src/context/llm/claude-code-runtime.test.ts diff --git a/packages/context/src/llm/claude-code-runtime.ts b/packages/cli/src/context/llm/claude-code-runtime.ts similarity index 99% rename from packages/context/src/llm/claude-code-runtime.ts rename to packages/cli/src/context/llm/claude-code-runtime.ts index bf815445..c6783d71 100644 --- a/packages/context/src/llm/claude-code-runtime.ts +++ b/packages/cli/src/context/llm/claude-code-runtime.ts @@ -6,7 +6,7 @@ import { type SDKResultMessage, } from '@anthropic-ai/claude-agent-sdk'; import { z } from 'zod'; -import { noopLogger, type KtxLogger } from '../core/index.js'; +import { noopLogger, type KtxLogger } from '../../context/core/config.js'; import { createKtxClaudeCodeEnv } from './claude-code-env.js'; import { resolveClaudeCodeModel } from './claude-code-models.js'; import { createClaudeSdkTools, mcpToolIds } from './runtime-tools.js'; @@ -59,6 +59,7 @@ function resultError(result: SDKResultMessage): Error | undefined { return new Error(`Claude Code query failed (${result.subtype})${details}`); } +/** @internal */ export function mapClaudeCodeStopReason(result: SDKResultMessage): RunLoopStopReason { if (result.subtype === 'error_max_turns') { return 'budget'; diff --git a/packages/context/src/llm/debug-request-recorder.test.ts b/packages/cli/src/context/llm/debug-request-recorder.test.ts similarity index 100% rename from packages/context/src/llm/debug-request-recorder.test.ts rename to packages/cli/src/context/llm/debug-request-recorder.test.ts diff --git a/packages/context/src/llm/debug-request-recorder.ts b/packages/cli/src/context/llm/debug-request-recorder.ts similarity index 96% rename from packages/context/src/llm/debug-request-recorder.ts rename to packages/cli/src/context/llm/debug-request-recorder.ts index 143b6890..92ae6ab9 100644 --- a/packages/context/src/llm/debug-request-recorder.ts +++ b/packages/cli/src/context/llm/debug-request-recorder.ts @@ -1,12 +1,12 @@ import { appendFile, mkdir } from 'node:fs/promises'; import { dirname } from 'node:path'; import type { ModelMessage } from 'ai'; -import type { KtxModelRole } from '@ktx/llm'; +import type { KtxModelRole } from '../../llm/types.js'; type ProviderOptionsCarrier = { providerOptions?: unknown; [key: string]: unknown }; type ToolMap = Record; -export interface KtxLlmDebugProviderOptionsEntry { +interface KtxLlmDebugProviderOptionsEntry { target: 'message' | 'message-part' | 'tool'; index?: number; role?: string; @@ -121,6 +121,7 @@ export function summarizeKtxLlmDebugRequest(input: SummarizeKtxLlmDebugRequestIn }; } +/** @internal */ export function createJsonlKtxLlmDebugRequestRecorder(filePath: string): KtxLlmDebugRequestRecorder { return { async record(request) { diff --git a/packages/context/src/llm/embedding-port.test.ts b/packages/cli/src/context/llm/embedding-port.test.ts similarity index 87% rename from packages/context/src/llm/embedding-port.test.ts rename to packages/cli/src/context/llm/embedding-port.test.ts index 129ef4d9..323e7c5b 100644 --- a/packages/context/src/llm/embedding-port.test.ts +++ b/packages/cli/src/context/llm/embedding-port.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it, vi } from 'vitest'; import { KtxIngestEmbeddingPortAdapter, KtxScanEmbeddingPortAdapter } from './embedding-port.js'; describe('KTX embedding port adapters', () => { - it('adapts @ktx/llm embeddings to ingest embedding port shape', async () => { + it('adapts LLM modules embeddings to ingest embedding port shape', async () => { const provider = { dimensions: 3, maxBatchSize: 2, @@ -22,7 +22,7 @@ describe('KTX embedding port adapters', () => { expect(adapter.maxBatchSize).toBe(2); }); - it('adapts @ktx/llm embeddings to scan embedding port shape', async () => { + it('adapts LLM modules embeddings to scan embedding port shape', async () => { const provider = { dimensions: 3, maxBatchSize: 2, diff --git a/packages/context/src/llm/embedding-port.ts b/packages/cli/src/context/llm/embedding-port.ts similarity index 95% rename from packages/context/src/llm/embedding-port.ts rename to packages/cli/src/context/llm/embedding-port.ts index 6eb0ad80..e4501053 100644 --- a/packages/context/src/llm/embedding-port.ts +++ b/packages/cli/src/context/llm/embedding-port.ts @@ -1,4 +1,4 @@ -import type { KtxEmbeddingProvider } from '@ktx/llm'; +import type { KtxEmbeddingProvider } from '../../llm/types.js'; import type { KtxEmbeddingPort as KtxIngestEmbeddingPort } from '../core/embedding.js'; import type { KtxEmbeddingPort as KtxScanEmbeddingPort } from '../scan/types.js'; diff --git a/packages/context/src/llm/local-config.test.ts b/packages/cli/src/context/llm/local-config.test.ts similarity index 97% rename from packages/context/src/llm/local-config.test.ts rename to packages/cli/src/context/llm/local-config.test.ts index cfa2e826..930ee8a5 100644 --- a/packages/context/src/llm/local-config.test.ts +++ b/packages/cli/src/context/llm/local-config.test.ts @@ -90,7 +90,7 @@ describe('local KTX LLM config', () => { ).toBeNull(); }); - it('constructs providers through @ktx/llm', () => { + it('constructs providers through LLM modules', () => { const createKtxLlmProvider = vi.fn(() => ({ getModel: vi.fn() }) as never); const result = createLocalKtxLlmProviderFromConfig( { @@ -112,7 +112,7 @@ describe('local KTX LLM config', () => { }); }); - it('inherits enabled prompt caching from @ktx/llm when local config omits promptCaching', () => { + it('inherits enabled prompt caching from LLM modules when local config omits promptCaching', () => { const provider = createLocalKtxLlmProviderFromConfig({ provider: { backend: 'gateway', diff --git a/packages/context/src/llm/local-config.ts b/packages/cli/src/context/llm/local-config.ts similarity index 95% rename from packages/context/src/llm/local-config.ts rename to packages/cli/src/context/llm/local-config.ts index 56997356..c64a85cf 100644 --- a/packages/context/src/llm/local-config.ts +++ b/packages/cli/src/context/llm/local-config.ts @@ -1,12 +1,6 @@ -import { - createKtxEmbeddingProvider, - createKtxLlmProvider, - type KtxEmbeddingConfig, - type KtxEmbeddingProvider, - type KtxLlmConfig, - type KtxLlmProvider, - type KtxModelRole, -} from '@ktx/llm'; +import { createKtxEmbeddingProvider } from '../../llm/embedding-provider.js'; +import { createKtxLlmProvider } from '../../llm/model-provider.js'; +import type { KtxEmbeddingConfig, KtxEmbeddingProvider, KtxLlmConfig, KtxLlmProvider, KtxModelRole } from '../../llm/types.js'; import { resolveKtxConfigReference } from '../core/config-reference.js'; import type { KtxProjectEmbeddingConfig, KtxProjectLlmConfig } from '../project/config.js'; import { AiSdkKtxLlmRuntime } from './ai-sdk-runtime.js'; @@ -104,6 +98,7 @@ export function resolveLocalKtxLlmConfig(config: KtxProjectLlmConfig, env: NodeJ }; } +/** @internal */ export function createLocalKtxLlmProviderFromConfig( config: KtxProjectLlmConfig, deps: LocalConfigDeps = {}, @@ -177,6 +172,7 @@ export function resolveLocalKtxEmbeddingConfig( throw new Error(`Unsupported KTX embedding backend: ${String((config as { backend?: string }).backend)}`); } +/** @internal */ export function createLocalKtxEmbeddingProviderFromConfig( config: KtxProjectEmbeddingConfig, deps: LocalConfigDeps = {}, diff --git a/packages/context/src/llm/runtime-local-config.test.ts b/packages/cli/src/context/llm/runtime-local-config.test.ts similarity index 100% rename from packages/context/src/llm/runtime-local-config.test.ts rename to packages/cli/src/context/llm/runtime-local-config.test.ts diff --git a/packages/context/src/llm/runtime-port.ts b/packages/cli/src/context/llm/runtime-port.ts similarity index 96% rename from packages/context/src/llm/runtime-port.ts rename to packages/cli/src/context/llm/runtime-port.ts index 6fddfd80..c1f5ca10 100644 --- a/packages/context/src/llm/runtime-port.ts +++ b/packages/cli/src/context/llm/runtime-port.ts @@ -1,4 +1,4 @@ -import type { KtxModelRole } from '@ktx/llm'; +import type { KtxModelRole } from '../../llm/types.js'; import type { z } from 'zod'; export interface KtxRuntimeToolOutput { @@ -17,6 +17,7 @@ export type KtxRuntimeToolSet = Record; export type RunLoopStopReason = 'budget' | 'natural' | 'error'; +/** @internal */ export interface RunLoopStepInfo { stepIndex: number; stepBudget: number; diff --git a/packages/context/src/llm/runtime-tools.test.ts b/packages/cli/src/context/llm/runtime-tools.test.ts similarity index 100% rename from packages/context/src/llm/runtime-tools.test.ts rename to packages/cli/src/context/llm/runtime-tools.test.ts diff --git a/packages/context/src/llm/runtime-tools.ts b/packages/cli/src/context/llm/runtime-tools.ts similarity index 92% rename from packages/context/src/llm/runtime-tools.ts rename to packages/cli/src/context/llm/runtime-tools.ts index ab2e088d..4a52bdd1 100644 --- a/packages/context/src/llm/runtime-tools.ts +++ b/packages/cli/src/context/llm/runtime-tools.ts @@ -83,9 +83,3 @@ export function createRuntimeToolDescriptorFromAiTool(name: string, aiSdkTool: T }, }; } - -export function createRuntimeToolSetFromAiSdkTools(tools: ToolSet = {}): KtxRuntimeToolSet { - return Object.fromEntries( - Object.entries(tools).map(([name, aiSdkTool]) => [name, createRuntimeToolDescriptorFromAiTool(name, aiSdkTool as Tool)]), - ); -} diff --git a/packages/context/src/mcp/__snapshots__/mcp-tools-list.json b/packages/cli/src/context/mcp/__snapshots__/mcp-tools-list.json similarity index 100% rename from packages/context/src/mcp/__snapshots__/mcp-tools-list.json rename to packages/cli/src/context/mcp/__snapshots__/mcp-tools-list.json diff --git a/packages/context/src/mcp/context-tools.ts b/packages/cli/src/context/mcp/context-tools.ts similarity index 99% rename from packages/context/src/mcp/context-tools.ts rename to packages/cli/src/context/mcp/context-tools.ts index e040df87..6778bb64 100644 --- a/packages/context/src/mcp/context-tools.ts +++ b/packages/cli/src/context/mcp/context-tools.ts @@ -1,7 +1,7 @@ import { randomUUID } from 'node:crypto'; import type { ToolAnnotations } from '@modelcontextprotocol/sdk/types.js'; import { z } from 'zod'; -import type { MemoryAgentInput } from '../memory/index.js'; +import type { MemoryAgentInput } from '../../context/memory/types.js'; import type { KtxMcpContextPorts, KtxMcpProgressCallback, @@ -445,6 +445,7 @@ const memoryIngestStatusOutputSchema = z.object({ signalDetected: z.boolean(), }); +/** @internal */ export function jsonToolResult(structuredContent: T): KtxMcpToolResult { return { content: [{ type: 'text', text: JSON.stringify(structuredContent, null, 2) }], @@ -452,7 +453,7 @@ export function jsonToolResult(structuredContent: T): }; } -export function jsonErrorToolResult(text: string): KtxMcpToolResult> { +function jsonErrorToolResult(text: string): KtxMcpToolResult> { return { content: [{ type: 'text', text }], isError: true, diff --git a/packages/context/src/mcp/local-project-ports.test.ts b/packages/cli/src/context/mcp/local-project-ports.test.ts similarity index 98% rename from packages/context/src/mcp/local-project-ports.test.ts rename to packages/cli/src/context/mcp/local-project-ports.test.ts index b9ca1fc0..aa06b47e 100644 --- a/packages/context/src/mcp/local-project-ports.test.ts +++ b/packages/cli/src/context/mcp/local-project-ports.test.ts @@ -2,14 +2,9 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { initKtxProject } from '../project/index.js'; -import { - createKtxConnectorCapabilities, - type KtxQueryResult, - type KtxScanConnector, - type KtxSchemaSnapshot, -} from '../scan/index.js'; -import { writeLocalSlSource } from '../sl/index.js'; +import { initKtxProject } from '../../context/project/project.js'; +import { createKtxConnectorCapabilities, type KtxQueryResult, type KtxScanConnector, type KtxSchemaSnapshot } from '../../context/scan/types.js'; +import { writeLocalSlSource } from '../../context/sl/local-sl.js'; import { createLocalProjectMcpContextPorts } from './local-project-ports.js'; describe('createLocalProjectMcpContextPorts', () => { diff --git a/packages/context/src/mcp/local-project-ports.ts b/packages/cli/src/context/mcp/local-project-ports.ts similarity index 88% rename from packages/context/src/mcp/local-project-ports.ts rename to packages/cli/src/context/mcp/local-project-ports.ts index bf40dc80..dbbb36d2 100644 --- a/packages/context/src/mcp/local-project-ports.ts +++ b/packages/cli/src/context/mcp/local-project-ports.ts @@ -1,11 +1,15 @@ -import { type KtxSqlQueryExecutorPort, localConnectionInfoFromConfig } from '../connections/index.js'; -import type { KtxEmbeddingPort } from '../core/index.js'; -import type { KtxSemanticLayerComputePort } from '../daemon/index.js'; -import type { KtxLocalProject } from '../project/index.js'; -import { createKtxEntityDetailsService, type KtxScanConnector, type LocalScanMcpOptions } from '../scan/index.js'; -import { createKtxDiscoverDataService } from '../search/index.js'; -import type { SqlAnalysisDialect, SqlAnalysisPort } from '../sql-analysis/index.js'; -import { compileLocalSlQuery, createKtxDictionarySearchService } from '../sl/index.js'; +import type { KtxSqlQueryExecutorPort } from '../../context/connections/query-executor.js'; +import { localConnectionInfoFromConfig } from '../../context/connections/local-warehouse-descriptor.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { createKtxEntityDetailsService } from '../../context/scan/entity-details.js'; +import type { KtxScanConnector } from '../../context/scan/types.js'; +import type { LocalScanMcpOptions } from '../../context/scan/local-scan.js'; +import { createKtxDiscoverDataService } from '../../context/search/discover.js'; +import type { SqlAnalysisDialect, SqlAnalysisPort } from '../../context/sql-analysis/ports.js'; +import { compileLocalSlQuery } from '../../context/sl/local-query.js'; +import { createKtxDictionarySearchService } from '../../context/sl/dictionary-search.js'; import { readLocalKnowledgePage, searchLocalKnowledgePages } from '../wiki/local-knowledge.js'; import type { KtxMcpContextPorts, KtxMcpProgressCallback, KtxSqlExecutionResponse } from './types.js'; diff --git a/packages/context/src/mcp/server.test.ts b/packages/cli/src/context/mcp/server.test.ts similarity index 99% rename from packages/context/src/mcp/server.test.ts rename to packages/cli/src/context/mcp/server.test.ts index f02b4adb..3532a327 100644 --- a/packages/context/src/mcp/server.test.ts +++ b/packages/cli/src/context/mcp/server.test.ts @@ -4,12 +4,10 @@ import { join } from 'node:path'; import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; import { describe, expect, it, vi } from 'vitest'; -import { - createLocalProjectMemoryIngest, - detectCaptureSignals, - type MemoryAgentInput, -} from '../memory/index.js'; -import { initKtxProject } from '../project/index.js'; +import { createLocalProjectMemoryIngest } from '../../context/memory/local-memory.js'; +import { detectCaptureSignals } from '../../context/memory/capture-signals.js'; +import type { MemoryAgentInput } from '../../context/memory/types.js'; +import { initKtxProject } from '../../context/project/project.js'; import { jsonToolResult } from './context-tools.js'; import { createDefaultKtxMcpServer, createKtxMcpServer } from './server.js'; import type { diff --git a/packages/context/src/mcp/server.ts b/packages/cli/src/context/mcp/server.ts similarity index 93% rename from packages/context/src/mcp/server.ts rename to packages/cli/src/context/mcp/server.ts index 8931650f..73f970d6 100644 --- a/packages/context/src/mcp/server.ts +++ b/packages/cli/src/context/mcp/server.ts @@ -2,6 +2,7 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { registerKtxContextTools } from './context-tools.js'; import type { KtxMcpServerDeps, KtxMcpServerLike } from './types.js'; +/** @internal */ export function createKtxMcpServer(deps: KtxMcpServerDeps): KtxMcpServerDeps['server'] { if (deps.contextTools) { registerKtxContextTools({ @@ -15,11 +16,11 @@ export function createKtxMcpServer(deps: KtxMcpServerDeps): KtxMcpServerDeps['se } export function createDefaultKtxMcpServer( - deps: Omit & { name?: string; version?: string }, + deps: Omit & { name?: string; version: string }, ): McpServer { const server = new McpServer({ name: deps.name ?? 'ktx', - version: deps.version ?? '0.0.0-private', + version: deps.version, }); createKtxMcpServer({ server: server as KtxMcpServerLike, diff --git a/packages/context/src/mcp/types.ts b/packages/cli/src/context/mcp/types.ts similarity index 85% rename from packages/context/src/mcp/types.ts rename to packages/cli/src/context/mcp/types.ts index 49bf7b55..223aa394 100644 --- a/packages/context/src/mcp/types.ts +++ b/packages/cli/src/context/mcp/types.ts @@ -1,10 +1,11 @@ -import type { MemoryIngestService } from '../memory/index.js'; +import type { MemoryIngestService } from '../../context/memory/memory-runs.js'; import type { KtxEntityDetailsInput, KtxEntityDetailsResponse } from '../scan/entity-details.js'; -import type { KtxDiscoverDataInput, KtxDiscoverDataResponse } from '../search/index.js'; -import type { KtxDictionarySearchInput, KtxDictionarySearchResponse, SemanticLayerQueryInput } from '../sl/index.js'; -import type { WikiSearchLaneSummary, WikiSearchMatchReason } from '../wiki/index.js'; +import type { KtxDiscoverDataInput, KtxDiscoverDataResponse } from '../../context/search/discover.js'; +import type { KtxDictionarySearchInput, KtxDictionarySearchResponse } from '../../context/sl/dictionary-search.js'; +import type { SemanticLayerQueryInput } from '../../context/sl/types.js'; +import type { WikiSearchLaneSummary, WikiSearchMatchReason } from '../../context/wiki/types.js'; -export interface KtxMcpTextContent { +interface KtxMcpTextContent { type: 'text'; text: string; } @@ -38,6 +39,7 @@ export interface KtxMcpToolHandlerContext { }) => Promise; } +/** @internal */ export interface MemoryIngestPort { ingest: MemoryIngestService['ingest']; status: MemoryIngestService['status']; @@ -61,17 +63,17 @@ export interface KtxMcpServerLike { ): void; } -export interface KtxConnectionSummary { +interface KtxConnectionSummary { id: string; name: string; connectionType: string; } -export interface KtxConnectionsMcpPort { +interface KtxConnectionsMcpPort { list(): Promise; } -export interface KtxKnowledgeSearchResult { +interface KtxKnowledgeSearchResult { key: string; path: string; scope: 'GLOBAL' | 'USER'; @@ -81,12 +83,12 @@ export interface KtxKnowledgeSearchResult { lanes?: WikiSearchLaneSummary[]; } -export interface KtxKnowledgeSearchResponse { +interface KtxKnowledgeSearchResponse { results: KtxKnowledgeSearchResult[]; totalFound: number; } -export interface KtxKnowledgePage { +interface KtxKnowledgePage { key: string; summary: string; content: string; @@ -96,17 +98,18 @@ export interface KtxKnowledgePage { slRefs?: string[]; } +/** @internal */ export interface KtxKnowledgeMcpPort { search(input: { userId: string; query: string; limit: number }): Promise; read(input: { userId: string; key: string }): Promise; } -export interface KtxSemanticLayerReadResponse { +interface KtxSemanticLayerReadResponse { sourceName: string; yaml: string; } -export interface KtxSemanticLayerQueryResponse { +interface KtxSemanticLayerQueryResponse { sql: string; headers: string[]; rows: unknown[][]; @@ -114,6 +117,7 @@ export interface KtxSemanticLayerQueryResponse { plan?: Record; } +/** @internal */ export interface KtxSemanticLayerMcpPort { readSource(input: { connectionId: string; sourceName: string }): Promise; query( @@ -122,14 +126,17 @@ export interface KtxSemanticLayerMcpPort { ): Promise; } +/** @internal */ export interface KtxEntityDetailsMcpPort { read(input: KtxEntityDetailsInput): Promise; } +/** @internal */ export interface KtxDictionarySearchMcpPort { search(input: KtxDictionarySearchInput): Promise; } +/** @internal */ export interface KtxDiscoverDataMcpPort { search(input: KtxDiscoverDataInput): Promise; } @@ -141,6 +148,7 @@ export interface KtxSqlExecutionResponse { rowCount: number; } +/** @internal */ export interface KtxSqlExecutionMcpPort { execute( input: { connectionId: string; sql: string; maxRows: number }, diff --git a/packages/context/src/memory/capture-signals.ts b/packages/cli/src/context/memory/capture-signals.ts similarity index 99% rename from packages/context/src/memory/capture-signals.ts rename to packages/cli/src/context/memory/capture-signals.ts index 360f0b7c..86a123e0 100644 --- a/packages/context/src/memory/capture-signals.ts +++ b/packages/cli/src/context/memory/capture-signals.ts @@ -103,6 +103,7 @@ export function prefilterSkipReason(input: MemoryAgentInput, signals = detectCap return null; } +/** @internal */ export function isWorthAnalyzing(input: MemoryAgentInput): boolean { return prefilterSkipReason(input, detectCaptureSignals(input)) === null; } diff --git a/packages/context/src/memory/local-memory-runs.ts b/packages/cli/src/context/memory/local-memory-runs.ts similarity index 100% rename from packages/context/src/memory/local-memory-runs.ts rename to packages/cli/src/context/memory/local-memory-runs.ts diff --git a/packages/context/src/memory/local-memory.test.ts b/packages/cli/src/context/memory/local-memory.test.ts similarity index 99% rename from packages/context/src/memory/local-memory.test.ts rename to packages/cli/src/context/memory/local-memory.test.ts index 04aabce1..1a7240c9 100644 --- a/packages/context/src/memory/local-memory.test.ts +++ b/packages/cli/src/context/memory/local-memory.test.ts @@ -2,7 +2,7 @@ import { access, mkdtemp, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { initKtxProject } from '../project/index.js'; +import { initKtxProject } from '../../context/project/project.js'; import { createLocalProjectMemoryIngest } from './local-memory.js'; import { LocalMemoryRunStore } from './local-memory-runs.js'; diff --git a/packages/context/src/memory/local-memory.ts b/packages/cli/src/context/memory/local-memory.ts similarity index 83% rename from packages/context/src/memory/local-memory.ts rename to packages/cli/src/context/memory/local-memory.ts index 7ac4fe90..b72bc9ce 100644 --- a/packages/context/src/memory/local-memory.ts +++ b/packages/cli/src/context/memory/local-memory.ts @@ -1,56 +1,43 @@ import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import YAML from 'yaml'; -import { localConnectionInfoFromConfig } from '../connections/index.js'; -import type { KtxEmbeddingPort, KtxFileStorePort, KtxFileWriteResult } from '../core/index.js'; -import { type KtxLogger, noopLogger, SessionWorktreeService } from '../core/index.js'; -import type { KtxSemanticLayerComputePort } from '../daemon/index.js'; -import type { KtxEmbeddingProvider } from '@ktx/llm'; -import { - createLocalKtxLlmRuntimeFromConfig, - KtxIngestEmbeddingPortAdapter, - RuntimeAgentRunner, - type AgentRunnerPort, - type KtxLlmRuntimePort, - type KtxRuntimeToolSet, -} from '../llm/index.js'; -import type { KtxLocalProject } from '../project/index.js'; -import { PromptService } from '../prompts/index.js'; -import { SkillsRegistryService } from '../skills/index.js'; -import { - type KtxConnectionInfo, - type KtxQueryResult, - SemanticLayerService, - type SemanticLayerSource, - type SlConnectionCatalogPort, - SlDiscoverTool, - SlEditSourceTool, - type SlPythonPort, - SlReadSourceTool, - SlRollbackTool, - SlSearchService, - type SlSourcesIndexPort, - SlValidateTool, - type SlValidationDeps, - type SlValidatorPort, - SlWriteSourceTool, - SqliteSlSourcesIndex, - sourceDefinitionSchema, - sourceOverlaySchema, -} from '../sl/index.js'; -import { BaseTool, type GitAuthorResolverPort, type ToolContext } from '../tools/index.js'; -import { - type KnowledgeEventPort, - type KnowledgeIndexPort, - type KnowledgeIndexPageListing, - KnowledgeWikiService, - searchLocalKnowledgePages, - WikiListTagsTool, - WikiReadTool, - WikiRemoveTool, - WikiSearchTool, - WikiWriteTool, -} from '../wiki/index.js'; +import { localConnectionInfoFromConfig } from '../../context/connections/local-warehouse-descriptor.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxFileStorePort, KtxFileWriteResult } from '../../context/core/file-store.js'; +import { type KtxLogger, noopLogger } from '../../context/core/config.js'; +import { SessionWorktreeService } from '../../context/core/session-worktree.service.js'; +import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js'; +import { KtxIngestEmbeddingPortAdapter } from '../../context/llm/embedding-port.js'; +import { createLocalKtxLlmRuntimeFromConfig } from '../../context/llm/local-config.js'; +import { RuntimeAgentRunner, type AgentRunnerPort, type KtxLlmRuntimePort, type KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; +import type { KtxEmbeddingProvider } from '../../llm/types.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { PromptService } from '../../context/prompts/prompt.service.js'; +import { SkillsRegistryService } from '../../context/skills/skills-registry.service.js'; +import type { KtxConnectionInfo, KtxQueryResult, SlConnectionCatalogPort, SlPythonPort, SlSourcesIndexPort } from '../../context/sl/ports.js'; +import { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import type { SemanticLayerSource } from '../../context/sl/types.js'; +import { SlDiscoverTool } from '../../context/sl/tools/sl-discover.tool.js'; +import { SlEditSourceTool } from '../../context/sl/tools/sl-edit-source.tool.js'; +import { SlReadSourceTool } from '../../context/sl/tools/sl-read-source.tool.js'; +import { SlRollbackTool } from '../../context/sl/tools/sl-rollback.tool.js'; +import { SlSearchService } from '../../context/sl/sl-search.service.js'; +import { SlValidateTool } from '../../context/sl/tools/sl-validate.tool.js'; +import type { SlValidationDeps } from '../../context/sl/tools/sl-warehouse-validation.js'; +import type { SlValidatorPort } from '../../context/sl/sl-validator.port.js'; +import { SlWriteSourceTool } from '../../context/sl/tools/sl-write-source.tool.js'; +import { SqliteSlSourcesIndex } from '../../context/sl/sqlite-sl-sources-index.js'; +import { sourceDefinitionSchema, sourceOverlaySchema } from '../../context/sl/schemas.js'; +import { BaseTool, type ToolContext } from '../../context/tools/base-tool.js'; +import type { GitAuthorResolverPort } from '../../context/tools/authors.js'; +import type { KnowledgeEventPort, KnowledgeIndexPort, KnowledgeIndexPageListing } from '../../context/wiki/ports.js'; +import { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; +import { searchLocalKnowledgePages } from '../../context/wiki/local-knowledge.js'; +import { WikiListTagsTool } from '../../context/wiki/tools/wiki-list-tags.tool.js'; +import { WikiReadTool } from '../../context/wiki/tools/wiki-read.tool.js'; +import { WikiRemoveTool } from '../../context/wiki/tools/wiki-remove.tool.js'; +import { WikiSearchTool } from '../../context/wiki/tools/wiki-search.tool.js'; +import { WikiWriteTool } from '../../context/wiki/tools/wiki-write.tool.js'; import { LocalMemoryRunStore } from './local-memory-runs.js'; import { MemoryAgentService } from './memory-agent.service.js'; import { MemoryIngestService } from './memory-runs.js'; diff --git a/packages/context/src/memory/memory-agent.service.ingest.test.ts b/packages/cli/src/context/memory/memory-agent.service.ingest.test.ts similarity index 99% rename from packages/context/src/memory/memory-agent.service.ingest.test.ts rename to packages/cli/src/context/memory/memory-agent.service.ingest.test.ts index e9985c20..1c13bdd2 100644 --- a/packages/context/src/memory/memory-agent.service.ingest.test.ts +++ b/packages/cli/src/context/memory/memory-agent.service.ingest.test.ts @@ -13,7 +13,7 @@ vi.mock('ai', () => ({ // Imported AFTER vi.mock so the mocked module is used. import { generateText } from 'ai'; -import { SYSTEM_GIT_AUTHOR } from '../tools/index.js'; +import { SYSTEM_GIT_AUTHOR } from '../../context/tools/authors.js'; import { MemoryAgentService } from './memory-agent.service.js'; interface BuiltMocks { diff --git a/packages/context/src/memory/memory-agent.service.test.ts b/packages/cli/src/context/memory/memory-agent.service.test.ts similarity index 98% rename from packages/context/src/memory/memory-agent.service.test.ts rename to packages/cli/src/context/memory/memory-agent.service.test.ts index 63f97e14..cea83674 100644 --- a/packages/context/src/memory/memory-agent.service.test.ts +++ b/packages/cli/src/context/memory/memory-agent.service.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { validateSingleSource } from '../sl/index.js'; -import { createTouchedSlSources, hasTouchedSlSource } from '../tools/index.js'; +import { validateSingleSource } from '../../context/sl/tools/sl-warehouse-validation.js'; +import { createTouchedSlSources, hasTouchedSlSource } from '../../context/tools/touched-sl-sources.js'; import { detectCaptureSignals, isWorthAnalyzing } from './capture-signals.js'; import { MemoryAgentService } from './memory-agent.service.js'; diff --git a/packages/context/src/memory/memory-agent.service.ts b/packages/cli/src/context/memory/memory-agent.service.ts similarity index 96% rename from packages/context/src/memory/memory-agent.service.ts rename to packages/cli/src/context/memory/memory-agent.service.ts index e29ad2f1..7f6438d3 100644 --- a/packages/context/src/memory/memory-agent.service.ts +++ b/packages/cli/src/context/memory/memory-agent.service.ts @@ -3,24 +3,15 @@ import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import * as YAML from 'yaml'; import { z } from 'zod'; -import { type KtxLogger, noopLogger } from '../core/index.js'; -import type { KtxRuntimeToolSet } from '../llm/index.js'; -import { - revertSourceToPreHead, - type SemanticLayerSource, - type SlValidationDeps, - type SlValidatorPort, -} from '../sl/index.js'; -import { - createTouchedSlSources, - deleteTouchedSlSource, - listTouchedSlSources, - SYSTEM_GIT_AUTHOR, - type ToolContext, - type ToolSession, - touchedSlSourceCount, - touchedSlSourceNamesForConnection, -} from '../tools/index.js'; +import { type KtxLogger, noopLogger } from '../../context/core/config.js'; +import type { KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; +import { revertSourceToPreHead, type SlValidationDeps } from '../../context/sl/tools/sl-warehouse-validation.js'; +import type { SemanticLayerSource } from '../../context/sl/types.js'; +import type { SlValidatorPort } from '../../context/sl/sl-validator.port.js'; +import { createTouchedSlSources, deleteTouchedSlSource, listTouchedSlSources, touchedSlSourceCount, touchedSlSourceNamesForConnection } from '../../context/tools/touched-sl-sources.js'; +import { SYSTEM_GIT_AUTHOR } from '../../context/tools/authors.js'; +import type { ToolContext } from '../../context/tools/base-tool.js'; +import type { ToolSession } from '../../context/tools/tool-session.js'; import { buildRequiredSkillsBlock, DEFAULT_SKILL_NAMES, diff --git a/packages/context/src/memory/memory-runs.test.ts b/packages/cli/src/context/memory/memory-runs.test.ts similarity index 96% rename from packages/context/src/memory/memory-runs.test.ts rename to packages/cli/src/context/memory/memory-runs.test.ts index 9fd9c36d..e4d0d4ba 100644 --- a/packages/context/src/memory/memory-runs.test.ts +++ b/packages/cli/src/context/memory/memory-runs.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import type { MemoryAgentInput, MemoryAgentResult, MemoryAgentService } from './index.js'; +import type { MemoryAgentInput, MemoryAgentResult } from '../../context/memory/types.js'; +import type { MemoryAgentService } from '../../context/memory/memory-agent.service.js'; import { MemoryIngestService, type MemoryRunStorePort } from './memory-runs.js'; class InMemoryRunStore implements MemoryRunStorePort { diff --git a/packages/context/src/memory/memory-runs.ts b/packages/cli/src/context/memory/memory-runs.ts similarity index 95% rename from packages/context/src/memory/memory-runs.ts rename to packages/cli/src/context/memory/memory-runs.ts index 2b34657e..9bf5dedf 100644 --- a/packages/context/src/memory/memory-runs.ts +++ b/packages/cli/src/context/memory/memory-runs.ts @@ -1,5 +1,6 @@ import { createHash } from 'node:crypto'; -import type { MemoryAction, MemoryAgentInput, MemoryAgentResult, MemoryAgentService } from './index.js'; +import type { MemoryAction, MemoryAgentInput, MemoryAgentResult } from '../../context/memory/types.js'; +import type { MemoryAgentService } from '../../context/memory/memory-agent.service.js'; export type MemoryRunStatus = 'running' | 'done' | 'error'; diff --git a/packages/context/src/memory/memory-runtime-assets.test.ts b/packages/cli/src/context/memory/memory-runtime-assets.test.ts similarity index 96% rename from packages/context/src/memory/memory-runtime-assets.test.ts rename to packages/cli/src/context/memory/memory-runtime-assets.test.ts index 68a53cfd..55d9047c 100644 --- a/packages/context/src/memory/memory-runtime-assets.test.ts +++ b/packages/cli/src/context/memory/memory-runtime-assets.test.ts @@ -2,9 +2,10 @@ import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; -import { PromptService } from '../prompts/index.js'; -import { SkillsRegistryService } from '../skills/index.js'; -import { DEFAULT_SKILL_NAMES, type MemoryAgentSourceType, promptNameFor } from './index.js'; +import { PromptService } from '../../context/prompts/prompt.service.js'; +import { SkillsRegistryService } from '../../context/skills/skills-registry.service.js'; +import { DEFAULT_SKILL_NAMES, promptNameFor } from '../../context/memory/capture-signals.js'; +import type { MemoryAgentSourceType } from '../../context/memory/types.js'; const promptsDir = fileURLToPath(new URL('../../prompts', import.meta.url)); const skillsDir = fileURLToPath(new URL('../../skills', import.meta.url)); diff --git a/packages/context/src/memory/types.ts b/packages/cli/src/context/memory/types.ts similarity index 70% rename from packages/context/src/memory/types.ts rename to packages/cli/src/context/memory/types.ts index bb2f3bad..5cc7dab2 100644 --- a/packages/context/src/memory/types.ts +++ b/packages/cli/src/context/memory/types.ts @@ -1,19 +1,21 @@ -import type { AgentRunnerPort, KtxRuntimeToolSet } from '../llm/index.js'; -import type { GitService, KtxFileStorePort, KtxLogger, SessionWorktreeService } from '../core/index.js'; -import type { PromptService } from '../prompts/index.js'; -import type { SkillsRegistryService } from '../skills/index.js'; -import type { - KtxConnectionInfo, - KtxQueryResult, - SemanticLayerService, - SemanticLayerSource, - SlSearchService, - SlSourcesIndexPort, - SlValidationDeps, - SlValidatorPort, -} from '../sl/index.js'; -import type { ToolContext, ToolSession, TouchedSlSourceSet } from '../tools/index.js'; -import type { KnowledgeIndexPort, KnowledgeWikiService } from '../wiki/index.js'; +import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; +import type { GitService } from '../../context/core/git.service.js'; +import type { KtxFileStorePort } from '../../context/core/file-store.js'; +import type { KtxLogger } from '../../context/core/config.js'; +import type { SessionWorktreeService } from '../../context/core/session-worktree.service.js'; +import type { PromptService } from '../../context/prompts/prompt.service.js'; +import type { SkillsRegistryService } from '../../context/skills/skills-registry.service.js'; +import type { KtxConnectionInfo, KtxQueryResult, SlSourcesIndexPort } from '../../context/sl/ports.js'; +import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import type { SemanticLayerSource } from '../../context/sl/types.js'; +import type { SlSearchService } from '../../context/sl/sl-search.service.js'; +import type { SlValidationDeps } from '../../context/sl/tools/sl-warehouse-validation.js'; +import type { SlValidatorPort } from '../../context/sl/sl-validator.port.js'; +import type { ToolContext } from '../../context/tools/base-tool.js'; +import type { ToolSession } from '../../context/tools/tool-session.js'; +import type { TouchedSlSourceSet } from '../../context/tools/touched-sl-sources.js'; +import type { KnowledgeIndexPort } from '../../context/wiki/ports.js'; +import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; export type MemoryAgentSourceType = 'research' | 'external_ingest' | 'backfill'; @@ -62,7 +64,7 @@ export interface CaptureSession { preHead: string | null; } -export interface MemoryAgentSettings { +interface MemoryAgentSettings { knowledge: { userScopedKnowledgeEnabled: boolean; }; @@ -74,7 +76,7 @@ export interface MemoryAgentSettings { }; } -export interface MemoryTelemetryPort { +interface MemoryTelemetryPort { trackMemoryIngestion( userId: string, properties: { @@ -106,7 +108,7 @@ export interface MemoryConnectionPort { executeQuery(connectionId: string, sql: string): Promise; } -export interface MemoryCommitMessagePort { +interface MemoryCommitMessagePort { enqueueCommitMessageJobForExternalCommit( commit: { commitHash: string }, message: string, @@ -129,7 +131,7 @@ export interface MemorySlSourceReconcilerPort { upsertRow(parsed: SemanticLayerSource, path: string, contentHash: string): Promise; } -export interface MemoryLockPort { +interface MemoryLockPort { withLock(key: 'config:repo', fn: () => Promise): Promise; } diff --git a/packages/context/src/project/config.test.ts b/packages/cli/src/context/project/config.test.ts similarity index 100% rename from packages/context/src/project/config.test.ts rename to packages/cli/src/context/project/config.test.ts diff --git a/packages/context/src/project/config.ts b/packages/cli/src/context/project/config.ts similarity index 97% rename from packages/context/src/project/config.ts rename to packages/cli/src/context/project/config.ts index 18676ef5..2824ca59 100644 --- a/packages/context/src/project/config.ts +++ b/packages/cli/src/context/project/config.ts @@ -1,4 +1,4 @@ -import { KTX_MODEL_ROLES } from '@ktx/llm'; +import { KTX_MODEL_ROLES } from '../../llm/types.js'; import YAML from 'yaml'; import * as z from 'zod'; import { connectionConfigSchema } from './driver-schemas.js'; @@ -264,16 +264,10 @@ const ktxProjectConfigSchema = z export type KtxProjectConfig = z.infer; export type KtxProjectLlmConfig = z.infer; -export type KtxProjectLlmProviderConfig = z.infer; export type KtxProjectEmbeddingConfig = z.infer; export type KtxScanEnrichmentConfig = z.infer; -export type KtxIngestWorkUnitsConfig = z.infer; export type KtxScanRelationshipConfig = z.infer; -export type KtxProjectScanConfig = z.infer; export type KtxProjectConnectionConfig = z.infer; -export type KtxProjectSetupConfig = z.infer; -export type KtxStorageState = z.infer['state']; -export type KtxSearchBackend = z.infer['search']; export interface KtxConfigIssue { path: string; diff --git a/packages/context/src/project/driver-schemas.test.ts b/packages/cli/src/context/project/driver-schemas.test.ts similarity index 100% rename from packages/context/src/project/driver-schemas.test.ts rename to packages/cli/src/context/project/driver-schemas.test.ts diff --git a/packages/context/src/project/driver-schemas.ts b/packages/cli/src/context/project/driver-schemas.ts similarity index 99% rename from packages/context/src/project/driver-schemas.ts rename to packages/cli/src/context/project/driver-schemas.ts index c3b819ea..a3b71bff 100644 --- a/packages/context/src/project/driver-schemas.ts +++ b/packages/cli/src/context/project/driver-schemas.ts @@ -207,5 +207,3 @@ export const connectionConfigSchema = z.discriminatedUnion('driver', [ dbtConnectionSchema, metricflowConnectionSchema, ]); - -export type KtxConnectionConfig = z.infer; diff --git a/packages/context/src/project/local-git-file-store.test.ts b/packages/cli/src/context/project/local-git-file-store.test.ts similarity index 96% rename from packages/context/src/project/local-git-file-store.test.ts rename to packages/cli/src/context/project/local-git-file-store.test.ts index 62b7fc8c..1bee3c1e 100644 --- a/packages/context/src/project/local-git-file-store.test.ts +++ b/packages/cli/src/context/project/local-git-file-store.test.ts @@ -2,7 +2,8 @@ import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { GitService, type KtxCoreConfig } from '../core/index.js'; +import { GitService } from '../../context/core/git.service.js'; +import type { KtxCoreConfig } from '../../context/core/config.js'; import { LocalGitFileStore } from './local-git-file-store.js'; describe('LocalGitFileStore', () => { diff --git a/packages/context/src/project/local-git-file-store.ts b/packages/cli/src/context/project/local-git-file-store.ts similarity index 95% rename from packages/context/src/project/local-git-file-store.ts rename to packages/cli/src/context/project/local-git-file-store.ts index 8d1bd065..51757cb2 100644 --- a/packages/context/src/project/local-git-file-store.ts +++ b/packages/cli/src/context/project/local-git-file-store.ts @@ -1,14 +1,7 @@ import { promises as fs } from 'node:fs'; import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; -import type { - GitCommitInfo, - GitService, - KtxFileHistoryEntry, - KtxFileListResult, - KtxFileReadResult, - KtxFileStorePort, - KtxFileWriteResult, -} from '../core/index.js'; +import type { GitCommitInfo, GitService } from '../../context/core/git.service.js'; +import type { KtxFileHistoryEntry, KtxFileListResult, KtxFileReadResult, KtxFileStorePort, KtxFileWriteResult } from '../../context/core/file-store.js'; export interface LocalGitFileStoreDeps { rootDir: string; diff --git a/packages/context/src/project/local-state-db.ts b/packages/cli/src/context/project/local-state-db.ts similarity index 100% rename from packages/context/src/project/local-state-db.ts rename to packages/cli/src/context/project/local-state-db.ts diff --git a/packages/context/src/project/mappings-yaml-schema.test.ts b/packages/cli/src/context/project/mappings-yaml-schema.test.ts similarity index 100% rename from packages/context/src/project/mappings-yaml-schema.test.ts rename to packages/cli/src/context/project/mappings-yaml-schema.test.ts diff --git a/packages/context/src/project/mappings-yaml-schema.ts b/packages/cli/src/context/project/mappings-yaml-schema.ts similarity index 99% rename from packages/context/src/project/mappings-yaml-schema.ts rename to packages/cli/src/context/project/mappings-yaml-schema.ts index 43ad6515..9c69e1b3 100644 --- a/packages/context/src/project/mappings-yaml-schema.ts +++ b/packages/cli/src/context/project/mappings-yaml-schema.ts @@ -69,6 +69,7 @@ export type LookerMappingBootstrap = { connectionMappings: Record; }; +/** @internal */ export type LookmlMappingBootstrap = { adapter: 'lookml'; connectionId: string; @@ -117,6 +118,7 @@ export function parseMetabaseMappingBootstrap( }; } +/** @internal */ export function parseLookerMappingBootstrap( connectionId: string, connection: MappingConnectionInput, @@ -129,6 +131,7 @@ export function parseLookerMappingBootstrap( }; } +/** @internal */ export function parseLookmlMappingBootstrap( connectionId: string, connection: MappingConnectionInput, diff --git a/packages/context/src/project/project.test.ts b/packages/cli/src/context/project/project.test.ts similarity index 100% rename from packages/context/src/project/project.test.ts rename to packages/cli/src/context/project/project.test.ts diff --git a/packages/context/src/project/project.ts b/packages/cli/src/context/project/project.ts similarity index 96% rename from packages/context/src/project/project.ts rename to packages/cli/src/context/project/project.ts index 572ac325..56fca5b2 100644 --- a/packages/context/src/project/project.ts +++ b/packages/cli/src/context/project/project.ts @@ -1,6 +1,7 @@ import { promises as fs } from 'node:fs'; import { basename, dirname, join, resolve } from 'node:path'; -import { GitService, type KtxCoreConfig, type KtxLogger, noopLogger } from '../core/index.js'; +import { GitService } from '../../context/core/git.service.js'; +import { type KtxCoreConfig, type KtxLogger, noopLogger } from '../../context/core/config.js'; import type { KtxProjectConfig } from './config.js'; import { buildDefaultKtxProjectConfig, parseKtxProjectConfig, serializeKtxProjectConfig } from './config.js'; import { LocalGitFileStore } from './local-git-file-store.js'; diff --git a/packages/context/src/project/setup-config.test.ts b/packages/cli/src/context/project/setup-config.test.ts similarity index 100% rename from packages/context/src/project/setup-config.test.ts rename to packages/cli/src/context/project/setup-config.test.ts diff --git a/packages/context/src/project/setup-config.ts b/packages/cli/src/context/project/setup-config.ts similarity index 96% rename from packages/context/src/project/setup-config.ts rename to packages/cli/src/context/project/setup-config.ts index 3f2d6534..f790597e 100644 --- a/packages/context/src/project/setup-config.ts +++ b/packages/cli/src/context/project/setup-config.ts @@ -2,7 +2,7 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import type { KtxProjectConfig } from './config.js'; -export const KTX_SETUP_STEPS = [ +const KTX_SETUP_STEPS = [ 'project', 'llm', 'embeddings', @@ -40,7 +40,7 @@ function uniqueSetupSteps(steps: unknown): KtxSetupStep[] { return [...new Set(steps.filter(isKtxSetupStep))]; } -export function ktxSetupStatePath(projectDir: string): string { +function ktxSetupStatePath(projectDir: string): string { return join(projectDir, '.ktx', 'setup', 'state.json'); } @@ -56,6 +56,7 @@ export async function readKtxSetupState(projectDir: string): Promise { await mkdir(join(projectDir, '.ktx', 'setup'), { recursive: true }); await writeFile( diff --git a/packages/context/src/prompts/prompt.service.test.ts b/packages/cli/src/context/prompts/prompt.service.test.ts similarity index 100% rename from packages/context/src/prompts/prompt.service.test.ts rename to packages/cli/src/context/prompts/prompt.service.test.ts diff --git a/packages/context/src/prompts/prompt.service.ts b/packages/cli/src/context/prompts/prompt.service.ts similarity index 98% rename from packages/context/src/prompts/prompt.service.ts rename to packages/cli/src/context/prompts/prompt.service.ts index 1864f120..b18fc0b0 100644 --- a/packages/context/src/prompts/prompt.service.ts +++ b/packages/cli/src/context/prompts/prompt.service.ts @@ -1,7 +1,7 @@ import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import Handlebars from 'handlebars'; -import { type KtxLogger, noopLogger } from '../core/index.js'; +import { type KtxLogger, noopLogger } from '../../context/core/config.js'; export interface PromptContext { current_date?: string; diff --git a/packages/context/src/scan/credentials.test.ts b/packages/cli/src/context/scan/credentials.test.ts similarity index 98% rename from packages/context/src/scan/credentials.test.ts rename to packages/cli/src/context/scan/credentials.test.ts index 60b6c2e9..891c58a9 100644 --- a/packages/context/src/scan/credentials.test.ts +++ b/packages/cli/src/context/scan/credentials.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; +import { REDACTED_KTX_CREDENTIAL_VALUE } from '../core/redaction.js'; import { - REDACTED_KTX_CREDENTIAL_VALUE, redactKtxCredentialEnvelope, redactKtxCredentialValue, redactKtxScanMetadata, diff --git a/packages/context/src/scan/credentials.ts b/packages/cli/src/context/scan/credentials.ts similarity index 94% rename from packages/context/src/scan/credentials.ts rename to packages/cli/src/context/scan/credentials.ts index 23fd634e..07ad8e24 100644 --- a/packages/context/src/scan/credentials.ts +++ b/packages/cli/src/context/scan/credentials.ts @@ -2,20 +2,20 @@ import { redactKtxSensitiveMetadata, redactKtxSensitiveText, redactKtxSensitiveValue, - REDACTED_KTX_CREDENTIAL_VALUE, } from '../core/redaction.js'; import type { KtxCredentialEnvelope, KtxScanReport, KtxScanWarning } from './types.js'; -export { REDACTED_KTX_CREDENTIAL_VALUE }; - +/** @internal */ export function redactKtxCredentialValue(key: string, value: unknown): unknown { return redactKtxSensitiveValue(key, value); } +/** @internal */ export function redactKtxScanMetadata(metadata: Record): Record { return redactKtxSensitiveMetadata(metadata); } +/** @internal */ export function redactKtxCredentialEnvelope(envelope: KtxCredentialEnvelope): KtxCredentialEnvelope { if (envelope.kind !== 'resolved') { return envelope; @@ -28,6 +28,7 @@ export function redactKtxCredentialEnvelope(envelope: KtxCredentialEnvelope): Kt }; } +/** @internal */ export function redactKtxScanWarning(warning: KtxScanWarning): KtxScanWarning { if (!warning.metadata) { return { diff --git a/packages/context/src/scan/data-dictionary.test.ts b/packages/cli/src/context/scan/data-dictionary.test.ts similarity index 100% rename from packages/context/src/scan/data-dictionary.test.ts rename to packages/cli/src/context/scan/data-dictionary.test.ts diff --git a/packages/context/src/scan/data-dictionary.ts b/packages/cli/src/context/scan/data-dictionary.ts similarity index 96% rename from packages/context/src/scan/data-dictionary.ts rename to packages/cli/src/context/scan/data-dictionary.ts index 6c1acab7..859b9cc6 100644 --- a/packages/context/src/scan/data-dictionary.ts +++ b/packages/cli/src/context/scan/data-dictionary.ts @@ -36,17 +36,19 @@ export const defaultKtxDataDictionarySettings: KtxDataDictionarySettings = { ], }; -export type KtxDataDictionarySkipReason = +type KtxDataDictionarySkipReason = | 'not_candidate' | 'already_populated' | 'empty_column' | 'high_cardinality'; +/** @internal */ export interface KtxDataDictionarySampleDecision { sample: boolean; reason?: KtxDataDictionarySkipReason; } +/** @internal */ export interface KtxDataDictionaryColumnState { columnType: string; columnName: string; @@ -83,6 +85,7 @@ export function isKtxDataDictionaryCandidate( return true; } +/** @internal */ export function shouldKtxSampleColumnForDictionary( input: KtxDataDictionaryColumnState, ): KtxDataDictionarySampleDecision { diff --git a/packages/context/src/scan/description-generation.test.ts b/packages/cli/src/context/scan/description-generation.test.ts similarity index 100% rename from packages/context/src/scan/description-generation.test.ts rename to packages/cli/src/context/scan/description-generation.test.ts diff --git a/packages/context/src/scan/description-generation.ts b/packages/cli/src/context/scan/description-generation.ts similarity index 98% rename from packages/context/src/scan/description-generation.ts rename to packages/cli/src/context/scan/description-generation.ts index ba61fb62..4526215d 100644 --- a/packages/context/src/scan/description-generation.ts +++ b/packages/cli/src/context/scan/description-generation.ts @@ -1,4 +1,4 @@ -import type { KtxLlmRuntimePort } from '../llm/index.js'; +import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; import type { KtxColumnSampleInput, KtxColumnSampleResult, @@ -24,13 +24,13 @@ export interface KtxDescriptionCachePort { set(key: string, value: string): Promise; } -export interface KtxDescriptionSamplingPort { +interface KtxDescriptionSamplingPort { id: string; sampleColumn?(input: KtxColumnSampleInput, ctx: KtxScanContext): Promise; sampleTable?(input: KtxTableSampleInput, ctx: KtxScanContext): Promise; } -export interface KtxDescriptionGenerationSettings { +interface KtxDescriptionGenerationSettings { columnMaxWords: number; tableMaxWords: number; dataSourceMaxWords: number; @@ -57,7 +57,7 @@ export interface KtxDescriptionColumnTable extends KtxTableRef { columns: KtxDescriptionColumn[]; } -export interface KtxDescriptionTableInput extends KtxTableRef { +interface KtxDescriptionTableInput extends KtxTableRef { rawDescriptions?: Record; columns?: KtxDescriptionTableColumn[]; } @@ -68,6 +68,7 @@ export interface KtxColumnAnalysisResult { skippedColumns: string[]; } +/** @internal */ export interface KtxColumnDescriptionPromptInput { columnName: string; columnValues: unknown[]; @@ -77,6 +78,7 @@ export interface KtxColumnDescriptionPromptInput { rawDescriptions?: Record; } +/** @internal */ export interface KtxTableDescriptionPromptInput { tableName: string; sampleData?: KtxTableSampleResult; @@ -85,6 +87,7 @@ export interface KtxTableDescriptionPromptInput { rawDescriptions?: Record; } +/** @internal */ export interface KtxDataSourceDescriptionPromptInput { tableSamples: Array<[string, KtxTableSampleResult]>; dataSourceType: string; @@ -247,6 +250,7 @@ function wordLimitLine(maxWords: number): string { return `Please provide a concise description in ${maxWords} words or less.`; } +/** @internal */ export function buildKtxColumnDescriptionPrompt( input: KtxColumnDescriptionPromptInput & { maxWords?: number }, ): KtxDescriptionPrompt { @@ -295,6 +299,7 @@ Example: return { system: systemParts.join('\n\n'), user: user.trim() }; } +/** @internal */ export function buildKtxTableDescriptionPrompt( input: KtxTableDescriptionPromptInput & { maxWords?: number }, ): KtxDescriptionPrompt { @@ -363,6 +368,7 @@ Data source type: ${input.dataSourceType}`; return { system: systemParts.join('\n\n'), user: user.trim() }; } +/** @internal */ export function buildKtxDataSourceDescriptionPrompt( input: KtxDataSourceDescriptionPromptInput & { maxWords?: number }, ): KtxDescriptionPrompt { diff --git a/packages/context/src/scan/embedding-text.test.ts b/packages/cli/src/context/scan/embedding-text.test.ts similarity index 100% rename from packages/context/src/scan/embedding-text.test.ts rename to packages/cli/src/context/scan/embedding-text.test.ts diff --git a/packages/context/src/scan/embedding-text.ts b/packages/cli/src/context/scan/embedding-text.ts similarity index 96% rename from packages/context/src/scan/embedding-text.ts rename to packages/cli/src/context/scan/embedding-text.ts index 927f5779..375e24c2 100644 --- a/packages/context/src/scan/embedding-text.ts +++ b/packages/cli/src/context/scan/embedding-text.ts @@ -1,4 +1,4 @@ -export interface KtxColumnEmbeddingForeignKeys { +interface KtxColumnEmbeddingForeignKeys { outgoing: Array<{ toTable: string; toColumn: string }>; incoming: Array<{ fromTable: string; fromColumn: string }>; } diff --git a/packages/context/src/scan/enabled-tables.ts b/packages/cli/src/context/scan/enabled-tables.ts similarity index 100% rename from packages/context/src/scan/enabled-tables.ts rename to packages/cli/src/context/scan/enabled-tables.ts diff --git a/packages/context/src/scan/enrichment-state.test.ts b/packages/cli/src/context/scan/enrichment-state.test.ts similarity index 100% rename from packages/context/src/scan/enrichment-state.test.ts rename to packages/cli/src/context/scan/enrichment-state.test.ts diff --git a/packages/context/src/scan/enrichment-state.ts b/packages/cli/src/context/scan/enrichment-state.ts similarity index 97% rename from packages/context/src/scan/enrichment-state.ts rename to packages/cli/src/context/scan/enrichment-state.ts index 8eb5f421..4b4913e7 100644 --- a/packages/context/src/scan/enrichment-state.ts +++ b/packages/cli/src/context/scan/enrichment-state.ts @@ -1,7 +1,7 @@ import { createHash } from 'node:crypto'; import type { KtxScanEnrichmentStage, KtxScanEnrichmentStateSummary, KtxScanMode, KtxSchemaSnapshot } from './types.js'; -export const KTX_SCAN_ENRICHMENT_STAGES: readonly KtxScanEnrichmentStage[] = [ +const KTX_SCAN_ENRICHMENT_STAGES: readonly KtxScanEnrichmentStage[] = [ 'descriptions', 'embeddings', 'relationships', diff --git a/packages/context/src/scan/enrichment-summary.test.ts b/packages/cli/src/context/scan/enrichment-summary.test.ts similarity index 100% rename from packages/context/src/scan/enrichment-summary.test.ts rename to packages/cli/src/context/scan/enrichment-summary.test.ts diff --git a/packages/context/src/scan/enrichment-summary.ts b/packages/cli/src/context/scan/enrichment-summary.ts similarity index 100% rename from packages/context/src/scan/enrichment-summary.ts rename to packages/cli/src/context/scan/enrichment-summary.ts diff --git a/packages/context/src/scan/enrichment-types.test.ts b/packages/cli/src/context/scan/enrichment-types.test.ts similarity index 100% rename from packages/context/src/scan/enrichment-types.test.ts rename to packages/cli/src/context/scan/enrichment-types.test.ts diff --git a/packages/context/src/scan/enrichment-types.ts b/packages/cli/src/context/scan/enrichment-types.ts similarity index 88% rename from packages/context/src/scan/enrichment-types.ts rename to packages/cli/src/context/scan/enrichment-types.ts index 645951c3..2d19fa5c 100644 --- a/packages/context/src/scan/enrichment-types.ts +++ b/packages/cli/src/context/scan/enrichment-types.ts @@ -1,8 +1,8 @@ import type { KtxSchemaDimensionType, KtxTableRef } from './types.js'; -export type KtxDescriptionSource = 'ai' | 'db' | 'dbt' | 'user' | (string & {}); +type KtxDescriptionSource = 'ai' | 'db' | 'dbt' | 'user' | (string & {}); -export type KtxRelationshipSource = 'formal' | 'inferred' | 'manual'; +type KtxRelationshipSource = 'formal' | 'inferred' | 'manual'; export type KtxRelationshipType = 'many_to_one' | 'one_to_many' | 'one_to_one'; @@ -54,12 +54,14 @@ export interface KtxEnrichedSchema { relationships: KtxEnrichedRelationship[]; } +/** @internal */ export interface KtxStructuralSyncPlan { connectionId: string; snapshotId: string; operations: Array>; } +/** @internal */ export interface KtxDescriptionUpdate { connectionId: string; table: KtxTableRef; @@ -68,14 +70,8 @@ export interface KtxDescriptionUpdate { columnDescriptions?: Record; } -export interface KtxMetadataUpdate { - connectionId: string; - table: KtxTableRef; - source: KtxDescriptionSource; - tableFields?: Record; - columnFields?: Record>; -} +/** @internal */ export interface KtxJoinUpdate { connectionId: string; fromTable: string; @@ -87,6 +83,7 @@ export interface KtxJoinUpdate { authorEmail: string; } +/** @internal */ export interface KtxColumnSampleUpdate { columnId: string; sampleValues: string[] | null; @@ -111,6 +108,7 @@ export interface KtxRelationshipUpdate { skipped: KtxSkippedRelationship[]; } +/** @internal */ export interface KtxScanMetadataStore { loadSchema(connectionId: string): Promise; applyStructuralPlan(plan: KtxStructuralSyncPlan): Promise; diff --git a/packages/context/src/scan/entity-details.test.ts b/packages/cli/src/context/scan/entity-details.test.ts similarity index 98% rename from packages/context/src/scan/entity-details.test.ts rename to packages/cli/src/context/scan/entity-details.test.ts index db81ad11..ddccef87 100644 --- a/packages/context/src/scan/entity-details.test.ts +++ b/packages/cli/src/context/scan/entity-details.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { createKtxEntityDetailsService } from './entity-details.js'; import type { KtxConnectionDriver, KtxScanReport, KtxSchemaTable } from './types.js'; diff --git a/packages/context/src/scan/entity-details.ts b/packages/cli/src/context/scan/entity-details.ts similarity index 96% rename from packages/context/src/scan/entity-details.ts rename to packages/cli/src/context/scan/entity-details.ts index 6e95690e..37e766b6 100644 --- a/packages/context/src/scan/entity-details.ts +++ b/packages/cli/src/context/scan/entity-details.ts @@ -1,4 +1,4 @@ -import type { KtxLocalProject } from '../project/index.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import { readLocalScanStructuralSnapshot } from './local-structural-artifacts.js'; import type { KtxConnectionDriver, @@ -25,7 +25,7 @@ export interface KtxEntityDetailsSnapshotInfo { scanRunId: string | null; } -export interface KtxEntityDetailsColumn { +interface KtxEntityDetailsColumn { name: string; nativeType: string; normalizedType: string; @@ -35,7 +35,7 @@ export interface KtxEntityDetailsColumn { comment: string | null; } -export interface KtxEntityDetailsRecord { +interface KtxEntityDetailsRecord { ok: true; connectionId: string; tableRef: KtxTableRef; @@ -48,9 +48,9 @@ export interface KtxEntityDetailsRecord { snapshot: KtxEntityDetailsSnapshotInfo; } -export type KtxEntityDetailsErrorCode = 'scan_missing' | 'table_not_found' | 'ambiguous_table' | 'column_not_found'; +type KtxEntityDetailsErrorCode = 'scan_missing' | 'table_not_found' | 'ambiguous_table' | 'column_not_found'; -export interface KtxEntityDetailsErrorResult { +interface KtxEntityDetailsErrorResult { ok: false; connectionId: string; table: KtxEntityDetailsTableInput; diff --git a/packages/context/src/scan/local-enrichment-artifacts.test.ts b/packages/cli/src/context/scan/local-enrichment-artifacts.test.ts similarity index 99% rename from packages/context/src/scan/local-enrichment-artifacts.test.ts rename to packages/cli/src/context/scan/local-enrichment-artifacts.test.ts index 068e6cb1..56994568 100644 --- a/packages/context/src/scan/local-enrichment-artifacts.test.ts +++ b/packages/cli/src/context/scan/local-enrichment-artifacts.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import YAML from 'yaml'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import type { KtxLocalScanEnrichmentResult } from './local-enrichment.js'; import { writeLocalScanEnrichmentArtifacts, writeLocalScanManifestShards } from './local-enrichment-artifacts.js'; import type { KtxSchemaSnapshot } from './types.js'; diff --git a/packages/context/src/scan/local-enrichment-artifacts.ts b/packages/cli/src/context/scan/local-enrichment-artifacts.ts similarity index 96% rename from packages/context/src/scan/local-enrichment-artifacts.ts rename to packages/cli/src/context/scan/local-enrichment-artifacts.ts index ed698806..3a2d15f6 100644 --- a/packages/context/src/scan/local-enrichment-artifacts.ts +++ b/packages/cli/src/context/scan/local-enrichment-artifacts.ts @@ -1,15 +1,8 @@ import YAML from 'yaml'; -import { - buildLiveDatabaseManifestShards, - type LiveDatabaseManifestExistingDescriptions, - type LiveDatabaseManifestJoinData, - type LiveDatabaseManifestJoinEntry, - type LiveDatabaseManifestShard, - type LiveDatabaseManifestTableData, - type TableUsageOutput, -} from '../ingest/index.js'; +import { buildLiveDatabaseManifestShards, type LiveDatabaseManifestExistingDescriptions, type LiveDatabaseManifestJoinData, type LiveDatabaseManifestJoinEntry, type LiveDatabaseManifestShard, type LiveDatabaseManifestTableData } from '../../context/ingest/adapters/live-database/manifest.js'; +import type { TableUsageOutput } from '../../context/ingest/adapters/historic-sql/skill-schemas.js'; import type { KtxScanRelationshipConfig } from '../project/config.js'; -import type { KtxLocalProject } from '../project/index.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import type { KtxLocalScanEnrichmentResult } from './local-enrichment.js'; import { buildKtxRelationshipArtifacts, diff --git a/packages/context/src/scan/local-enrichment.test.ts b/packages/cli/src/context/scan/local-enrichment.test.ts similarity index 94% rename from packages/context/src/scan/local-enrichment.test.ts rename to packages/cli/src/context/scan/local-enrichment.test.ts index fe4239c1..66b66fc2 100644 --- a/packages/context/src/scan/local-enrichment.test.ts +++ b/packages/cli/src/context/scan/local-enrichment.test.ts @@ -12,7 +12,6 @@ import { runLocalScanEnrichment, snapshotToKtxEnrichedSchema, } from './local-enrichment.js'; -import { createLocalScanEnrichmentProvidersFromConfig } from './local-scan.js'; import { createKtxConnectorCapabilities, type KtxQueryResult, @@ -813,46 +812,4 @@ describe('local scan enrichment', () => { } }); - it('resolves gateway LLM providers and passes injected embedding provider through to scan enrichment', () => { - const createKtxLlmProvider = vi.fn(() => ({ - getModel: vi.fn().mockReturnValue({ modelId: 'provider/language-model', provider: 'gateway' }), - })); - const embeddingProvider = { - dimensions: 1536, - maxBatchSize: 8, - embed: vi.fn(), - [['embed', 'Many'].join('')]: vi.fn(), - }; - - const providers = createLocalScanEnrichmentProvidersFromConfig( - { - mode: 'llm', - embeddings: { - backend: 'openai', - model: 'provider/embedding-model', - dimensions: 1536, - batchSize: 8, - openai: { api_key: 'env:OPENAI_API_KEY' }, // pragma: allowlist secret - }, - }, - { - provider: { - backend: 'gateway', - gateway: {}, - }, - models: { default: 'provider/language-model' }, - }, - { - createKtxLlmProvider: createKtxLlmProvider as any, - env: { OPENAI_API_KEY: 'openai-key' }, // pragma: allowlist secret - embeddingProvider: embeddingProvider as any, - }, - ); - - expect(providers?.embedding?.dimensions).toBe(1536); - expect(providers?.embedding?.maxBatchSize).toBe(8); - expect(createKtxLlmProvider).toHaveBeenCalledWith( - expect.objectContaining({ backend: 'gateway', modelSlots: { default: 'provider/language-model' } }), - ); - }); }); diff --git a/packages/context/src/scan/local-enrichment.ts b/packages/cli/src/context/scan/local-enrichment.ts similarity index 99% rename from packages/context/src/scan/local-enrichment.ts rename to packages/cli/src/context/scan/local-enrichment.ts index 381f3cb0..680f8f60 100644 --- a/packages/context/src/scan/local-enrichment.ts +++ b/packages/cli/src/context/scan/local-enrichment.ts @@ -1,5 +1,5 @@ import pLimit from 'p-limit'; -import type { KtxLlmRuntimePort } from '../llm/index.js'; +import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; import { buildDefaultKtxProjectConfig, type KtxScanRelationshipConfig } from '../project/config.js'; import { type KtxDescriptionColumnTable, KtxDescriptionGenerator } from './description-generation.js'; import { buildKtxColumnEmbeddingText } from './embedding-text.js'; diff --git a/packages/context/src/scan/local-scan.test.ts b/packages/cli/src/context/scan/local-scan.test.ts similarity index 99% rename from packages/context/src/scan/local-scan.test.ts rename to packages/cli/src/context/scan/local-scan.test.ts index d04f9953..081fa055 100644 --- a/packages/context/src/scan/local-scan.test.ts +++ b/packages/cli/src/context/scan/local-scan.test.ts @@ -3,10 +3,11 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import YAML from 'yaml'; -import type { SourceAdapter } from '../ingest/index.js'; -import type { KtxLlmRuntimePort } from '../llm/index.js'; -import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js'; -import { filterSnapshotTables, getLocalScanReport, getLocalScanStatus, resolveEnabledTables, runLocalScan } from './local-scan.js'; +import type { SourceAdapter } from '../../context/ingest/types.js'; +import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; +import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js'; +import { filterSnapshotTables, resolveEnabledTables } from './enabled-tables.js'; +import { getLocalScanReport, getLocalScanStatus, runLocalScan } from './local-scan.js'; import type { KtxQueryResult, KtxReadOnlyQueryInput, KtxSchemaSnapshot, KtxSchemaTable } from './types.js'; function relationshipSqlResult( diff --git a/packages/context/src/scan/local-scan.ts b/packages/cli/src/context/scan/local-scan.ts similarity index 96% rename from packages/context/src/scan/local-scan.ts rename to packages/cli/src/context/scan/local-scan.ts index 4dbc448d..35333f79 100644 --- a/packages/context/src/scan/local-scan.ts +++ b/packages/cli/src/context/scan/local-scan.ts @@ -1,14 +1,13 @@ -import type { createKtxEmbeddingProvider, createKtxLlmProvider, KtxEmbeddingProvider } from '@ktx/llm'; -import { - createDefaultLocalIngestAdapters, - getLocalStageOnlyIngestStatus, - type LocalIngestRunRecord, - runLocalStageOnlyIngest, - type SourceAdapter, -} from '../ingest/index.js'; -import { createLocalKtxLlmRuntimeFromConfig, KtxScanEmbeddingPortAdapter } from '../llm/index.js'; +import type { createKtxEmbeddingProvider } from '../../llm/embedding-provider.js'; +import type { createKtxLlmProvider } from '../../llm/model-provider.js'; +import type { KtxEmbeddingProvider } from '../../llm/types.js'; +import { createDefaultLocalIngestAdapters } from '../../context/ingest/local-adapters.js'; +import { getLocalStageOnlyIngestStatus, type LocalIngestRunRecord, runLocalStageOnlyIngest } from '../../context/ingest/local-stage-ingest.js'; +import type { SourceAdapter } from '../../context/ingest/types.js'; +import { createLocalKtxLlmRuntimeFromConfig } from '../../context/llm/local-config.js'; +import { KtxScanEmbeddingPortAdapter } from '../../context/llm/embedding-port.js'; import type { KtxProjectLlmConfig, KtxScanEnrichmentConfig, KtxScanRelationshipConfig } from '../project/config.js'; -import type { KtxLocalProject } from '../project/index.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import { ktxLocalStateDbPath } from '../project/local-state-db.js'; import { redactKtxScanReport } from './credentials.js'; import { filterSnapshotTables, resolveEnabledTables } from './enabled-tables.js'; @@ -96,6 +95,7 @@ export interface LocalScanRunResult { report: KtxScanReport; } +/** @internal */ export interface LocalScanStatusResponse { runId: string; status: LocalIngestRunRecord['status']; @@ -226,15 +226,6 @@ function resolveLocalScanEnrichmentProviders( }; } -export function createLocalScanEnrichmentProvidersFromConfig( - config: KtxScanEnrichmentConfig, - llmConfig: KtxProjectLlmConfig, - deps: LocalScanEnrichmentProviderDeps = {}, -): KtxLocalScanEnrichmentProviders | null { - const resolved = resolveLocalScanEnrichmentProviders(config, llmConfig, deps); - return resolved.status === 'ready' ? resolved.providers : null; -} - function createLocalScanEnrichmentStateStore(options: RunLocalScanOptions): SqliteLocalScanEnrichmentStateStore | null { if (options.dryRun) { return null; @@ -389,7 +380,6 @@ function createFilteredConnector(connector: KtxScanConnector, enabledTables: Set }; } -export { filterSnapshotTables, resolveEnabledTables } from './enabled-tables.js'; function withInternalLiveDatabaseAdapter(project: KtxLocalProject): KtxLocalProject { if (project.config.ingest.adapters.includes(LIVE_DATABASE_ADAPTER)) { @@ -597,6 +587,7 @@ export async function runLocalScan(options: RunLocalScanOptions): Promise { const status = await getLocalStageOnlyIngestStatus(project, runId); if (!status || status.adapter !== LIVE_DATABASE_ADAPTER) { @@ -614,6 +605,7 @@ export async function getLocalScanReport(project: KtxLocalProject, runId: string }; } +/** @internal */ export async function getLocalScanStatus( project: KtxLocalProject, runId: string, diff --git a/packages/context/src/scan/local-structural-artifacts.test.ts b/packages/cli/src/context/scan/local-structural-artifacts.test.ts similarity index 98% rename from packages/context/src/scan/local-structural-artifacts.test.ts rename to packages/cli/src/context/scan/local-structural-artifacts.test.ts index e8089158..71019597 100644 --- a/packages/context/src/scan/local-structural-artifacts.test.ts +++ b/packages/cli/src/context/scan/local-structural-artifacts.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { readLocalScanStructuralSnapshot } from './local-structural-artifacts.js'; describe('readLocalScanStructuralSnapshot', () => { diff --git a/packages/context/src/scan/local-structural-artifacts.ts b/packages/cli/src/context/scan/local-structural-artifacts.ts similarity index 98% rename from packages/context/src/scan/local-structural-artifacts.ts rename to packages/cli/src/context/scan/local-structural-artifacts.ts index 5eeeea6d..2c968384 100644 --- a/packages/context/src/scan/local-structural-artifacts.ts +++ b/packages/cli/src/context/scan/local-structural-artifacts.ts @@ -1,4 +1,4 @@ -import type { KtxLocalProject } from '../project/index.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import type { KtxConnectionDriver, KtxSchemaColumn, diff --git a/packages/context/src/scan/relationship-benchmark-report.test.ts b/packages/cli/src/context/scan/relationship-benchmark-report.test.ts similarity index 100% rename from packages/context/src/scan/relationship-benchmark-report.test.ts rename to packages/cli/src/context/scan/relationship-benchmark-report.test.ts diff --git a/packages/context/src/scan/relationship-benchmark-report.ts b/packages/cli/src/context/scan/relationship-benchmark-report.ts similarity index 100% rename from packages/context/src/scan/relationship-benchmark-report.ts rename to packages/cli/src/context/scan/relationship-benchmark-report.ts diff --git a/packages/context/src/scan/relationship-benchmarks.test.ts b/packages/cli/src/context/scan/relationship-benchmarks.test.ts similarity index 100% rename from packages/context/src/scan/relationship-benchmarks.test.ts rename to packages/cli/src/context/scan/relationship-benchmarks.test.ts diff --git a/packages/context/src/scan/relationship-benchmarks.ts b/packages/cli/src/context/scan/relationship-benchmarks.ts similarity index 99% rename from packages/context/src/scan/relationship-benchmarks.ts rename to packages/cli/src/context/scan/relationship-benchmarks.ts index e3c65056..f4367b5a 100644 --- a/packages/context/src/scan/relationship-benchmarks.ts +++ b/packages/cli/src/context/scan/relationship-benchmarks.ts @@ -6,7 +6,7 @@ import { gunzipSync } from 'node:zlib'; import Database from 'better-sqlite3'; import YAML from 'yaml'; import { z } from 'zod'; -import type { KtxLlmRuntimePort } from '../llm/index.js'; +import type { KtxLlmRuntimePort } from '../llm/runtime-port.js'; import type { KtxEnrichedRelationship, KtxEnrichedSchema, KtxRelationshipType } from './enrichment-types.js'; import { snapshotToKtxEnrichedSchema } from './local-enrichment.js'; import type { KtxRelationshipDiscoveryCandidate } from './relationship-candidates.js'; diff --git a/packages/context/src/scan/relationship-budget.test.ts b/packages/cli/src/context/scan/relationship-budget.test.ts similarity index 100% rename from packages/context/src/scan/relationship-budget.test.ts rename to packages/cli/src/context/scan/relationship-budget.test.ts diff --git a/packages/context/src/scan/relationship-budget.ts b/packages/cli/src/context/scan/relationship-budget.ts similarity index 96% rename from packages/context/src/scan/relationship-budget.ts rename to packages/cli/src/context/scan/relationship-budget.ts index 13209301..78d1a348 100644 --- a/packages/context/src/scan/relationship-budget.ts +++ b/packages/cli/src/context/scan/relationship-budget.ts @@ -1,6 +1,6 @@ export type KtxRelationshipValidationBudget = number | 'all' | undefined; -export interface KtxRelationshipBudgetedCandidate { +interface KtxRelationshipBudgetedCandidate { candidate: TCandidate; originalIndex: number; score: number; @@ -19,6 +19,7 @@ export interface ApplyKtxRelationshipValidationBudgetInput { score: (candidate: TCandidate) => number; } +/** @internal */ export function defaultKtxRelationshipValidationBudget(tableCount: number): number { const safeTableCount = Number.isFinite(tableCount) ? Math.max(0, Math.floor(tableCount)) : 0; return Math.min(2 * safeTableCount, 1000); diff --git a/packages/context/src/scan/relationship-candidates.test.ts b/packages/cli/src/context/scan/relationship-candidates.test.ts similarity index 100% rename from packages/context/src/scan/relationship-candidates.test.ts rename to packages/cli/src/context/scan/relationship-candidates.test.ts diff --git a/packages/context/src/scan/relationship-candidates.ts b/packages/cli/src/context/scan/relationship-candidates.ts similarity index 99% rename from packages/context/src/scan/relationship-candidates.ts rename to packages/cli/src/context/scan/relationship-candidates.ts index cd3b7767..21b2c040 100644 --- a/packages/context/src/scan/relationship-candidates.ts +++ b/packages/cli/src/context/scan/relationship-candidates.ts @@ -12,7 +12,6 @@ import { pluralizeKtxRelationshipToken, singularizeKtxRelationshipToken, } from './relationship-name-similarity.js'; -export type { KtxRelationshipNormalizedName } from './relationship-name-similarity.js'; export { normalizeKtxRelationshipName } from './relationship-name-similarity.js'; import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js'; import { @@ -21,7 +20,7 @@ import { type KtxRelationshipSignalVector, } from './relationship-scoring.js'; -export type KtxRelationshipDiscoveryCandidateSource = +type KtxRelationshipDiscoveryCandidateSource = | 'exact_column_match' | 'normalized_table_match' | 'parent_table_name_match' @@ -32,9 +31,9 @@ export type KtxRelationshipDiscoveryCandidateSource = | 'embedding_similarity' | 'llm_proposal'; -export type KtxRelationshipDiscoveryCandidateStatus = 'review'; +type KtxRelationshipDiscoveryCandidateStatus = 'review'; -export interface KtxRelationshipDiscoveryCandidateEvidence { +interface KtxRelationshipDiscoveryCandidateEvidence { sourceColumnBase: string; targetTableBase: string; targetColumnBase: string; @@ -69,6 +68,7 @@ export interface KtxRelationshipDiscoveryCandidateOptions { profiles?: KtxRelationshipProfileArtifact; } +/** @internal */ export interface KtxRelationshipInferredTargetPk { table: string; columns: string[]; @@ -758,6 +758,7 @@ export function mergeKtxRelationshipDiscoveryCandidates( return Array.from(byId.values()).sort((left, right) => candidateSortKey(left).localeCompare(candidateSortKey(right))); } +/** @internal */ export function inferKtxRelationshipTargetPks( candidates: readonly KtxRelationshipDiscoveryCandidate[], ): KtxRelationshipInferredTargetPk[] { diff --git a/packages/context/src/scan/relationship-composite-candidates.test.ts b/packages/cli/src/context/scan/relationship-composite-candidates.test.ts similarity index 100% rename from packages/context/src/scan/relationship-composite-candidates.test.ts rename to packages/cli/src/context/scan/relationship-composite-candidates.test.ts diff --git a/packages/context/src/scan/relationship-composite-candidates.ts b/packages/cli/src/context/scan/relationship-composite-candidates.ts similarity index 99% rename from packages/context/src/scan/relationship-composite-candidates.ts rename to packages/cli/src/context/scan/relationship-composite-candidates.ts index f93d5153..d8ee650d 100644 --- a/packages/context/src/scan/relationship-composite-candidates.ts +++ b/packages/cli/src/context/scan/relationship-composite-candidates.ts @@ -7,9 +7,9 @@ import { } from './relationship-profiling.js'; import type { KtxConnectionDriver, KtxQueryResult, KtxScanContext, KtxTableRef } from './types.js'; -export type KtxCompositeRelationshipStatus = 'accepted' | 'review' | 'rejected'; +type KtxCompositeRelationshipStatus = 'accepted' | 'review' | 'rejected'; -export interface KtxCompositeRelationshipTupleEndpoint { +interface KtxCompositeRelationshipTupleEndpoint { tableId: string; columnIds: string[]; table: KtxTableRef; @@ -33,7 +33,7 @@ export interface KtxCompositePrimaryKeyCandidate { }; } -export interface KtxCompositeRelationshipValidationEvidence { +interface KtxCompositeRelationshipValidationEvidence { targetUniqueness: number; sourceCoverage: number; violationCount: number; diff --git a/packages/context/src/scan/relationship-diagnostics.test.ts b/packages/cli/src/context/scan/relationship-diagnostics.test.ts similarity index 100% rename from packages/context/src/scan/relationship-diagnostics.test.ts rename to packages/cli/src/context/scan/relationship-diagnostics.test.ts diff --git a/packages/context/src/scan/relationship-diagnostics.ts b/packages/cli/src/context/scan/relationship-diagnostics.ts similarity index 97% rename from packages/context/src/scan/relationship-diagnostics.ts rename to packages/cli/src/context/scan/relationship-diagnostics.ts index aa9564ae..e50e892e 100644 --- a/packages/context/src/scan/relationship-diagnostics.ts +++ b/packages/cli/src/context/scan/relationship-diagnostics.ts @@ -12,7 +12,7 @@ import type { KtxCompositeRelationshipCandidate } from './relationship-composite import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js'; import type { KtxConnectionDriver, KtxScanWarning } from './types.js'; -export interface KtxRelationshipArtifactEndpoint { +interface KtxRelationshipArtifactEndpoint { tableId: string; columnIds: string[]; table: { @@ -23,7 +23,7 @@ export interface KtxRelationshipArtifactEndpoint { columns: string[]; } -export interface KtxRelationshipArtifactEdge { +interface KtxRelationshipArtifactEdge { id: string; status: KtxResolvedRelationshipStatus; source: string; @@ -48,25 +48,25 @@ export interface KtxRelationshipArtifact { skipped: KtxRelationshipUpdate['skipped']; } -export interface KtxRelationshipDiagnosticsSummary { +interface KtxRelationshipDiagnosticsSummary { accepted: number; review: number; rejected: number; skipped: number; } -export interface KtxRelationshipDiagnosticsValidation { +interface KtxRelationshipDiagnosticsValidation { available: boolean; sqlAvailable: boolean; queryCount: number; } -export interface KtxRelationshipDiagnosticsThresholds { +interface KtxRelationshipDiagnosticsThresholds { acceptThreshold: number; reviewThreshold: number; } -export interface KtxRelationshipDiagnosticsPolicy { +interface KtxRelationshipDiagnosticsPolicy { validationRequiredForManifest: boolean; maxCandidatesPerColumn: number; profileSampleRows: number; diff --git a/packages/context/src/scan/relationship-discovery.test.ts b/packages/cli/src/context/scan/relationship-discovery.test.ts similarity index 99% rename from packages/context/src/scan/relationship-discovery.test.ts rename to packages/cli/src/context/scan/relationship-discovery.test.ts index 36025ffd..400fae62 100644 --- a/packages/context/src/scan/relationship-discovery.test.ts +++ b/packages/cli/src/context/scan/relationship-discovery.test.ts @@ -1,6 +1,6 @@ import Database from 'better-sqlite3'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import type { KtxLlmRuntimePort } from '../llm/index.js'; +import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; import { buildDefaultKtxProjectConfig } from '../project/config.js'; import { snapshotToKtxEnrichedSchema } from './local-enrichment.js'; import { diff --git a/packages/context/src/scan/relationship-discovery.ts b/packages/cli/src/context/scan/relationship-discovery.ts similarity index 99% rename from packages/context/src/scan/relationship-discovery.ts rename to packages/cli/src/context/scan/relationship-discovery.ts index ce6dfba9..c1197866 100644 --- a/packages/context/src/scan/relationship-discovery.ts +++ b/packages/cli/src/context/scan/relationship-discovery.ts @@ -1,4 +1,4 @@ -import type { KtxLlmRuntimePort } from '../llm/index.js'; +import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; import type { KtxScanRelationshipConfig } from '../project/config.js'; import type { KtxEnrichedRelationship, KtxEnrichedSchema, KtxRelationshipUpdate } from './enrichment-types.js'; import { diff --git a/packages/context/src/scan/relationship-formal-metadata.test.ts b/packages/cli/src/context/scan/relationship-formal-metadata.test.ts similarity index 100% rename from packages/context/src/scan/relationship-formal-metadata.test.ts rename to packages/cli/src/context/scan/relationship-formal-metadata.test.ts diff --git a/packages/context/src/scan/relationship-formal-metadata.ts b/packages/cli/src/context/scan/relationship-formal-metadata.ts similarity index 100% rename from packages/context/src/scan/relationship-formal-metadata.ts rename to packages/cli/src/context/scan/relationship-formal-metadata.ts diff --git a/packages/context/src/scan/relationship-graph-resolver.test.ts b/packages/cli/src/context/scan/relationship-graph-resolver.test.ts similarity index 100% rename from packages/context/src/scan/relationship-graph-resolver.test.ts rename to packages/cli/src/context/scan/relationship-graph-resolver.test.ts diff --git a/packages/context/src/scan/relationship-graph-resolver.ts b/packages/cli/src/context/scan/relationship-graph-resolver.ts similarity index 98% rename from packages/context/src/scan/relationship-graph-resolver.ts rename to packages/cli/src/context/scan/relationship-graph-resolver.ts index 36ab44b2..e18d1132 100644 --- a/packages/context/src/scan/relationship-graph-resolver.ts +++ b/packages/cli/src/context/scan/relationship-graph-resolver.ts @@ -11,14 +11,14 @@ import type { KtxValidatedRelationshipDiscoveryCandidate } from './relationship- export type KtxResolvedRelationshipStatus = 'accepted' | 'review' | 'rejected'; -export interface KtxRelationshipGraphResolverSettings { +interface KtxRelationshipGraphResolverSettings { acceptThreshold: number; reviewThreshold: number; minTargetPkScoreForAcceptance: number; validationRequiredForManifest: boolean; } -export interface KtxResolvedRelationshipPkEvidence { +interface KtxResolvedRelationshipPkEvidence { declaredPrimaryKey: boolean; targetUniqueness: number; incomingAcceptedCount: number; @@ -26,7 +26,7 @@ export interface KtxResolvedRelationshipPkEvidence { reasons: string[]; } -export interface KtxResolvedRelationshipPk { +interface KtxResolvedRelationshipPk { table: string; columns: string[]; pkScore: number; @@ -35,7 +35,7 @@ export interface KtxResolvedRelationshipPk { evidence: KtxResolvedRelationshipPkEvidence; } -export interface KtxResolvedRelationshipGraphEvidence { +interface KtxResolvedRelationshipGraphEvidence { targetPkScore: number; incomingCandidateCount: number; conflictRank: number; diff --git a/packages/context/src/scan/relationship-llm-proposal.test.ts b/packages/cli/src/context/scan/relationship-llm-proposal.test.ts similarity index 98% rename from packages/context/src/scan/relationship-llm-proposal.test.ts rename to packages/cli/src/context/scan/relationship-llm-proposal.test.ts index 1aa994de..46e22dcb 100644 --- a/packages/context/src/scan/relationship-llm-proposal.test.ts +++ b/packages/cli/src/context/scan/relationship-llm-proposal.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import type { KtxLlmRuntimePort } from '../llm/index.js'; +import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from './enrichment-types.js'; import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js'; import { proposeKtxRelationshipCandidatesWithLlm } from './relationship-llm-proposal.js'; diff --git a/packages/context/src/scan/relationship-llm-proposal.ts b/packages/cli/src/context/scan/relationship-llm-proposal.ts similarity index 98% rename from packages/context/src/scan/relationship-llm-proposal.ts rename to packages/cli/src/context/scan/relationship-llm-proposal.ts index 7144035f..04b36abe 100644 --- a/packages/context/src/scan/relationship-llm-proposal.ts +++ b/packages/cli/src/context/scan/relationship-llm-proposal.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import type { KtxLlmRuntimePort } from '../llm/index.js'; +import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from './enrichment-types.js'; import { normalizeKtxRelationshipName, @@ -31,7 +31,7 @@ const relationshipLlmProposalSchema = z.object({ type KtxRelationshipLlmProposalOutput = z.infer; -export interface KtxRelationshipLlmProposalSettings { +interface KtxRelationshipLlmProposalSettings { maxTablesPerBatch: number; maxColumnsPerTable: number; maxSampleValuesPerColumn: number; diff --git a/packages/context/src/scan/relationship-locality.test.ts b/packages/cli/src/context/scan/relationship-locality.test.ts similarity index 100% rename from packages/context/src/scan/relationship-locality.test.ts rename to packages/cli/src/context/scan/relationship-locality.test.ts diff --git a/packages/context/src/scan/relationship-locality.ts b/packages/cli/src/context/scan/relationship-locality.ts similarity index 100% rename from packages/context/src/scan/relationship-locality.ts rename to packages/cli/src/context/scan/relationship-locality.ts diff --git a/packages/context/src/scan/relationship-name-similarity.test.ts b/packages/cli/src/context/scan/relationship-name-similarity.test.ts similarity index 100% rename from packages/context/src/scan/relationship-name-similarity.test.ts rename to packages/cli/src/context/scan/relationship-name-similarity.test.ts diff --git a/packages/context/src/scan/relationship-name-similarity.ts b/packages/cli/src/context/scan/relationship-name-similarity.ts similarity index 99% rename from packages/context/src/scan/relationship-name-similarity.ts rename to packages/cli/src/context/scan/relationship-name-similarity.ts index 14d53fe3..63378c0c 100644 --- a/packages/context/src/scan/relationship-name-similarity.ts +++ b/packages/cli/src/context/scan/relationship-name-similarity.ts @@ -6,6 +6,7 @@ export interface KtxRelationshipNormalizedName { tokens: string[]; } +/** @internal */ export type KtxRelationshipTokenInput = string | readonly string[] | KtxRelationshipNormalizedName; const WAREHOUSE_LAYER_PREFIXES = new Set(['stg', 'stage', 'staging', 'dim', 'fct', 'fact', 'int', 'mart']); @@ -132,6 +133,7 @@ function roundedScore(value: number): number { return Number(Math.max(0, Math.min(1, value)).toFixed(3)); } +/** @internal */ export function tokenSimilarity(leftInput: KtxRelationshipTokenInput, rightInput: KtxRelationshipTokenInput): number { const left = tokensFromInput(leftInput); const right = tokensFromInput(rightInput); diff --git a/packages/context/src/scan/relationship-profiling.test.ts b/packages/cli/src/context/scan/relationship-profiling.test.ts similarity index 100% rename from packages/context/src/scan/relationship-profiling.test.ts rename to packages/cli/src/context/scan/relationship-profiling.test.ts diff --git a/packages/context/src/scan/relationship-profiling.ts b/packages/cli/src/context/scan/relationship-profiling.ts similarity index 99% rename from packages/context/src/scan/relationship-profiling.ts rename to packages/cli/src/context/scan/relationship-profiling.ts index fa6acfac..2172ac24 100644 --- a/packages/context/src/scan/relationship-profiling.ts +++ b/packages/cli/src/context/scan/relationship-profiling.ts @@ -26,6 +26,7 @@ export interface KtxRelationshipColumnProfile { maxTextLength: number | null; } +/** @internal */ export interface KtxRelationshipTableProfile { table: KtxTableRef; rowCount: number; diff --git a/packages/context/src/scan/relationship-scoring.test.ts b/packages/cli/src/context/scan/relationship-scoring.test.ts similarity index 100% rename from packages/context/src/scan/relationship-scoring.test.ts rename to packages/cli/src/context/scan/relationship-scoring.test.ts diff --git a/packages/context/src/scan/relationship-scoring.ts b/packages/cli/src/context/scan/relationship-scoring.ts similarity index 96% rename from packages/context/src/scan/relationship-scoring.ts rename to packages/cli/src/context/scan/relationship-scoring.ts index ce76f9f4..a6f03e92 100644 --- a/packages/context/src/scan/relationship-scoring.ts +++ b/packages/cli/src/context/scan/relationship-scoring.ts @@ -1,4 +1,4 @@ -export const KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS = [ +const KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS = [ 'nameSimilarity', 'typeCompatibility', 'valueOverlap', @@ -8,7 +8,7 @@ export const KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS = [ 'structuralPrior', ] as const; -export type KtxRelationshipScoreSignal = (typeof KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS)[number]; +type KtxRelationshipScoreSignal = (typeof KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS)[number]; export type KtxRelationshipFixtureOrigin = 'synthetic' | 'public' | 'customer'; @@ -31,6 +31,7 @@ export interface KtxRelationshipScoreBreakdown { contributions: KtxRelationshipScoreWeights; } +/** @internal */ export interface KtxRelationshipScoringCalibrationObservation { fixtureId: string; origin: KtxRelationshipFixtureOrigin; @@ -71,10 +72,12 @@ function sanitizeSignalVector(signals: KtxRelationshipSignalVector): KtxRelation }; } +/** @internal */ export function defaultKtxRelationshipScoreWeights(): KtxRelationshipScoreWeights { return { ...DEFAULT_WEIGHTS }; } +/** @internal */ export function normalizeKtxRelationshipScoreWeights( weights: Partial = DEFAULT_WEIGHTS, ): KtxRelationshipScoreWeights { @@ -123,6 +126,7 @@ function averageSignal( return observations.reduce((sum, observation) => sum + clampScore(observation.signals[key]), 0) / observations.length; } +/** @internal */ export function calibrateWeightsFromSyntheticFixtures( observations: readonly KtxRelationshipScoringCalibrationObservation[], ): KtxRelationshipScoreWeights { diff --git a/packages/context/src/scan/relationship-validation.test.ts b/packages/cli/src/context/scan/relationship-validation.test.ts similarity index 100% rename from packages/context/src/scan/relationship-validation.test.ts rename to packages/cli/src/context/scan/relationship-validation.test.ts diff --git a/packages/context/src/scan/relationship-validation.ts b/packages/cli/src/context/scan/relationship-validation.ts similarity index 98% rename from packages/context/src/scan/relationship-validation.ts rename to packages/cli/src/context/scan/relationship-validation.ts index b735afb2..63d7328a 100644 --- a/packages/context/src/scan/relationship-validation.ts +++ b/packages/cli/src/context/scan/relationship-validation.ts @@ -9,9 +9,9 @@ import { } from './relationship-profiling.js'; import type { KtxConnectionDriver, KtxQueryResult, KtxScanContext } from './types.js'; -export type KtxValidatedRelationshipStatus = 'accepted' | 'review' | 'rejected'; +type KtxValidatedRelationshipStatus = 'accepted' | 'review' | 'rejected'; -export interface KtxRelationshipValidationSettings { +interface KtxRelationshipValidationSettings { acceptThreshold: number; reviewThreshold: number; minTargetUniqueness: number; @@ -22,7 +22,7 @@ export interface KtxRelationshipValidationSettings { validationBudget?: KtxRelationshipValidationBudget; } -export interface KtxRelationshipValidationEvidence { +interface KtxRelationshipValidationEvidence { targetUniqueness: number; sourceCoverage: number; violationCount: number; diff --git a/packages/context/src/scan/sqlite-local-enrichment-state-store.ts b/packages/cli/src/context/scan/sqlite-local-enrichment-state-store.ts similarity index 100% rename from packages/context/src/scan/sqlite-local-enrichment-state-store.ts rename to packages/cli/src/context/scan/sqlite-local-enrichment-state-store.ts diff --git a/packages/context/src/scan/type-normalization.test.ts b/packages/cli/src/context/scan/type-normalization.test.ts similarity index 100% rename from packages/context/src/scan/type-normalization.test.ts rename to packages/cli/src/context/scan/type-normalization.test.ts diff --git a/packages/context/src/scan/type-normalization.ts b/packages/cli/src/context/scan/type-normalization.ts similarity index 96% rename from packages/context/src/scan/type-normalization.ts rename to packages/cli/src/context/scan/type-normalization.ts index 5c696339..38d15ebc 100644 --- a/packages/context/src/scan/type-normalization.ts +++ b/packages/cli/src/context/scan/type-normalization.ts @@ -1,5 +1,6 @@ import type { KtxSchemaDimensionType } from './types.js'; +/** @internal */ export interface KtxColumnTypeMapping { normalizedType: string; dimensionType: KtxSchemaDimensionType; @@ -24,6 +25,7 @@ export function inferKtxDimensionType(nativeType: string): KtxSchemaDimensionTyp return 'string'; } +/** @internal */ export function ktxColumnTypeMappingFromNative(nativeType: string): KtxColumnTypeMapping { return { normalizedType: normalizeKtxNativeType(nativeType), diff --git a/packages/context/src/scan/types.test.ts b/packages/cli/src/context/scan/types.test.ts similarity index 100% rename from packages/context/src/scan/types.test.ts rename to packages/cli/src/context/scan/types.test.ts diff --git a/packages/context/src/scan/types.ts b/packages/cli/src/context/scan/types.ts similarity index 94% rename from packages/context/src/scan/types.ts rename to packages/cli/src/context/scan/types.ts index bc8959f5..f4299e86 100644 --- a/packages/context/src/scan/types.ts +++ b/packages/cli/src/context/scan/types.ts @@ -42,13 +42,13 @@ export function createKtxConnectorCapabilities( }; } -export interface KtxSchemaScope { +interface KtxSchemaScope { catalogs?: string[]; schemas?: string[]; datasets?: string[]; } -export type KtxSchemaTableKind = 'table' | 'view' | 'external' | 'event_stream'; +type KtxSchemaTableKind = 'table' | 'view' | 'external' | 'event_stream'; export type KtxSchemaDimensionType = 'time' | 'string' | 'number' | 'boolean'; @@ -91,17 +91,17 @@ export interface KtxSchemaSnapshot { metadata: Record; } -export interface KtxCredentialEnvReference { +interface KtxCredentialEnvReference { kind: 'env'; name: string; } -export interface KtxCredentialFileReference { +interface KtxCredentialFileReference { kind: 'file'; path: string; } -export interface KtxResolvedCredentialEnvelope { +interface KtxResolvedCredentialEnvelope { kind: 'resolved'; source: 'standalone' | 'host'; values: Record; @@ -113,13 +113,14 @@ export type KtxCredentialEnvelope = | KtxCredentialFileReference | KtxResolvedCredentialEnvelope; +/** @internal */ export interface KtxNetworkEndpoint { host: string; port: number; close?: () => Promise; } -export interface KtxNetworkTunnelRequest> { +interface KtxNetworkTunnelRequest> { connectionId: string; driver: KtxConnectionDriver; host: string; @@ -127,6 +128,7 @@ export interface KtxNetworkTunnelRequest> connection: TConnection; } +/** @internal */ export interface KtxNetworkTunnelPort> { resolveEndpoint(input: KtxNetworkTunnelRequest): Promise; } @@ -211,6 +213,7 @@ export interface KtxColumnStatsResult { distinctCount: number | null; } +/** @internal */ export interface KtxEventTypeDiscoveryInput { connectionId: string; table: KtxTableRef; @@ -220,11 +223,13 @@ export interface KtxEventTypeDiscoveryInput { lookbackDays?: number; } +/** @internal */ export interface KtxEventTypeDiscovery { value: string; count: number; } +/** @internal */ export interface KtxEventPropertyDiscoveryInput { connectionId: string; table: KtxTableRef; @@ -234,11 +239,13 @@ export interface KtxEventPropertyDiscoveryInput { lookbackDays?: number; } +/** @internal */ export interface KtxEventPropertyDiscovery { key: string; count: number; } +/** @internal */ export interface KtxEventPropertyValuesInput { connectionId: string; table: KtxTableRef; @@ -249,11 +256,13 @@ export interface KtxEventPropertyValuesInput { lookbackDays?: number; } +/** @internal */ export interface KtxEventPropertyValuesResult { values: string[]; cardinality: number; } +/** @internal */ export interface KtxEventStreamDiscoveryPort { listEventTypes(input: KtxEventTypeDiscoveryInput, ctx: KtxScanContext): Promise; listPropertyKeys(input: KtxEventPropertyDiscoveryInput, ctx: KtxScanContext): Promise; @@ -283,7 +292,7 @@ export interface KtxTableListEntry { kind: 'table' | 'view'; } -export interface KtxConnectorTestResult { +interface KtxConnectorTestResult { success: boolean; error?: string; } @@ -308,7 +317,7 @@ export interface KtxEmbeddingPort { embedBatch(texts: string[]): Promise; } -export interface KtxStructuralSyncStats { +interface KtxStructuralSyncStats { tablesCreated: number; tablesUpdated: number; tablesDeleted: number; @@ -317,7 +326,7 @@ export interface KtxStructuralSyncStats { columnsDeleted: number; } -export interface KtxScanDiffSummary { +interface KtxScanDiffSummary { tablesAdded: number; tablesModified: number; tablesDeleted: number; @@ -327,14 +336,14 @@ export interface KtxScanDiffSummary { columnsDeleted: number; } -export interface KtxScanArtifactPaths { +interface KtxScanArtifactPaths { rawSourcesDir: string | null; reportPath: string | null; manifestShards: string[]; enrichmentArtifacts: string[]; } -export type KtxScanWarningCode = +type KtxScanWarningCode = | 'connector_capability_missing' | 'sampling_failed' | 'statistics_failed' diff --git a/packages/context/src/scan/warehouse-catalog.test.ts b/packages/cli/src/context/scan/warehouse-catalog.test.ts similarity index 98% rename from packages/context/src/scan/warehouse-catalog.test.ts rename to packages/cli/src/context/scan/warehouse-catalog.test.ts index 7cabc1df..6ef1f03a 100644 --- a/packages/context/src/scan/warehouse-catalog.test.ts +++ b/packages/cli/src/context/scan/warehouse-catalog.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { WarehouseCatalogService } from './warehouse-catalog.js'; describe('WarehouseCatalogService', () => { diff --git a/packages/context/src/scan/warehouse-catalog.ts b/packages/cli/src/context/scan/warehouse-catalog.ts similarity index 98% rename from packages/context/src/scan/warehouse-catalog.ts rename to packages/cli/src/context/scan/warehouse-catalog.ts index 8cbe324d..2f360eeb 100644 --- a/packages/context/src/scan/warehouse-catalog.ts +++ b/packages/cli/src/context/scan/warehouse-catalog.ts @@ -1,5 +1,5 @@ -import { getDialectForDriver } from '../connections/index.js'; -import type { KtxFileStorePort } from '../core/index.js'; +import { getDialectForDriver } from '../../context/connections/dialects.js'; +import type { KtxFileStorePort } from '../../context/core/file-store.js'; import type { KtxConnectionDriver, KtxSchemaColumn, diff --git a/packages/context/src/search/backend-conformance.test.ts b/packages/cli/src/context/search/backend-conformance.test-utils.test.ts similarity index 99% rename from packages/context/src/search/backend-conformance.test.ts rename to packages/cli/src/context/search/backend-conformance.test-utils.test.ts index 31519c8b..c9ecebb7 100644 --- a/packages/context/src/search/backend-conformance.test.ts +++ b/packages/cli/src/context/search/backend-conformance.test-utils.test.ts @@ -2,9 +2,9 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, it } from 'vitest'; -import { SqliteContextEvidenceStore } from '../ingest/context-evidence/index.js'; +import { SqliteContextEvidenceStore } from '../ingest/context-evidence/sqlite-context-evidence-store.js'; import type { JsonValue } from '../ingest/ports.js'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../project/project.js'; import { type LocalSlSourceSearchResult, searchLocalSlSources, writeLocalSlSource } from '../sl/local-sl.js'; import type { ContextEvidenceSearchResult } from '../tools/context-evidence-tool-store.js'; import { @@ -16,7 +16,7 @@ import { assertSearchBackendCapabilities, assertSearchBackendConformanceCase, type SearchBackendConformanceResult, -} from './backend-conformance.js'; +} from './backend-conformance.test-utils.js'; import type { SearchBackendCapabilities } from './types.js'; const SQLITE_SEARCH_CAPABILITIES = { diff --git a/packages/context/src/search/backend-conformance.ts b/packages/cli/src/context/search/backend-conformance.test-utils.ts similarity index 100% rename from packages/context/src/search/backend-conformance.ts rename to packages/cli/src/context/search/backend-conformance.test-utils.ts diff --git a/packages/context/src/search/discover.test.ts b/packages/cli/src/context/search/discover.test.ts similarity index 99% rename from packages/context/src/search/discover.test.ts rename to packages/cli/src/context/search/discover.test.ts index 7f9df413..931de2be 100644 --- a/packages/context/src/search/discover.test.ts +++ b/packages/cli/src/context/search/discover.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { writeLocalKnowledgePage } from '../wiki/local-knowledge.js'; import { createKtxDiscoverDataService } from './discover.js'; diff --git a/packages/context/src/search/discover.ts b/packages/cli/src/context/search/discover.ts similarity index 95% rename from packages/context/src/search/discover.ts rename to packages/cli/src/context/search/discover.ts index 53694f6a..b3456459 100644 --- a/packages/context/src/search/discover.ts +++ b/packages/cli/src/context/search/discover.ts @@ -1,12 +1,14 @@ -import type { KtxEmbeddingPort } from '../core/index.js'; -import type { KtxLocalProject } from '../project/index.js'; -import type { KtxScanReport, KtxSchemaColumn, KtxSchemaTable, KtxTableRef } from '../scan/index.js'; -import { DEFAULT_PRIORITY, loadLocalSlSourceRecords, resolveDescription } from '../sl/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import type { KtxScanReport, KtxSchemaColumn, KtxSchemaTable, KtxTableRef } from '../../context/scan/types.js'; +import { DEFAULT_PRIORITY, resolveDescription } from '../../context/sl/descriptions.js'; +import { loadLocalSlSourceRecords } from '../../context/sl/local-sl.js'; import { readLocalKnowledgePage, searchLocalKnowledgePages } from '../wiki/local-knowledge.js'; -import { HybridSearchCore, type FusedSearchCandidate, type SearchCandidateGenerator } from './index.js'; +import { HybridSearchCore } from '../../context/search/hybrid-search-core.js'; +import type { FusedSearchCandidate, SearchCandidateGenerator } from '../../context/search/types.js'; -export type KtxDiscoverDataKind = 'wiki' | 'sl_source' | 'sl_measure' | 'sl_dimension' | 'table' | 'column'; -export type KtxDiscoverDataMatchedOn = 'name' | 'display' | 'description' | 'comment' | 'expr' | 'sample_value' | 'body'; +type KtxDiscoverDataKind = 'wiki' | 'sl_source' | 'sl_measure' | 'sl_dimension' | 'table' | 'column'; +type KtxDiscoverDataMatchedOn = 'name' | 'display' | 'description' | 'comment' | 'expr' | 'sample_value' | 'body'; export interface KtxDiscoverDataInput { query: string; @@ -15,7 +17,7 @@ export interface KtxDiscoverDataInput { limit?: number; } -export interface KtxDiscoverDataRef { +interface KtxDiscoverDataRef { kind: KtxDiscoverDataKind; id: string; score: number; diff --git a/packages/context/src/search/hybrid-search-core.test.ts b/packages/cli/src/context/search/hybrid-search-core.test.ts similarity index 100% rename from packages/context/src/search/hybrid-search-core.test.ts rename to packages/cli/src/context/search/hybrid-search-core.test.ts diff --git a/packages/context/src/search/hybrid-search-core.ts b/packages/cli/src/context/search/hybrid-search-core.ts similarity index 100% rename from packages/context/src/search/hybrid-search-core.ts rename to packages/cli/src/context/search/hybrid-search-core.ts diff --git a/packages/context/src/search/pglite-owner-process.test.ts b/packages/cli/src/context/search/pglite-owner-process.test.ts similarity index 94% rename from packages/context/src/search/pglite-owner-process.test.ts rename to packages/cli/src/context/search/pglite-owner-process.test.ts index 0121b1e2..3a15eea9 100644 --- a/packages/context/src/search/pglite-owner-process.test.ts +++ b/packages/cli/src/context/search/pglite-owner-process.test.ts @@ -4,8 +4,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { Client } from 'pg'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { assertSearchBackendCapabilities, assertSearchBackendConformanceCase } from './index.js'; -import { KtxPGliteOwnerProcess, PGLITE_OWNER_PROCESS_BACKEND_CAPABILITIES } from './pglite-owner-process.js'; +import { assertSearchBackendConformanceCase } from '../../context/search/backend-conformance.test-utils.js'; +import { KtxPGliteOwnerProcess } from './pglite-owner-process.js'; async function allocatePort(): Promise { const server = createServer(); @@ -107,20 +107,6 @@ describe('KtxPGliteOwnerProcess', () => { await rm(tempDir, { recursive: true, force: true }); }); - it('declares the advanced PGlite search capabilities observed by the spike', () => { - assertSearchBackendCapabilities({ - backendName: 'pglite-owner-process', - capabilities: PGLITE_OWNER_PROCESS_BACKEND_CAPABILITIES, - expected: { - fts: true, - vector: true, - fuzzy: true, - jsonSearch: true, - arraySearch: false, - }, - }); - }); - it('starts a socket owner process and serves PostgreSQL clients', async () => { const owner = await KtxPGliteOwnerProcess.start({ dataDir, diff --git a/packages/context/src/search/pglite-owner-process.ts b/packages/cli/src/context/search/pglite-owner-process.ts similarity index 91% rename from packages/context/src/search/pglite-owner-process.ts rename to packages/cli/src/context/search/pglite-owner-process.ts index 373be59c..b6459639 100644 --- a/packages/context/src/search/pglite-owner-process.ts +++ b/packages/cli/src/context/search/pglite-owner-process.ts @@ -3,15 +3,6 @@ import { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm'; import { vector } from '@electric-sql/pglite/vector'; import { PGLiteSocketServer } from '@electric-sql/pglite-socket'; import { Client, type ClientConfig, type QueryResult, type QueryResultRow } from 'pg'; -import type { SearchBackendCapabilities } from './types.js'; - -export const PGLITE_OWNER_PROCESS_BACKEND_CAPABILITIES = { - fts: true, - vector: true, - fuzzy: true, - jsonSearch: true, - arraySearch: false, -} satisfies SearchBackendCapabilities; export interface KtxPGliteOwnerProcessOptions { dataDir: string; diff --git a/packages/context/src/search/pglite-runtime-boundary.test.ts b/packages/cli/src/context/search/pglite-runtime-boundary.test.ts similarity index 66% rename from packages/context/src/search/pglite-runtime-boundary.test.ts rename to packages/cli/src/context/search/pglite-runtime-boundary.test.ts index 2db3209b..feb7443d 100644 --- a/packages/context/src/search/pglite-runtime-boundary.test.ts +++ b/packages/cli/src/context/search/pglite-runtime-boundary.test.ts @@ -3,54 +3,44 @@ import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; -const ktxRoot = fileURLToPath(new URL('../../../../', import.meta.url)); +const ktxRoot = fileURLToPath(new URL('../../../../../', import.meta.url)); function readKtxFile(relativePath: string): string { return readFileSync(join(ktxRoot, relativePath), 'utf8'); } -function readContextPackageJson(): { +function readCliPackageJson(): { dependencies?: Record; devDependencies?: Record; exports?: Record; files?: string[]; } { - return JSON.parse(readKtxFile('packages/context/package.json')); + return JSON.parse(readKtxFile('packages/cli/package.json')); } describe('PGlite hybrid search runtime boundary', () => { it('keeps PGlite packages as dev-only prototype dependencies', () => { - const pkg = readContextPackageJson(); + const pkg = readCliPackageJson(); expect(pkg.dependencies?.['@electric-sql/pglite']).toBeUndefined(); expect(pkg.dependencies?.['@electric-sql/pglite-socket']).toBeUndefined(); expect(pkg.devDependencies?.['@electric-sql/pglite']).toBeDefined(); expect(pkg.devDependencies?.['@electric-sql/pglite-socket']).toBeDefined(); - expect(pkg.files).toEqual(['dist', 'prompts', 'skills']); + expect(pkg.files).toEqual(['dist', 'assets']); }); it('keeps PGlite prototypes out of public exports and production routing', () => { - const pkg = readContextPackageJson(); + const pkg = readCliPackageJson(); const packageExportKeys = Object.keys(pkg.exports ?? {}); expect(packageExportKeys.filter((key) => key.toLowerCase().includes('pglite'))).toEqual([]); - const publicExportFiles = [ - 'packages/context/src/index.ts', - 'packages/context/src/search/index.ts', - 'packages/context/src/sl/index.ts', - ]; - - for (const relativePath of publicExportFiles) { - expect(readKtxFile(relativePath), relativePath).not.toMatch(/pglite/i); - } - const productionRoutingFiles = [ 'packages/cli/src/sl.ts', 'packages/cli/src/knowledge.ts', - 'packages/context/src/mcp/local-project-ports.ts', - 'packages/context/src/wiki/local-knowledge.ts', - 'packages/context/src/ingest/context-evidence/sqlite-context-evidence-store.ts', + 'packages/cli/src/context/mcp/local-project-ports.ts', + 'packages/cli/src/context/wiki/local-knowledge.ts', + 'packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.ts', ]; for (const relativePath of productionRoutingFiles) { @@ -59,7 +49,7 @@ describe('PGlite hybrid search runtime boundary', () => { ); } - const localSlSource = readKtxFile('packages/context/src/sl/local-sl.ts'); + const localSlSource = readKtxFile('packages/cli/src/context/sl/local-sl.ts'); expect(localSlSource).toContain("input.backend === 'pglite-owner-prototype'"); expect(localSlSource).toContain('PGlite semantic-layer search prototype requires pglite owner-process options.'); expect(localSlSource).toContain("await import('./pglite-sl-search-prototype.js')"); diff --git a/packages/context/src/search/pglite-spike.test.ts b/packages/cli/src/context/search/pglite-spike.test.ts similarity index 97% rename from packages/context/src/search/pglite-spike.test.ts rename to packages/cli/src/context/search/pglite-spike.test.ts index e8dff54e..470000da 100644 --- a/packages/context/src/search/pglite-spike.test.ts +++ b/packages/cli/src/context/search/pglite-spike.test.ts @@ -5,11 +5,8 @@ import { PGlite, type PGliteInterface } from '@electric-sql/pglite'; import { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm'; import { vector } from '@electric-sql/pglite/vector'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { - assertSearchBackendCapabilities, - assertSearchBackendConformanceCase, - type SearchBackendCapabilities, -} from './index.js'; +import { assertSearchBackendCapabilities, assertSearchBackendConformanceCase } from '../../context/search/backend-conformance.test-utils.js'; +import type { SearchBackendCapabilities } from '../../context/search/types.js'; type PGliteDb = PGliteInterface; diff --git a/packages/context/src/search/query.test.ts b/packages/cli/src/context/search/query.test.ts similarity index 100% rename from packages/context/src/search/query.test.ts rename to packages/cli/src/context/search/query.test.ts diff --git a/packages/context/src/search/query.ts b/packages/cli/src/context/search/query.ts similarity index 100% rename from packages/context/src/search/query.ts rename to packages/cli/src/context/search/query.ts diff --git a/packages/context/src/search/rrf.test.ts b/packages/cli/src/context/search/rrf.test.ts similarity index 100% rename from packages/context/src/search/rrf.test.ts rename to packages/cli/src/context/search/rrf.test.ts diff --git a/packages/context/src/search/rrf.ts b/packages/cli/src/context/search/rrf.ts similarity index 100% rename from packages/context/src/search/rrf.ts rename to packages/cli/src/context/search/rrf.ts diff --git a/packages/context/src/search/types.ts b/packages/cli/src/context/search/types.ts similarity index 92% rename from packages/context/src/search/types.ts rename to packages/cli/src/context/search/types.ts index 658961f0..8080ddfa 100644 --- a/packages/context/src/search/types.ts +++ b/packages/cli/src/context/search/types.ts @@ -1,5 +1,6 @@ export type SearchLaneName = 'lexical' | 'semantic' | 'dictionary' | 'token' | string; +/** @internal */ export type SearchLaneStatus = 'available' | 'skipped' | 'failed'; export interface NormalizedSearchQuery { @@ -16,7 +17,7 @@ export interface SearchCandidate { evidence?: unknown; } -export interface SearchCandidateGeneratorArgs { +interface SearchCandidateGeneratorArgs { queryText: string; normalizedQuery: NormalizedSearchQuery; finalLimit: number; @@ -64,9 +65,6 @@ export interface FusedSearchCandidate { evidenceByLane: Record; } -export interface SearchResultHydrator { - hydrate(candidates: FusedSearchCandidate[]): Promise; -} export interface HybridSearchResult { query: NormalizedSearchQuery; @@ -76,6 +74,7 @@ export interface HybridSearchResult { lanes: SearchLaneBreakdown[]; } +/** @internal */ export interface SearchBackendCapabilities { fts: boolean; vector: boolean; diff --git a/packages/context/src/skills/skills-registry.service.test.ts b/packages/cli/src/context/skills/skills-registry.service.test.ts similarity index 100% rename from packages/context/src/skills/skills-registry.service.test.ts rename to packages/cli/src/context/skills/skills-registry.service.test.ts diff --git a/packages/context/src/skills/skills-registry.service.ts b/packages/cli/src/context/skills/skills-registry.service.ts similarity index 99% rename from packages/context/src/skills/skills-registry.service.ts rename to packages/cli/src/context/skills/skills-registry.service.ts index cd33e6d8..4e8bb5cb 100644 --- a/packages/context/src/skills/skills-registry.service.ts +++ b/packages/cli/src/context/skills/skills-registry.service.ts @@ -1,6 +1,6 @@ import { readFile, readdir, stat } from 'node:fs/promises'; import { join } from 'node:path'; -import { noopLogger, type KtxLogger } from '../core/index.js'; +import { noopLogger, type KtxLogger } from '../../context/core/config.js'; export type SkillCaller = 'research' | 'memory_agent'; diff --git a/packages/context/src/sl/description-normalization.ts b/packages/cli/src/context/sl/description-normalization.ts similarity index 100% rename from packages/context/src/sl/description-normalization.ts rename to packages/cli/src/context/sl/description-normalization.ts diff --git a/packages/context/src/sl/descriptions.ts b/packages/cli/src/context/sl/descriptions.ts similarity index 100% rename from packages/context/src/sl/descriptions.ts rename to packages/cli/src/context/sl/descriptions.ts diff --git a/packages/context/src/sl/dictionary-search.test.ts b/packages/cli/src/context/sl/dictionary-search.test.ts similarity index 98% rename from packages/context/src/sl/dictionary-search.test.ts rename to packages/cli/src/context/sl/dictionary-search.test.ts index 7c3e2d1f..1838f0d9 100644 --- a/packages/context/src/sl/dictionary-search.test.ts +++ b/packages/cli/src/context/sl/dictionary-search.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { createKtxDictionarySearchService } from './dictionary-search.js'; describe('createKtxDictionarySearchService', () => { diff --git a/packages/context/src/sl/dictionary-search.ts b/packages/cli/src/context/sl/dictionary-search.ts similarity index 92% rename from packages/context/src/sl/dictionary-search.ts rename to packages/cli/src/context/sl/dictionary-search.ts index 041b828d..14899482 100644 --- a/packages/context/src/sl/dictionary-search.ts +++ b/packages/cli/src/context/sl/dictionary-search.ts @@ -1,15 +1,15 @@ -import type { KtxLocalProject } from '../project/index.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import { loadLatestSlDictionaryEntries, type SlDictionaryEntry } from './sl-dictionary-profile.js'; -export type KtxDictionarySearchStatus = 'ready' | 'no_profile_artifact' | 'no_candidate_columns'; -export type KtxDictionarySearchMissReason = 'no_profile_artifact' | 'no_candidate_columns' | 'value_not_in_sample'; +type KtxDictionarySearchStatus = 'ready' | 'no_profile_artifact' | 'no_candidate_columns'; +type KtxDictionarySearchMissReason = 'no_profile_artifact' | 'no_candidate_columns' | 'value_not_in_sample'; export interface KtxDictionarySearchInput { values: string[]; connectionId?: string; } -export interface KtxDictionarySearchCoverage { +interface KtxDictionarySearchCoverage { sampledRows: number | null; valuesPerColumn: number | null; profiledColumns: number; @@ -17,13 +17,13 @@ export interface KtxDictionarySearchCoverage { profiledAt: string | null; } -export interface KtxDictionarySearchSearchedConnection { +interface KtxDictionarySearchSearchedConnection { connectionId: string; coverage: KtxDictionarySearchCoverage; status: KtxDictionarySearchStatus; } -export interface KtxDictionarySearchMatch { +interface KtxDictionarySearchMatch { connectionId: string; sourceName: string; columnName: string; @@ -31,12 +31,12 @@ export interface KtxDictionarySearchMatch { cardinality: number | null; } -export interface KtxDictionarySearchMiss { +interface KtxDictionarySearchMiss { connectionId: string; reason: KtxDictionarySearchMissReason; } -export interface KtxDictionarySearchValueResult { +interface KtxDictionarySearchValueResult { value: string; matches: KtxDictionarySearchMatch[]; misses: KtxDictionarySearchMiss[]; diff --git a/packages/context/src/sl/local-query.test.ts b/packages/cli/src/context/sl/local-query.test.ts similarity index 98% rename from packages/context/src/sl/local-query.test.ts rename to packages/cli/src/context/sl/local-query.test.ts index b4703fe6..800bdb95 100644 --- a/packages/context/src/sl/local-query.test.ts +++ b/packages/cli/src/context/sl/local-query.test.ts @@ -2,8 +2,8 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { KtxSemanticLayerComputePort } from '../daemon/index.js'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { compileLocalSlQuery } from './local-query.js'; describe('compileLocalSlQuery', () => { diff --git a/packages/context/src/sl/local-query.ts b/packages/cli/src/context/sl/local-query.ts similarity index 94% rename from packages/context/src/sl/local-query.ts rename to packages/cli/src/context/sl/local-query.ts index 05a264fc..4d71504e 100644 --- a/packages/context/src/sl/local-query.ts +++ b/packages/cli/src/context/sl/local-query.ts @@ -1,7 +1,7 @@ -import type { KtxSqlQueryExecutorPort } from '../connections/index.js'; -import type { KtxSemanticLayerComputePort } from '../daemon/index.js'; +import type { KtxSqlQueryExecutorPort } from '../../context/connections/query-executor.js'; +import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js'; import type { KtxMcpProgressCallback } from '../mcp/types.js'; -import type { KtxLocalProject } from '../project/index.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import { loadLocalSlSourceRecords } from './local-sl.js'; import { toResolvedWire } from './semantic-layer.service.js'; import type { SemanticLayerQueryExecutionResult, SemanticLayerQueryInput } from './types.js'; diff --git a/packages/context/src/sl/local-sl.test.ts b/packages/cli/src/context/sl/local-sl.test.ts similarity index 99% rename from packages/context/src/sl/local-sl.test.ts rename to packages/cli/src/context/sl/local-sl.test.ts index 00c00874..18cc7392 100644 --- a/packages/context/src/sl/local-sl.test.ts +++ b/packages/cli/src/context/sl/local-sl.test.ts @@ -2,7 +2,7 @@ import { access, mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { listLocalSlSources, readLocalSlSource, diff --git a/packages/context/src/sl/local-sl.ts b/packages/cli/src/context/sl/local-sl.ts similarity index 98% rename from packages/context/src/sl/local-sl.ts rename to packages/cli/src/context/sl/local-sl.ts index f4844b7b..18ec8417 100644 --- a/packages/context/src/sl/local-sl.ts +++ b/packages/cli/src/context/sl/local-sl.ts @@ -1,9 +1,11 @@ import { join } from 'node:path'; import YAML from 'yaml'; import { z } from 'zod'; -import type { KtxEmbeddingPort, KtxFileWriteResult } from '../core/index.js'; -import type { KtxLocalProject } from '../project/index.js'; -import { HybridSearchCore, type SearchCandidateGenerator } from '../search/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxFileWriteResult } from '../../context/core/file-store.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { HybridSearchCore } from '../../context/search/hybrid-search-core.js'; +import type { SearchCandidateGenerator } from '../../context/search/types.js'; import { DEFAULT_PRIORITY, resolveDescription } from './descriptions.js'; import { normalizeSemanticLayerDescriptions } from './description-normalization.js'; import { sourceDefinitionSchema, sourceOverlaySchema } from './schemas.js'; @@ -301,6 +303,7 @@ export async function validateLocalSlSource( } } +/** @internal */ export async function writeLocalSlSource( project: KtxLocalProject, input: { connectionId: string; sourceName: string; yaml: string }, diff --git a/packages/context/src/sl/pglite-sl-search-prototype.test.ts b/packages/cli/src/context/sl/pglite-sl-search-prototype.test.ts similarity index 97% rename from packages/context/src/sl/pglite-sl-search-prototype.test.ts rename to packages/cli/src/context/sl/pglite-sl-search-prototype.test.ts index 29c81062..372f8668 100644 --- a/packages/context/src/sl/pglite-sl-search-prototype.test.ts +++ b/packages/cli/src/context/sl/pglite-sl-search-prototype.test.ts @@ -3,8 +3,8 @@ import { createServer } from 'node:net'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; -import { assertSearchBackendConformanceCase } from '../search/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; +import { assertSearchBackendConformanceCase } from '../../context/search/backend-conformance.test-utils.js'; import { searchLocalSlSources, writeLocalSlSource, type LocalSlSourceSearchResult } from './local-sl.js'; import { searchLocalSlSourcesWithPglitePrototype } from './pglite-sl-search-prototype.js'; diff --git a/packages/context/src/sl/pglite-sl-search-prototype.ts b/packages/cli/src/context/sl/pglite-sl-search-prototype.ts similarity index 98% rename from packages/context/src/sl/pglite-sl-search-prototype.ts rename to packages/cli/src/context/sl/pglite-sl-search-prototype.ts index 4a521437..95d07505 100644 --- a/packages/context/src/sl/pglite-sl-search-prototype.ts +++ b/packages/cli/src/context/sl/pglite-sl-search-prototype.ts @@ -1,8 +1,9 @@ import { mkdir } from 'node:fs/promises'; import { join } from 'node:path'; -import type { KtxEmbeddingPort } from '../core/index.js'; -import type { KtxLocalProject } from '../project/index.js'; -import { HybridSearchCore, type SearchCandidateGenerator } from '../search/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { HybridSearchCore } from '../../context/search/hybrid-search-core.js'; +import type { SearchCandidateGenerator } from '../../context/search/types.js'; import { KtxPGliteOwnerProcess } from '../search/pglite-owner-process.js'; import { listLocalSlSources, diff --git a/packages/context/src/sl/ports.ts b/packages/cli/src/context/sl/ports.ts similarity index 100% rename from packages/context/src/sl/ports.ts rename to packages/cli/src/context/sl/ports.ts diff --git a/packages/context/src/sl/schemas.contract.test.ts b/packages/cli/src/context/sl/schemas.contract.test.ts similarity index 100% rename from packages/context/src/sl/schemas.contract.test.ts rename to packages/cli/src/context/sl/schemas.contract.test.ts diff --git a/packages/context/src/sl/schemas.ts b/packages/cli/src/context/sl/schemas.ts similarity index 100% rename from packages/context/src/sl/schemas.ts rename to packages/cli/src/context/sl/schemas.ts diff --git a/packages/context/src/sl/semantic-layer.service.test.ts b/packages/cli/src/context/sl/semantic-layer.service.test.ts similarity index 100% rename from packages/context/src/sl/semantic-layer.service.test.ts rename to packages/cli/src/context/sl/semantic-layer.service.test.ts diff --git a/packages/context/src/sl/semantic-layer.service.ts b/packages/cli/src/context/sl/semantic-layer.service.ts similarity index 99% rename from packages/context/src/sl/semantic-layer.service.ts rename to packages/cli/src/context/sl/semantic-layer.service.ts index 2ab96688..e6afdeaf 100644 --- a/packages/context/src/sl/semantic-layer.service.ts +++ b/packages/cli/src/context/sl/semantic-layer.service.ts @@ -1,6 +1,7 @@ import YAML from 'yaml'; -import type { KtxFileStorePort, KtxLogger } from '../core/index.js'; -import { noopLogger } from '../core/index.js'; +import type { KtxFileStorePort } from '../../context/core/file-store.js'; +import type { KtxLogger } from '../../context/core/config.js'; +import { noopLogger } from '../../context/core/config.js'; import type { TableUsageOutput } from '../ingest/adapters/historic-sql/skill-schemas.js'; import type { SlConnectionCatalogPort, SlPythonPort } from './ports.js'; import { normalizeSemanticLayerDescriptions } from './description-normalization.js'; @@ -25,8 +26,11 @@ export interface LoadAllSourcesResult { loadErrors: string[]; } +/** @internal */ export class UnknownColumnOverrideError extends Error {} +/** @internal */ export class ColumnNameCollisionError extends Error {} +/** @internal */ export class ConflictingExcludeAndOverrideError extends Error {} class ComposeContractError extends Error {} @@ -1417,6 +1421,7 @@ function parseJoinColumns( * Returns one message per measure-level segment reference that doesn't resolve to * a segment defined on the source. Array is empty when every reference checks out. */ +/** @internal */ export function findDanglingSegmentRefs(source: Record): string[] { const segmentDefs = (source.segments as Array<{ name: string }> | undefined) ?? []; const segmentNames = new Set(segmentDefs.map((s) => s.name)); @@ -1584,6 +1589,7 @@ function parseJoinOn( * matching manifest column (by name). Local values always win. Columns absent from * the manifest pass through unchanged. Returns a new source; does not mutate input. */ +/** @internal */ export function enrichColumnsFromManifest( source: SemanticLayerSource, manifestEntry: SemanticLayerSource | null | undefined, diff --git a/packages/context/src/sl/sl-dictionary-profile.test.ts b/packages/cli/src/context/sl/sl-dictionary-profile.test.ts similarity index 97% rename from packages/context/src/sl/sl-dictionary-profile.test.ts rename to packages/cli/src/context/sl/sl-dictionary-profile.test.ts index 21400a71..f7aa3854 100644 --- a/packages/context/src/sl/sl-dictionary-profile.test.ts +++ b/packages/cli/src/context/sl/sl-dictionary-profile.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { loadLatestSlDictionaryEntries } from './sl-dictionary-profile.js'; describe('loadLatestSlDictionaryEntries', () => { diff --git a/packages/context/src/sl/sl-dictionary-profile.ts b/packages/cli/src/context/sl/sl-dictionary-profile.ts similarity index 96% rename from packages/context/src/sl/sl-dictionary-profile.ts rename to packages/cli/src/context/sl/sl-dictionary-profile.ts index 141d71ce..53798318 100644 --- a/packages/context/src/sl/sl-dictionary-profile.ts +++ b/packages/cli/src/context/sl/sl-dictionary-profile.ts @@ -1,5 +1,5 @@ -import type { KtxLocalProject } from '../project/index.js'; -import { defaultKtxDataDictionarySettings, isKtxDataDictionaryCandidate } from '../scan/index.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { defaultKtxDataDictionarySettings, isKtxDataDictionaryCandidate } from '../../context/scan/data-dictionary.js'; export interface SlDictionaryEntry { connectionId: string; diff --git a/packages/context/src/sl/sl-search.service.test.ts b/packages/cli/src/context/sl/sl-search.service.test.ts similarity index 100% rename from packages/context/src/sl/sl-search.service.test.ts rename to packages/cli/src/context/sl/sl-search.service.test.ts diff --git a/packages/context/src/sl/sl-search.service.ts b/packages/cli/src/context/sl/sl-search.service.ts similarity index 97% rename from packages/context/src/sl/sl-search.service.ts rename to packages/cli/src/context/sl/sl-search.service.ts index 0a7ecfb5..414f6011 100644 --- a/packages/context/src/sl/sl-search.service.ts +++ b/packages/cli/src/context/sl/sl-search.service.ts @@ -1,5 +1,6 @@ -import type { KtxEmbeddingPort, KtxLogger } from '../core/index.js'; -import { noopLogger } from '../core/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxLogger } from '../../context/core/config.js'; +import { noopLogger } from '../../context/core/config.js'; import type { ReindexWorkResult } from '../index-sync/types.js'; import { DEFAULT_PRIORITY, resolveDescription } from './descriptions.js'; import { normalizeSemanticLayerDescriptions } from './description-normalization.js'; diff --git a/packages/context/src/sl/sl-validator.port.ts b/packages/cli/src/context/sl/sl-validator.port.ts similarity index 84% rename from packages/context/src/sl/sl-validator.port.ts rename to packages/cli/src/context/sl/sl-validator.port.ts index 83a29968..0b9ecf37 100644 --- a/packages/context/src/sl/sl-validator.port.ts +++ b/packages/cli/src/context/sl/sl-validator.port.ts @@ -1,4 +1,4 @@ -export interface SlValidationResult { +interface SlValidationResult { errors: string[]; warnings: string[]; } diff --git a/packages/context/src/sl/sqlite-sl-sources-index.test.ts b/packages/cli/src/context/sl/sqlite-sl-sources-index.test.ts similarity index 100% rename from packages/context/src/sl/sqlite-sl-sources-index.test.ts rename to packages/cli/src/context/sl/sqlite-sl-sources-index.test.ts diff --git a/packages/context/src/sl/sqlite-sl-sources-index.ts b/packages/cli/src/context/sl/sqlite-sl-sources-index.ts similarity index 100% rename from packages/context/src/sl/sqlite-sl-sources-index.ts rename to packages/cli/src/context/sl/sqlite-sl-sources-index.ts diff --git a/packages/context/src/sl/tools/base-semantic-layer.tool.ts b/packages/cli/src/context/sl/tools/base-semantic-layer.tool.ts similarity index 95% rename from packages/context/src/sl/tools/base-semantic-layer.tool.ts rename to packages/cli/src/context/sl/tools/base-semantic-layer.tool.ts index 2ec6891f..22822368 100644 --- a/packages/context/src/sl/tools/base-semantic-layer.tool.ts +++ b/packages/cli/src/context/sl/tools/base-semantic-layer.tool.ts @@ -1,14 +1,13 @@ import type { ZodType } from 'zod'; -import type { GitAuthorResolverPort, ToolContext, ToolOutput } from '../../tools/index.js'; -import { BaseTool } from '../../tools/index.js'; +import type { GitAuthorResolverPort } from '../../../context/tools/authors.js'; +import type { ToolContext, ToolOutput } from '../../../context/tools/base-tool.js'; +import { BaseTool } from '../../../context/tools/base-tool.js'; import { sourceDefinitionSchema } from '../schemas.js'; import { SemanticLayerService } from '../semantic-layer.service.js'; import { SlSearchService } from '../sl-search.service.js'; export { sourceDefinitionSchema }; - // ── Shared output types ── - export interface SemanticLayerStructured { success: boolean; sourceName: string; diff --git a/packages/context/src/sl/tools/connection-id-schema.test.ts b/packages/cli/src/context/sl/tools/connection-id-schema.test.ts similarity index 100% rename from packages/context/src/sl/tools/connection-id-schema.test.ts rename to packages/cli/src/context/sl/tools/connection-id-schema.test.ts diff --git a/packages/context/src/sl/tools/connection-id-schema.ts b/packages/cli/src/context/sl/tools/connection-id-schema.ts similarity index 100% rename from packages/context/src/sl/tools/connection-id-schema.ts rename to packages/cli/src/context/sl/tools/connection-id-schema.ts diff --git a/packages/context/src/sl/tools/sl-discover.tool.test.ts b/packages/cli/src/context/sl/tools/sl-discover.tool.test.ts similarity index 92% rename from packages/context/src/sl/tools/sl-discover.tool.test.ts rename to packages/cli/src/context/sl/tools/sl-discover.tool.test.ts index 1b961141..6dc30478 100644 --- a/packages/context/src/sl/tools/sl-discover.tool.test.ts +++ b/packages/cli/src/context/sl/tools/sl-discover.tool.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolContext, ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources } from '../../tools/index.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; import type { SemanticLayerSource } from '../types.js'; import { SlDiscoverTool } from './sl-discover.tool.js'; diff --git a/packages/context/src/sl/tools/sl-discover.tool.ts b/packages/cli/src/context/sl/tools/sl-discover.tool.ts similarity index 99% rename from packages/context/src/sl/tools/sl-discover.tool.ts rename to packages/cli/src/context/sl/tools/sl-discover.tool.ts index fb55175d..a8b67699 100644 --- a/packages/context/src/sl/tools/sl-discover.tool.ts +++ b/packages/cli/src/context/sl/tools/sl-discover.tool.ts @@ -2,7 +2,7 @@ import { z } from 'zod'; import { DEFAULT_PRIORITY, resolveDescription } from '../descriptions.js'; import type { SemanticLayerService } from '../semantic-layer.service.js'; import type { SemanticLayerSource } from '../types.js'; -import type { ToolContext, ToolOutput } from '../../tools/index.js'; +import type { ToolContext, ToolOutput } from '../../../context/tools/base-tool.js'; import { BaseSemanticLayerTool, type BaseSemanticLayerToolDeps } from './base-semantic-layer.tool.js'; import { slToolConnectionIdSchema } from './connection-id-schema.js'; diff --git a/packages/context/src/sl/tools/sl-edit-source.tool.test.ts b/packages/cli/src/context/sl/tools/sl-edit-source.tool.test.ts similarity index 97% rename from packages/context/src/sl/tools/sl-edit-source.tool.test.ts rename to packages/cli/src/context/sl/tools/sl-edit-source.tool.test.ts index 75c753ef..cf66baf8 100644 --- a/packages/context/src/sl/tools/sl-edit-source.tool.test.ts +++ b/packages/cli/src/context/sl/tools/sl-edit-source.tool.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources, hasTouchedSlSource, type ToolContext } from '../../tools/index.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources, hasTouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import { SlEditSourceTool } from './sl-edit-source.tool.js'; function makeTool(overrides: any = {}) { diff --git a/packages/context/src/sl/tools/sl-edit-source.tool.ts b/packages/cli/src/context/sl/tools/sl-edit-source.tool.ts similarity index 96% rename from packages/context/src/sl/tools/sl-edit-source.tool.ts rename to packages/cli/src/context/sl/tools/sl-edit-source.tool.ts index f6669120..94abbb36 100644 --- a/packages/context/src/sl/tools/sl-edit-source.tool.ts +++ b/packages/cli/src/context/sl/tools/sl-edit-source.tool.ts @@ -1,12 +1,9 @@ import YAML from 'yaml'; import { z } from 'zod'; -import { - addTouchedSlSource, - type ToolContext, - type ToolOutput, - validateActionRawPaths, - validateActionTargetConnection, -} from '../../tools/index.js'; +import { addTouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext, ToolOutput } from '../../../context/tools/base-tool.js'; +import { validateActionRawPaths } from '../../../context/tools/action-raw-paths.js'; +import { validateActionTargetConnection } from '../../../context/tools/action-target-connection.js'; import { applySqlEdits } from '../../tools/sql-edit-replacer.js'; import { normalizeSemanticLayerDescriptions } from '../description-normalization.js'; import type { SemanticLayerSource } from '../types.js'; diff --git a/packages/context/src/sl/tools/sl-read-source.tool.session.test.ts b/packages/cli/src/context/sl/tools/sl-read-source.tool.session.test.ts similarity index 91% rename from packages/context/src/sl/tools/sl-read-source.tool.session.test.ts rename to packages/cli/src/context/sl/tools/sl-read-source.tool.session.test.ts index dcb2a919..481c4cfe 100644 --- a/packages/context/src/sl/tools/sl-read-source.tool.session.test.ts +++ b/packages/cli/src/context/sl/tools/sl-read-source.tool.session.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources, type ToolContext } from '../../tools/index.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import { SlReadSourceTool } from './sl-read-source.tool.js'; function makeTool(overrides: Partial> = {}) { diff --git a/packages/context/src/sl/tools/sl-read-source.tool.ts b/packages/cli/src/context/sl/tools/sl-read-source.tool.ts similarity index 96% rename from packages/context/src/sl/tools/sl-read-source.tool.ts rename to packages/cli/src/context/sl/tools/sl-read-source.tool.ts index e4602f31..d037f4f1 100644 --- a/packages/context/src/sl/tools/sl-read-source.tool.ts +++ b/packages/cli/src/context/sl/tools/sl-read-source.tool.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import type { ToolContext, ToolOutput } from '../../tools/index.js'; +import type { ToolContext, ToolOutput } from '../../../context/tools/base-tool.js'; import { BaseSemanticLayerTool, type BaseSemanticLayerToolDeps } from './base-semantic-layer.tool.js'; import { slToolConnectionIdSchema } from './connection-id-schema.js'; diff --git a/packages/context/src/sl/tools/sl-rollback.tool.test.ts b/packages/cli/src/context/sl/tools/sl-rollback.tool.test.ts similarity index 92% rename from packages/context/src/sl/tools/sl-rollback.tool.test.ts rename to packages/cli/src/context/sl/tools/sl-rollback.tool.test.ts index 73461e87..5a1927a4 100644 --- a/packages/context/src/sl/tools/sl-rollback.tool.test.ts +++ b/packages/cli/src/context/sl/tools/sl-rollback.tool.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources, hasTouchedSlSource, type ToolContext } from '../../tools/index.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources, hasTouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import { SlRollbackTool } from './sl-rollback.tool.js'; function makeSession(overrides: Partial = {}): ToolSession { diff --git a/packages/context/src/sl/tools/sl-rollback.tool.ts b/packages/cli/src/context/sl/tools/sl-rollback.tool.ts similarity index 93% rename from packages/context/src/sl/tools/sl-rollback.tool.ts rename to packages/cli/src/context/sl/tools/sl-rollback.tool.ts index f3354ac1..bbafa380 100644 --- a/packages/context/src/sl/tools/sl-rollback.tool.ts +++ b/packages/cli/src/context/sl/tools/sl-rollback.tool.ts @@ -1,5 +1,6 @@ import { z } from 'zod'; -import { BaseTool, deleteTouchedSlSource, type ToolContext, type ToolOutput } from '../../tools/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../context/tools/base-tool.js'; +import { deleteTouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; import type { SlConnectionCatalogPort, SlSourcesIndexPort } from '../ports.js'; import { revertSourceToPreHead } from './sl-warehouse-validation.js'; diff --git a/packages/context/src/sl/tools/sl-validate.tool.test.ts b/packages/cli/src/context/sl/tools/sl-validate.tool.test.ts similarity index 92% rename from packages/context/src/sl/tools/sl-validate.tool.test.ts rename to packages/cli/src/context/sl/tools/sl-validate.tool.test.ts index b6725c08..f0c18eac 100644 --- a/packages/context/src/sl/tools/sl-validate.tool.test.ts +++ b/packages/cli/src/context/sl/tools/sl-validate.tool.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources, type ToolContext } from '../../tools/index.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import type { SemanticLayerService } from '../semantic-layer.service.js'; import type { SemanticLayerSource } from '../types.js'; import { SlValidateTool, validateSemanticLayerEndpoint } from './sl-validate.tool.js'; diff --git a/packages/context/src/sl/tools/sl-validate.tool.ts b/packages/cli/src/context/sl/tools/sl-validate.tool.ts similarity index 95% rename from packages/context/src/sl/tools/sl-validate.tool.ts rename to packages/cli/src/context/sl/tools/sl-validate.tool.ts index 8117fbcf..c3690c6f 100644 --- a/packages/context/src/sl/tools/sl-validate.tool.ts +++ b/packages/cli/src/context/sl/tools/sl-validate.tool.ts @@ -1,5 +1,6 @@ import { z } from 'zod'; -import { type ToolContext, type ToolOutput, touchedSlSourceNamesForConnection } from '../../tools/index.js'; +import type { ToolContext, ToolOutput } from '../../../context/tools/base-tool.js'; +import { touchedSlSourceNamesForConnection } from '../../../context/tools/touched-sl-sources.js'; import { SemanticLayerService } from '../semantic-layer.service.js'; import { BaseSemanticLayerTool, @@ -19,6 +20,7 @@ type ValidationReport = { warnings: string[]; }; +/** @internal */ export async function validateSemanticLayerEndpoint( connectionId: string, semanticLayerService: SemanticLayerService, diff --git a/packages/context/src/sl/tools/sl-warehouse-validation.test.ts b/packages/cli/src/context/sl/tools/sl-warehouse-validation.test.ts similarity index 100% rename from packages/context/src/sl/tools/sl-warehouse-validation.test.ts rename to packages/cli/src/context/sl/tools/sl-warehouse-validation.test.ts diff --git a/packages/context/src/sl/tools/sl-warehouse-validation.ts b/packages/cli/src/context/sl/tools/sl-warehouse-validation.ts similarity index 97% rename from packages/context/src/sl/tools/sl-warehouse-validation.ts rename to packages/cli/src/context/sl/tools/sl-warehouse-validation.ts index 36a97647..742d855a 100644 --- a/packages/context/src/sl/tools/sl-warehouse-validation.ts +++ b/packages/cli/src/context/sl/tools/sl-warehouse-validation.ts @@ -1,6 +1,7 @@ import YAML from 'yaml'; -import type { GitService, KtxFileStorePort } from '../../core/index.js'; -import { SYSTEM_GIT_AUTHOR } from '../../tools/index.js'; +import type { GitService } from '../../../context/core/git.service.js'; +import type { KtxFileStorePort } from '../../../context/core/file-store.js'; +import { SYSTEM_GIT_AUTHOR } from '../../../context/tools/authors.js'; import type { SlConnectionCatalogPort, SlSourcesIndexPort } from '../ports.js'; import { sourceOverlaySchema } from '../schemas.js'; import { SemanticLayerService } from '../semantic-layer.service.js'; @@ -16,6 +17,7 @@ export interface SlValidationDeps { probeRowCount: number; } +/** @internal */ export interface SourceValidationResult { errors: string[]; warnings: string[]; @@ -52,6 +54,7 @@ function wrapWithSingleRowQuery(sql: string, dialect: string): string { * Returns errors and hint-style warnings. An empty errors array means the YAML is * structurally valid AND the warehouse can execute a probe against its embedded sql. */ +/** @internal */ export async function validateSingleSource( deps: SlValidationDeps, connectionId: string, diff --git a/packages/context/src/sl/tools/sl-write-source.tool.test.ts b/packages/cli/src/context/sl/tools/sl-write-source.tool.test.ts similarity index 98% rename from packages/context/src/sl/tools/sl-write-source.tool.test.ts rename to packages/cli/src/context/sl/tools/sl-write-source.tool.test.ts index 186028b8..f168095c 100644 --- a/packages/context/src/sl/tools/sl-write-source.tool.test.ts +++ b/packages/cli/src/context/sl/tools/sl-write-source.tool.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources, hasTouchedSlSource, type ToolContext } from '../../tools/index.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources, hasTouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import { SlWriteSourceTool } from './sl-write-source.tool.js'; function makeTool(overrides: Partial> = {}) { diff --git a/packages/context/src/sl/tools/sl-write-source.tool.ts b/packages/cli/src/context/sl/tools/sl-write-source.tool.ts similarity index 97% rename from packages/context/src/sl/tools/sl-write-source.tool.ts rename to packages/cli/src/context/sl/tools/sl-write-source.tool.ts index b9a79e6b..1ff0f7ee 100644 --- a/packages/context/src/sl/tools/sl-write-source.tool.ts +++ b/packages/cli/src/context/sl/tools/sl-write-source.tool.ts @@ -1,12 +1,9 @@ import YAML from 'yaml'; import { z } from 'zod'; -import { - addTouchedSlSource, - type ToolContext, - type ToolOutput, - validateActionRawPaths, - validateActionTargetConnection, -} from '../../tools/index.js'; +import { addTouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext, ToolOutput } from '../../../context/tools/base-tool.js'; +import { validateActionRawPaths } from '../../../context/tools/action-raw-paths.js'; +import { validateActionTargetConnection } from '../../../context/tools/action-target-connection.js'; import { sourceOverlaySchema } from '../schemas.js'; import type { SemanticLayerService } from '../semantic-layer.service.js'; import type { SemanticLayerSource } from '../types.js'; diff --git a/packages/context/src/sl/types.ts b/packages/cli/src/context/sl/types.ts similarity index 94% rename from packages/context/src/sl/types.ts rename to packages/cli/src/context/sl/types.ts index da1ea9aa..cc9575b7 100644 --- a/packages/context/src/sl/types.ts +++ b/packages/cli/src/context/sl/types.ts @@ -108,10 +108,3 @@ export interface SlSearchLaneSummary { weight: number; reason?: string; } - -export interface SlSearchMetadata { - score: number; - matchReasons: SlSearchMatchReason[]; - dictionaryMatches?: SlDictionaryMatch[]; - lanes?: SlSearchLaneSummary[]; -} diff --git a/packages/context/src/sql-analysis/http-sql-analysis-port.test.ts b/packages/cli/src/context/sql-analysis/http-sql-analysis-port.test.ts similarity index 100% rename from packages/context/src/sql-analysis/http-sql-analysis-port.test.ts rename to packages/cli/src/context/sql-analysis/http-sql-analysis-port.test.ts diff --git a/packages/context/src/sql-analysis/http-sql-analysis-port.ts b/packages/cli/src/context/sql-analysis/http-sql-analysis-port.ts similarity index 100% rename from packages/context/src/sql-analysis/http-sql-analysis-port.ts rename to packages/cli/src/context/sql-analysis/http-sql-analysis-port.ts diff --git a/packages/context/src/sql-analysis/ports.ts b/packages/cli/src/context/sql-analysis/ports.ts similarity index 92% rename from packages/context/src/sql-analysis/ports.ts rename to packages/cli/src/context/sql-analysis/ports.ts index 891515b7..887be605 100644 --- a/packages/context/src/sql-analysis/ports.ts +++ b/packages/cli/src/context/sql-analysis/ports.ts @@ -25,7 +25,7 @@ export interface SqlAnalysisFingerprintResult { error?: string | null; } -export type SqlAnalysisClause = 'select' | 'where' | 'join' | 'groupBy' | 'having' | 'orderBy' | (string & {}); +type SqlAnalysisClause = 'select' | 'where' | 'join' | 'groupBy' | 'having' | 'orderBy' | (string & {}); export interface SqlAnalysisBatchItem { id: string; diff --git a/packages/context/src/test/make-local-git-repo.ts b/packages/cli/src/context/test/make-local-git-repo.ts similarity index 100% rename from packages/context/src/test/make-local-git-repo.ts rename to packages/cli/src/context/test/make-local-git-repo.ts diff --git a/packages/context/src/tools/action-raw-paths.ts b/packages/cli/src/context/tools/action-raw-paths.ts similarity index 100% rename from packages/context/src/tools/action-raw-paths.ts rename to packages/cli/src/context/tools/action-raw-paths.ts diff --git a/packages/context/src/tools/action-target-connection.ts b/packages/cli/src/context/tools/action-target-connection.ts similarity index 100% rename from packages/context/src/tools/action-target-connection.ts rename to packages/cli/src/context/tools/action-target-connection.ts diff --git a/packages/context/src/tools/authors.ts b/packages/cli/src/context/tools/authors.ts similarity index 100% rename from packages/context/src/tools/authors.ts rename to packages/cli/src/context/tools/authors.ts diff --git a/packages/context/src/tools/base-tool.ts b/packages/cli/src/context/tools/base-tool.ts similarity index 97% rename from packages/context/src/tools/base-tool.ts rename to packages/cli/src/context/tools/base-tool.ts index faf27e59..d002ac42 100644 --- a/packages/context/src/tools/base-tool.ts +++ b/packages/cli/src/context/tools/base-tool.ts @@ -1,6 +1,6 @@ import { tool } from 'ai'; import { z, type ZodType } from 'zod'; -import { noopLogger, type KtxLogger } from '../core/index.js'; +import { noopLogger, type KtxLogger } from '../../context/core/config.js'; import type { KtxRuntimeToolDescriptor } from '../llm/runtime-port.js'; import { normalizeKtxRuntimeToolOutput } from '../llm/runtime-tools.js'; import type { IngestToolMetadata, ToolSession } from './tool-session.js'; @@ -10,12 +10,12 @@ export interface ToolOutput { structured: T; } -export interface ToolTimingTrackerPort { +interface ToolTimingTrackerPort { recordToolExecutionStart(messageId: string, toolName: string, toolCallId: string): void; recordToolExecutionEnd(messageId: string, toolName: string, toolCallId: string, state: string): void; } -export interface ToolProgressRelayPort { +interface ToolProgressRelayPort { emit(event: unknown): void; } @@ -62,7 +62,7 @@ export interface ToolContext { }; } -export interface MethodologyEntry { +interface MethodologyEntry { key: string; toolName: string; label: string; diff --git a/packages/context/src/tools/context-candidate-mark.tool.ts b/packages/cli/src/context/tools/context-candidate-mark.tool.ts similarity index 100% rename from packages/context/src/tools/context-candidate-mark.tool.ts rename to packages/cli/src/context/tools/context-candidate-mark.tool.ts diff --git a/packages/context/src/tools/context-candidate-write.tool.ts b/packages/cli/src/context/tools/context-candidate-write.tool.ts similarity index 97% rename from packages/context/src/tools/context-candidate-write.tool.ts rename to packages/cli/src/context/tools/context-candidate-write.tool.ts index 16a28707..8c2b0ea3 100644 --- a/packages/context/src/tools/context-candidate-write.tool.ts +++ b/packages/cli/src/context/tools/context-candidate-write.tool.ts @@ -1,7 +1,7 @@ import { createHash } from 'node:crypto'; import { z } from 'zod'; -import type { KtxEmbeddingPort } from '../core/index.js'; -import { buildContextCandidateEmbeddingText } from '../ingest/context-candidates/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import { buildContextCandidateEmbeddingText } from '../../context/ingest/context-candidates/embedding-text.js'; import { BaseTool, type ToolContext, type ToolOutput } from './base-tool.js'; import { chunkIdSchema } from './context-evidence-ids.js'; import type { ContextEvidenceToolStorePort } from './context-evidence-tool-store.js'; diff --git a/packages/context/src/tools/context-evidence-ids.ts b/packages/cli/src/context/tools/context-evidence-ids.ts similarity index 100% rename from packages/context/src/tools/context-evidence-ids.ts rename to packages/cli/src/context/tools/context-evidence-ids.ts diff --git a/packages/context/src/tools/context-evidence-neighbors.tool.ts b/packages/cli/src/context/tools/context-evidence-neighbors.tool.ts similarity index 100% rename from packages/context/src/tools/context-evidence-neighbors.tool.ts rename to packages/cli/src/context/tools/context-evidence-neighbors.tool.ts diff --git a/packages/context/src/tools/context-evidence-read.tool.ts b/packages/cli/src/context/tools/context-evidence-read.tool.ts similarity index 100% rename from packages/context/src/tools/context-evidence-read.tool.ts rename to packages/cli/src/context/tools/context-evidence-read.tool.ts diff --git a/packages/context/src/tools/context-evidence-search.tool.ts b/packages/cli/src/context/tools/context-evidence-search.tool.ts similarity index 98% rename from packages/context/src/tools/context-evidence-search.tool.ts rename to packages/cli/src/context/tools/context-evidence-search.tool.ts index cdd32734..51534f7a 100644 --- a/packages/context/src/tools/context-evidence-search.tool.ts +++ b/packages/cli/src/context/tools/context-evidence-search.tool.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import type { KtxEmbeddingPort } from '../core/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; import { BaseTool, type ToolContext, type ToolOutput } from './base-tool.js'; import type { ContextEvidenceToolStorePort } from './context-evidence-tool-store.js'; import { ingestMetadataRequired, resolveIngestMetadata, type ToolFailure } from './context-ingest-metadata.js'; diff --git a/packages/context/src/tools/context-evidence-tool-store.ts b/packages/cli/src/context/tools/context-evidence-tool-store.ts similarity index 92% rename from packages/context/src/tools/context-evidence-tool-store.ts rename to packages/cli/src/context/tools/context-evidence-tool-store.ts index f7100706..713a224a 100644 --- a/packages/context/src/tools/context-evidence-tool-store.ts +++ b/packages/cli/src/context/tools/context-evidence-tool-store.ts @@ -1,4 +1,4 @@ -import type { InsertContextCandidateInput } from '../ingest/context-candidates/index.js'; +import type { InsertContextCandidateInput } from '../../context/ingest/context-candidates/types.js'; import type { JsonValue } from '../ingest/ports.js'; export interface ContextEvidenceSearchArgs { @@ -13,7 +13,7 @@ export interface ContextEvidenceSearchArgs { export type ContextEvidenceSearchMatchReason = 'lexical' | 'semantic' | 'token' | (string & {}); -export interface ContextEvidenceSearchLaneSummary { +interface ContextEvidenceSearchLaneSummary { lane: string; status: 'available' | 'skipped' | 'failed'; requestedCandidatePoolLimit: number; @@ -40,7 +40,7 @@ export interface ContextEvidenceSearchResult { lanes?: ContextEvidenceSearchLaneSummary[]; } -export interface ContextEvidenceDocumentForRead { +interface ContextEvidenceDocumentForRead { id: string; title: string; path: string; @@ -48,7 +48,7 @@ export interface ContextEvidenceDocumentForRead { url: string | null; } -export interface ContextEvidenceChunkForRead { +interface ContextEvidenceChunkForRead { id: string; content: string; citation?: JsonValue; @@ -89,7 +89,7 @@ export interface ContextEvidenceChunkForCandidate { lastEditedAt: Date | null; } -export interface ContextCandidateInsertResult { +interface ContextCandidateInsertResult { id: string; candidate_key: string; promotion_score: number; diff --git a/packages/context/src/tools/context-evidence-tools.test.ts b/packages/cli/src/context/tools/context-evidence-tools.test.ts similarity index 98% rename from packages/context/src/tools/context-evidence-tools.test.ts rename to packages/cli/src/context/tools/context-evidence-tools.test.ts index 2a76f924..08a8654f 100644 --- a/packages/context/src/tools/context-evidence-tools.test.ts +++ b/packages/cli/src/context/tools/context-evidence-tools.test.ts @@ -3,7 +3,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { KtxEmbeddingPort } from '../core/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; import { SqliteContextEvidenceStore } from '../ingest/context-evidence/sqlite-context-evidence-store.js'; import { ContextCandidateMarkTool } from './context-candidate-mark.tool.js'; import { ContextCandidateWriteTool } from './context-candidate-write.tool.js'; @@ -11,7 +11,9 @@ import { ContextEvidenceNeighborsTool } from './context-evidence-neighbors.tool. import { ContextEvidenceReadTool } from './context-evidence-read.tool.js'; import { ContextEvidenceSearchTool } from './context-evidence-search.tool.js'; import type { ContextEvidenceToolStorePort } from './context-evidence-tool-store.js'; -import { createTouchedSlSources, type ToolContext, type ToolSession } from './index.js'; +import { createTouchedSlSources } from '../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../context/tools/base-tool.js'; +import type { ToolSession } from '../../context/tools/tool-session.js'; const ingestContext = (): ToolContext => ({ sourceId: 'ingest', diff --git a/packages/context/src/tools/context-ingest-metadata.ts b/packages/cli/src/context/tools/context-ingest-metadata.ts similarity index 100% rename from packages/context/src/tools/context-ingest-metadata.ts rename to packages/cli/src/context/tools/context-ingest-metadata.ts diff --git a/packages/context/src/tools/sql-edit-replacer.ts b/packages/cli/src/context/tools/sql-edit-replacer.ts similarity index 100% rename from packages/context/src/tools/sql-edit-replacer.ts rename to packages/cli/src/context/tools/sql-edit-replacer.ts diff --git a/packages/context/src/tools/tool-session.ts b/packages/cli/src/context/tools/tool-session.ts similarity index 83% rename from packages/context/src/tools/tool-session.ts rename to packages/cli/src/context/tools/tool-session.ts index 05da85d9..a1c95a89 100644 --- a/packages/context/src/tools/tool-session.ts +++ b/packages/cli/src/context/tools/tool-session.ts @@ -1,6 +1,7 @@ -import type { GitService, KtxFileStorePort } from '../core/index.js'; -import type { SemanticLayerService } from '../sl/index.js'; -import type { KnowledgeWikiService } from '../wiki/index.js'; +import type { GitService } from '../../context/core/git.service.js'; +import type { KtxFileStorePort } from '../../context/core/file-store.js'; +import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; import type { TouchedSlSourceSet } from './touched-sl-sources.js'; export interface IngestToolMetadata { diff --git a/packages/context/src/tools/touched-sl-sources.test.ts b/packages/cli/src/context/tools/touched-sl-sources.test.ts similarity index 100% rename from packages/context/src/tools/touched-sl-sources.test.ts rename to packages/cli/src/context/tools/touched-sl-sources.test.ts diff --git a/packages/context/src/tools/touched-sl-sources.ts b/packages/cli/src/context/tools/touched-sl-sources.ts similarity index 99% rename from packages/context/src/tools/touched-sl-sources.ts rename to packages/cli/src/context/tools/touched-sl-sources.ts index 44c3a834..75f080ae 100644 --- a/packages/context/src/tools/touched-sl-sources.ts +++ b/packages/cli/src/context/tools/touched-sl-sources.ts @@ -30,6 +30,7 @@ export function deleteTouchedSlSource(touched: TouchedSlSourceSet, connectionId: } } +/** @internal */ export function hasTouchedSlSource(touched: TouchedSlSourceSet, connectionId: string, sourceName: string): boolean { return touched.get(connectionId)?.has(sourceName) ?? false; } diff --git a/packages/context/src/wiki/keys.ts b/packages/cli/src/context/wiki/keys.ts similarity index 93% rename from packages/context/src/wiki/keys.ts rename to packages/cli/src/context/wiki/keys.ts index 4bffae2e..8af66373 100644 --- a/packages/context/src/wiki/keys.ts +++ b/packages/cli/src/context/wiki/keys.ts @@ -10,7 +10,7 @@ export function suggestFlatWikiKey(key: string): string { return suggested.length > 0 ? suggested : 'page-key'; } -export function invalidFlatWikiKeyMessage(key: string): string { +function invalidFlatWikiKeyMessage(key: string): string { return `Invalid wiki key "${key}". Wiki keys must be flat; use "${suggestFlatWikiKey(key)}".`; } diff --git a/packages/context/src/wiki/knowledge-search-text.ts b/packages/cli/src/context/wiki/knowledge-search-text.ts similarity index 100% rename from packages/context/src/wiki/knowledge-search-text.ts rename to packages/cli/src/context/wiki/knowledge-search-text.ts diff --git a/packages/context/src/wiki/knowledge-wiki.service.test.ts b/packages/cli/src/context/wiki/knowledge-wiki.service.test.ts similarity index 100% rename from packages/context/src/wiki/knowledge-wiki.service.test.ts rename to packages/cli/src/context/wiki/knowledge-wiki.service.test.ts diff --git a/packages/context/src/wiki/knowledge-wiki.service.ts b/packages/cli/src/context/wiki/knowledge-wiki.service.ts similarity index 98% rename from packages/context/src/wiki/knowledge-wiki.service.ts rename to packages/cli/src/context/wiki/knowledge-wiki.service.ts index d7a9f095..e87756c3 100644 --- a/packages/context/src/wiki/knowledge-wiki.service.ts +++ b/packages/cli/src/context/wiki/knowledge-wiki.service.ts @@ -1,7 +1,9 @@ import { createHash } from 'node:crypto'; import YAML from 'yaml'; -import type { KtxEmbeddingPort, KtxFileStorePort, KtxLogger } from '../core/index.js'; -import { noopLogger } from '../core/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxFileStorePort } from '../../context/core/file-store.js'; +import type { KtxLogger } from '../../context/core/config.js'; +import { noopLogger } from '../../context/core/config.js'; import type { ReindexWorkResult } from '../index-sync/types.js'; import { assertFlatWikiKey, isFlatWikiKey } from './keys.js'; import { buildKnowledgeSearchText } from './knowledge-search-text.js'; @@ -11,10 +13,8 @@ import type { WikiFrontmatter, WikiPage, WikiPageWithScope } from './types.js'; const WIKI_PREFIX = 'wiki'; export type { WikiFrontmatter }; - export class KnowledgeWikiService { private isWorktreeScoped = false; - constructor( private readonly configService: KtxFileStorePort, private readonly embeddingService: KtxEmbeddingPort | null, diff --git a/packages/context/src/wiki/local-knowledge.test.ts b/packages/cli/src/context/wiki/local-knowledge.test.ts similarity index 99% rename from packages/context/src/wiki/local-knowledge.test.ts rename to packages/cli/src/context/wiki/local-knowledge.test.ts index 2a166fc5..8229d5e7 100644 --- a/packages/context/src/wiki/local-knowledge.test.ts +++ b/packages/cli/src/context/wiki/local-knowledge.test.ts @@ -2,7 +2,7 @@ import { access, mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { listLocalKnowledgePages, readLocalKnowledgePage, diff --git a/packages/context/src/wiki/local-knowledge.ts b/packages/cli/src/context/wiki/local-knowledge.ts similarity index 96% rename from packages/context/src/wiki/local-knowledge.ts rename to packages/cli/src/context/wiki/local-knowledge.ts index b228cfd4..b7132b50 100644 --- a/packages/context/src/wiki/local-knowledge.ts +++ b/packages/cli/src/context/wiki/local-knowledge.ts @@ -1,8 +1,10 @@ import { join } from 'node:path'; import YAML from 'yaml'; -import type { KtxEmbeddingPort, KtxFileWriteResult } from '../core/index.js'; -import type { KtxLocalProject } from '../project/index.js'; -import { HybridSearchCore, type SearchCandidateGenerator } from '../search/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxFileWriteResult } from '../../context/core/file-store.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { HybridSearchCore } from '../../context/search/hybrid-search-core.js'; +import type { SearchCandidateGenerator } from '../../context/search/types.js'; import { buildKnowledgeSearchText } from './knowledge-search-text.js'; import { assertFlatWikiKey, isFlatWikiKey } from './keys.js'; import { SqliteKnowledgeIndex, type SqliteKnowledgeIndexPage } from './sqlite-knowledge-index.js'; @@ -34,6 +36,7 @@ export interface LocalKnowledgeSearchResult extends LocalKnowledgeSummary { lanes?: WikiSearchLaneSummary[]; } +/** @internal */ export interface WriteLocalKnowledgePageInput { key: string; scope: LocalKnowledgeScope; @@ -148,6 +151,7 @@ async function readPageAtPath( } } +/** @internal */ export async function writeLocalKnowledgePage( project: KtxLocalProject, input: WriteLocalKnowledgePageInput, diff --git a/packages/context/src/wiki/ports.ts b/packages/cli/src/context/wiki/ports.ts similarity index 94% rename from packages/context/src/wiki/ports.ts rename to packages/cli/src/context/wiki/ports.ts index 6bcb2f17..aa5d2fe1 100644 --- a/packages/context/src/wiki/ports.ts +++ b/packages/cli/src/context/wiki/ports.ts @@ -1,4 +1,3 @@ -import type { KtxFileStorePort } from '../core/file-store.js'; export interface UpsertPageParams { scope: string; @@ -71,5 +70,3 @@ export interface KnowledgeGitDiffPort { ): Promise>; getFileAtCommit(path: string, sha: string): Promise; } - -export type WikiFileStorePort = KtxFileStorePort; diff --git a/packages/context/src/wiki/sqlite-knowledge-index.test.ts b/packages/cli/src/context/wiki/sqlite-knowledge-index.test.ts similarity index 100% rename from packages/context/src/wiki/sqlite-knowledge-index.test.ts rename to packages/cli/src/context/wiki/sqlite-knowledge-index.test.ts diff --git a/packages/context/src/wiki/sqlite-knowledge-index.ts b/packages/cli/src/context/wiki/sqlite-knowledge-index.ts similarity index 100% rename from packages/context/src/wiki/sqlite-knowledge-index.ts rename to packages/cli/src/context/wiki/sqlite-knowledge-index.ts diff --git a/packages/context/src/wiki/tools/wiki-list-tags.tool.test.ts b/packages/cli/src/context/wiki/tools/wiki-list-tags.tool.test.ts similarity index 96% rename from packages/context/src/wiki/tools/wiki-list-tags.tool.test.ts rename to packages/cli/src/context/wiki/tools/wiki-list-tags.tool.test.ts index e4b5b7f3..49605c4f 100644 --- a/packages/context/src/wiki/tools/wiki-list-tags.tool.test.ts +++ b/packages/cli/src/context/wiki/tools/wiki-list-tags.tool.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolContext } from '../../tools/index.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import { WikiListTagsTool } from './wiki-list-tags.tool.js'; describe('WikiListTagsTool', () => { diff --git a/packages/context/src/wiki/tools/wiki-list-tags.tool.ts b/packages/cli/src/context/wiki/tools/wiki-list-tags.tool.ts similarity index 97% rename from packages/context/src/wiki/tools/wiki-list-tags.tool.ts rename to packages/cli/src/context/wiki/tools/wiki-list-tags.tool.ts index 3a31ee41..e7b358e8 100644 --- a/packages/context/src/wiki/tools/wiki-list-tags.tool.ts +++ b/packages/cli/src/context/wiki/tools/wiki-list-tags.tool.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; import type { KnowledgeIndexPort } from '../ports.js'; -import { BaseTool, type ToolContext, type ToolOutput } from '../../tools/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../context/tools/base-tool.js'; const wikiListTagsInputSchema = z.object({}); diff --git a/packages/context/src/wiki/tools/wiki-read.tool.test.ts b/packages/cli/src/context/wiki/tools/wiki-read.tool.test.ts similarity index 93% rename from packages/context/src/wiki/tools/wiki-read.tool.test.ts rename to packages/cli/src/context/wiki/tools/wiki-read.tool.test.ts index 1457702c..ac75b174 100644 --- a/packages/context/src/wiki/tools/wiki-read.tool.test.ts +++ b/packages/cli/src/context/wiki/tools/wiki-read.tool.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources, type ToolContext } from '../../tools/index.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import { WikiReadTool } from './wiki-read.tool.js'; describe('WikiReadTool', () => { diff --git a/packages/context/src/wiki/tools/wiki-read.tool.ts b/packages/cli/src/context/wiki/tools/wiki-read.tool.ts similarity index 95% rename from packages/context/src/wiki/tools/wiki-read.tool.ts rename to packages/cli/src/context/wiki/tools/wiki-read.tool.ts index 858e717e..2600d0ad 100644 --- a/packages/context/src/wiki/tools/wiki-read.tool.ts +++ b/packages/cli/src/context/wiki/tools/wiki-read.tool.ts @@ -1,8 +1,8 @@ import { z } from 'zod'; import type { KnowledgeIndexPort } from '../ports.js'; -import { KnowledgeWikiService } from '../index.js'; +import { KnowledgeWikiService } from '../../../context/wiki/knowledge-wiki.service.js'; import { validateFlatWikiKey } from '../keys.js'; -import { BaseTool, type ToolContext, type ToolOutput } from '../../tools/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../context/tools/base-tool.js'; const WikiReadInputSchema = z.object({ key: z diff --git a/packages/context/src/wiki/tools/wiki-remove.tool.test.ts b/packages/cli/src/context/wiki/tools/wiki-remove.tool.test.ts similarity index 95% rename from packages/context/src/wiki/tools/wiki-remove.tool.test.ts rename to packages/cli/src/context/wiki/tools/wiki-remove.tool.test.ts index 2cdb87e0..8130613c 100644 --- a/packages/context/src/wiki/tools/wiki-remove.tool.test.ts +++ b/packages/cli/src/context/wiki/tools/wiki-remove.tool.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources, type ToolContext } from '../../tools/index.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import { WikiRemoveTool } from './wiki-remove.tool.js'; describe('WikiRemoveTool', () => { diff --git a/packages/context/src/wiki/tools/wiki-remove.tool.ts b/packages/cli/src/context/wiki/tools/wiki-remove.tool.ts similarity index 92% rename from packages/context/src/wiki/tools/wiki-remove.tool.ts rename to packages/cli/src/context/wiki/tools/wiki-remove.tool.ts index 4d4c1333..7f56fcb6 100644 --- a/packages/context/src/wiki/tools/wiki-remove.tool.ts +++ b/packages/cli/src/context/wiki/tools/wiki-remove.tool.ts @@ -2,9 +2,10 @@ import { z } from 'zod'; import type { KnowledgeIndexPort } from '../ports.js'; import type { KnowledgeEventPort } from '../ports.js'; type BlockScope = 'GLOBAL' | 'USER'; -import { KnowledgeWikiService } from '../index.js'; +import { KnowledgeWikiService } from '../../../context/wiki/knowledge-wiki.service.js'; import { validateFlatWikiKey } from '../keys.js'; -import { BaseTool, type ToolContext, type ToolOutput, validateActionRawPaths } from '../../tools/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../context/tools/base-tool.js'; +import { validateActionRawPaths } from '../../../context/tools/action-raw-paths.js'; const SYSTEM_AUTHOR = 'System User'; const SYSTEM_EMAIL = 'system@example.com'; diff --git a/packages/context/src/wiki/tools/wiki-search.tool.test.ts b/packages/cli/src/context/wiki/tools/wiki-search.tool.test.ts similarity index 100% rename from packages/context/src/wiki/tools/wiki-search.tool.test.ts rename to packages/cli/src/context/wiki/tools/wiki-search.tool.test.ts diff --git a/packages/context/src/wiki/tools/wiki-search.tool.ts b/packages/cli/src/context/wiki/tools/wiki-search.tool.ts similarity index 98% rename from packages/context/src/wiki/tools/wiki-search.tool.ts rename to packages/cli/src/context/wiki/tools/wiki-search.tool.ts index d8a40fc3..fd3f8c35 100644 --- a/packages/context/src/wiki/tools/wiki-search.tool.ts +++ b/packages/cli/src/context/wiki/tools/wiki-search.tool.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { BaseTool, type ToolContext, type ToolOutput } from '../../tools/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../context/tools/base-tool.js'; import type { WikiSearchLaneSummary, WikiSearchMatchReason } from '../types.js'; const WikiSearchInputSchema = z.object({ @@ -23,6 +23,7 @@ interface WikiSearchStructured { totalFound: number; } +/** @internal */ export interface WikiSearchAdapterPort { search(input: { userId: string; query: string; limit: number }): Promise<{ results: Array<{ diff --git a/packages/context/src/wiki/tools/wiki-write.tool.test.ts b/packages/cli/src/context/wiki/tools/wiki-write.tool.test.ts similarity index 98% rename from packages/context/src/wiki/tools/wiki-write.tool.test.ts rename to packages/cli/src/context/wiki/tools/wiki-write.tool.test.ts index 71d9ca6e..ad2bc54b 100644 --- a/packages/context/src/wiki/tools/wiki-write.tool.test.ts +++ b/packages/cli/src/context/wiki/tools/wiki-write.tool.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources, type ToolContext } from '../../tools/index.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import { WikiWriteTool } from './wiki-write.tool.js'; function makeTool(overrides: any = {}) { diff --git a/packages/context/src/wiki/tools/wiki-write.tool.ts b/packages/cli/src/context/wiki/tools/wiki-write.tool.ts similarity index 96% rename from packages/context/src/wiki/tools/wiki-write.tool.ts rename to packages/cli/src/context/wiki/tools/wiki-write.tool.ts index 5dd82578..4b0f1b39 100644 --- a/packages/context/src/wiki/tools/wiki-write.tool.ts +++ b/packages/cli/src/context/wiki/tools/wiki-write.tool.ts @@ -2,11 +2,13 @@ import { z } from 'zod'; import type { KnowledgeIndexPort } from '../ports.js'; import type { KnowledgeEventPort } from '../ports.js'; type BlockScope = 'GLOBAL' | 'USER'; -import { KnowledgeWikiService, type WikiFrontmatter } from '../index.js'; +import { KnowledgeWikiService } from '../../../context/wiki/knowledge-wiki.service.js'; +import type { WikiFrontmatter } from '../../../context/wiki/types.js'; import { validateFlatWikiKey } from '../keys.js'; import { findMissingWikiRefs } from '../wiki-ref-validation.js'; import { applySqlEdits } from '../../tools/sql-edit-replacer.js'; -import { BaseTool, type ToolContext, type ToolOutput, validateActionRawPaths } from '../../tools/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../context/tools/base-tool.js'; +import { validateActionRawPaths } from '../../../context/tools/action-raw-paths.js'; const MAX_USER_BLOCKS = 100; const SYSTEM_AUTHOR = 'System User'; diff --git a/packages/context/src/wiki/types.ts b/packages/cli/src/context/wiki/types.ts similarity index 90% rename from packages/context/src/wiki/types.ts rename to packages/cli/src/context/wiki/types.ts index bff57aa5..bd54d130 100644 --- a/packages/context/src/wiki/types.ts +++ b/packages/cli/src/context/wiki/types.ts @@ -49,9 +49,3 @@ export interface WikiSearchLaneSummary { weight: number; reason?: string; } - -export interface WikiSearchMetadata { - score: number; - matchReasons: WikiSearchMatchReason[]; - lanes?: WikiSearchLaneSummary[]; -} diff --git a/packages/context/src/wiki/wiki-ref-validation.test.ts b/packages/cli/src/context/wiki/wiki-ref-validation.test.ts similarity index 100% rename from packages/context/src/wiki/wiki-ref-validation.test.ts rename to packages/cli/src/context/wiki/wiki-ref-validation.test.ts diff --git a/packages/context/src/wiki/wiki-ref-validation.ts b/packages/cli/src/context/wiki/wiki-ref-validation.ts similarity index 97% rename from packages/context/src/wiki/wiki-ref-validation.ts rename to packages/cli/src/context/wiki/wiki-ref-validation.ts index 5a3ae8c2..28e67f2e 100644 --- a/packages/context/src/wiki/wiki-ref-validation.ts +++ b/packages/cli/src/context/wiki/wiki-ref-validation.ts @@ -1,4 +1,4 @@ -import type { MemoryAction } from '../tools/index.js'; +import type { MemoryAction } from '../../context/tools/tool-session.js'; import { isFlatWikiKey } from './keys.js'; import type { KnowledgeWikiService } from './knowledge-wiki.service.js'; import type { WikiScope } from './types.js'; diff --git a/packages/cli/src/database-tree-picker.ts b/packages/cli/src/database-tree-picker.ts index d494003d..aea3c092 100644 --- a/packages/cli/src/database-tree-picker.ts +++ b/packages/cli/src/database-tree-picker.ts @@ -1,4 +1,4 @@ -import type { KtxTableListEntry } from '@ktx/context/scan'; +import type { KtxTableListEntry } from './context/scan/types.js'; import type { KtxCliIo } from './cli-runtime.js'; import { profileMark } from './startup-profile.js'; import { diff --git a/packages/cli/src/demo-assets.ts b/packages/cli/src/demo-assets.ts index 674af8cf..dcc7ac1f 100644 --- a/packages/cli/src/demo-assets.ts +++ b/packages/cli/src/demo-assets.ts @@ -17,8 +17,11 @@ interface EnsureDemoProjectOptions { force: boolean; } +/** @internal */ export const DEMO_CONNECTION_ID = 'orbit_demo'; +/** @internal */ export const DEMO_ADAPTER = 'live-database'; +/** @internal */ export const DEMO_REPLAY_FILE = 'replay.memory-flow.v1.json'; const REQUIRED_PACKAGED_BASE_ASSET_PATHS = ['demo.db', 'manifest.json', DEMO_REPLAY_FILE] as const; @@ -115,6 +118,7 @@ async function assertPackagedSeededAssetsPresent(): Promise { } } +/** @internal */ export async function ensureDemoProject(options: EnsureDemoProjectOptions): Promise { const projectDir = resolve(options.projectDir); const configPath = join(projectDir, 'ktx.yaml'); diff --git a/packages/cli/src/demo-metrics.test.ts b/packages/cli/src/demo-metrics.test.ts index 42be0cde..9e40be36 100644 --- a/packages/cli/src/demo-metrics.test.ts +++ b/packages/cli/src/demo-metrics.test.ts @@ -1,4 +1,4 @@ -import type { MemoryFlowEvent, MemoryFlowReplayInput } from '@ktx/context/ingest/memory-flow'; +import type { MemoryFlowEvent, MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; import { describe, expect, it } from 'vitest'; import { buildDemoMetrics, diff --git a/packages/cli/src/demo-metrics.ts b/packages/cli/src/demo-metrics.ts index 4fee8de3..d6f9c207 100644 --- a/packages/cli/src/demo-metrics.ts +++ b/packages/cli/src/demo-metrics.ts @@ -1,4 +1,4 @@ -import type { MemoryFlowEvent, MemoryFlowReplayInput } from '@ktx/context/ingest/memory-flow'; +import type { MemoryFlowEvent, MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; const DEFAULT_INPUT_TOKENS_PER_STEP = 4500; const DEFAULT_OUTPUT_TOKENS_PER_STEP = 700; @@ -140,6 +140,7 @@ export function formatDuration(ms: number): string { return `${hr}h${(min % 60).toString().padStart(2, '0')}m`; } +/** @internal */ export function formatEta(ms: number | null, status: MemoryFlowReplayInput['status']): string { if (status !== 'running') return 'done'; if (ms === null) return 'estimating...'; @@ -153,6 +154,7 @@ export function formatCost(usd: number): string { return `$${usd.toFixed(2)}`; } +/** @internal */ export function formatTokens(n: number): string { if (!Number.isFinite(n) || n <= 0) return '0'; if (n < 1000) return `${Math.round(n)}`; @@ -160,6 +162,7 @@ export function formatTokens(n: number): string { return `${(n / 1_000_000).toFixed(2)}M`; } +/** @internal */ export function formatTokensPerSec(n: number): string { if (!Number.isFinite(n) || n <= 0) return '0/s'; if (n < 1000) return `${Math.round(n)}/s`; @@ -167,6 +170,7 @@ export function formatTokensPerSec(n: number): string { } const PROGRESS_BAR_WIDTH = 12; +/** @internal */ export function progressBar(ratio: number, width: number = PROGRESS_BAR_WIDTH): string { const clamped = Math.max(0, Math.min(1, ratio)); const filled = Math.round(clamped * width); diff --git a/packages/cli/src/doctor.test.ts b/packages/cli/src/doctor.test.ts index a6bfbff6..b483ea20 100644 --- a/packages/cli/src/doctor.test.ts +++ b/packages/cli/src/doctor.test.ts @@ -106,7 +106,7 @@ describe('runSetupDoctorChecks', () => { if (command === 'pnpm' && args[0] === '--version') return '10.28.0'; if (command === 'corepack' && args[0] === '--version') return '0.32.0'; if (command === 'uv' && args[0] === '--version') return 'uv 0.9.5'; - if (command === process.execPath && args.includes('--version')) return '@ktx/cli 0.0.0-private'; + if (command === process.execPath && args.includes('--version')) return '@kaelio/ktx 0.0.0-private'; throw new Error(`${command} ${args.join(' ')}`); }, pathExists: async () => true, @@ -163,7 +163,7 @@ describe('runSetupDoctorChecks', () => { if (command === 'pnpm' && args[0] === '--version') return '10.28.0'; if (command === 'corepack' && args[0] === '--version') throw new Error('spawn corepack ENOENT'); if (command === 'uv' && args[0] === '--version') return 'uv 0.9.5'; - if (command === process.execPath && args.includes('--version')) return '@ktx/cli 0.0.0-private'; + if (command === process.execPath && args.includes('--version')) return '@kaelio/ktx 0.0.0-private'; throw new Error(`${command} ${args.join(' ')}`); }, pathExists: async () => true, diff --git a/packages/cli/src/doctor.ts b/packages/cli/src/doctor.ts index 40ab4a72..8eac91fa 100644 --- a/packages/cli/src/doctor.ts +++ b/packages/cli/src/doctor.ts @@ -4,7 +4,7 @@ import { access, readFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; -import type { KtxConfigIssue } from '@ktx/context/project'; +import type { KtxConfigIssue } from './context/project/config.js'; import { KTX_NEXT_STEP_DIRECT_COMMANDS } from './next-steps.js'; import type { BuildProjectStatusOptions } from './status-project.js'; @@ -594,7 +594,7 @@ export async function runKtxDoctor( renderMissingProjectMessage(args.projectDir, args.outputMode, io); return 1; } - const { validateKtxProjectConfig } = await import('@ktx/context/project'); + const { validateKtxProjectConfig } = await import('./context/project/config.js');; const rawConfig = await readFile(configPath, 'utf-8'); const validation = validateKtxProjectConfig(rawConfig); if (!validation.ok) { @@ -611,7 +611,8 @@ export async function runKtxDoctor( renderMissingProjectMessage(args.projectDir, args.outputMode, io); return 1; } - const { loadKtxProject, validateKtxProjectConfig } = await import('@ktx/context/project'); + const { loadKtxProject } = await import('./context/project/project.js'); + const { validateKtxProjectConfig } = await import('./context/project/config.js');; const { buildProjectStatus, renderProjectStatus } = await import('./status-project.js'); const rawConfig = await readFile(configPath, 'utf-8'); const validation = validateKtxProjectConfig(rawConfig); diff --git a/packages/cli/src/embedding-resolution.test.ts b/packages/cli/src/embedding-resolution.test.ts index 210bd755..40c71538 100644 --- a/packages/cli/src/embedding-resolution.test.ts +++ b/packages/cli/src/embedding-resolution.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { buildDefaultKtxProjectConfig, type KtxLocalProject, type KtxProjectConfig } from '@ktx/context/project'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; +import type { KtxLocalProject } from './context/project/project.js'; import { resolveProjectEmbeddingProvider } from './embedding-resolution.js'; import type { ManagedLocalEmbeddingsDaemon } from './managed-local-embeddings.js'; diff --git a/packages/cli/src/embedding-resolution.ts b/packages/cli/src/embedding-resolution.ts index d7bfafae..0243943c 100644 --- a/packages/cli/src/embedding-resolution.ts +++ b/packages/cli/src/embedding-resolution.ts @@ -1,9 +1,8 @@ -import { - type KtxEmbeddingProvider, - createKtxEmbeddingProvider as defaultCreateKtxEmbeddingProvider, -} from '@ktx/llm'; -import type { KtxLocalProject, KtxProjectEmbeddingConfig } from '@ktx/context/project'; -import { resolveLocalKtxEmbeddingConfig } from '@ktx/context'; +import type { KtxEmbeddingProvider } from './llm/types.js'; +import { createKtxEmbeddingProvider as defaultCreateKtxEmbeddingProvider } from './llm/embedding-provider.js'; +import type { KtxLocalProject } from './context/project/project.js'; +import type { KtxProjectEmbeddingConfig } from './context/project/config.js'; +import { resolveLocalKtxEmbeddingConfig } from './context/llm/local-config.js'; import type { KtxCliIo } from './cli-runtime.js'; import { ensureManagedLocalEmbeddingsDaemon as defaultEnsureManagedDaemon, diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index 00ae6ca8..c482e452 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject } from '@ktx/context/project'; +import { initKtxProject } from './context/project/project.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { @@ -19,7 +19,7 @@ import { const require = createRequire(import.meta.url); -const cliPackageJson = require('@ktx/cli/package.json') as { name: string; version: string }; +const cliPackageJson = require('@kaelio/ktx/package.json') as { name: string; version: string }; const cliVersion = cliPackageJson.version; function makeIo(options: { stdoutIsTty?: boolean } = {}) { @@ -45,19 +45,18 @@ function makeIo(options: { stdoutIsTty?: boolean } = {}) { } describe('getKtxCliPackageInfo', () => { - it('identifies the CLI package and its context dependency', () => { + it('identifies the CLI package', () => { expect(getKtxCliPackageInfo()).toEqual({ - name: '@ktx/cli', + name: '@kaelio/ktx', version: cliVersion, - contextPackageName: '@ktx/context', }); }); it('exports package metadata for package managers and runtime diagnostics', () => { - const packageJson = require('@ktx/cli/package.json') as { name: string; version: string }; + const packageJson = require('@kaelio/ktx/package.json') as { name: string; version: string }; expect(packageJson).toMatchObject({ - name: '@ktx/cli', + name: '@kaelio/ktx', version: cliVersion, }); expect(cliVersion).toMatch(/^\d+\.\d+\.\d+/); @@ -72,7 +71,6 @@ describe('getKtxCliPackageInfo', () => { ).toEqual({ name: '@kaelio/ktx', version: '0.1.0', - contextPackageName: '@ktx/context', }); }); }); @@ -118,7 +116,7 @@ describe('runKtxCli', () => { await expect(runKtxCli(['--version'], testIo.io)).resolves.toBe(0); - expect(testIo.stdout()).toBe(`@ktx/cli ${cliVersion}\n`); + expect(testIo.stdout()).toBe(`@kaelio/ktx ${cliVersion}\n`); expect(testIo.stderr()).toBe(''); }); diff --git a/packages/cli/src/ingest-depth.ts b/packages/cli/src/ingest-depth.ts index f8e5d06e..489c44e8 100644 --- a/packages/cli/src/ingest-depth.ts +++ b/packages/cli/src/ingest-depth.ts @@ -1,4 +1,4 @@ -import type { KtxProjectConfig, KtxProjectConnectionConfig } from '@ktx/context/project'; +import type { KtxProjectConfig, KtxProjectConnectionConfig } from './context/project/config.js'; export type KtxDatabaseContextDepth = 'fast' | 'deep'; diff --git a/packages/cli/src/ingest-query-executor.test.ts b/packages/cli/src/ingest-query-executor.test.ts index 343202a1..14b714d9 100644 --- a/packages/cli/src/ingest-query-executor.test.ts +++ b/packages/cli/src/ingest-query-executor.test.ts @@ -1,5 +1,5 @@ -import type { KtxLocalProject } from '@ktx/context/project'; -import { createKtxConnectorCapabilities, type KtxScanConnector } from '@ktx/context/scan'; +import type { KtxLocalProject } from './context/project/project.js'; +import { createKtxConnectorCapabilities, type KtxScanConnector } from './context/scan/types.js'; import { describe, expect, it, vi } from 'vitest'; import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js'; diff --git a/packages/cli/src/ingest-query-executor.ts b/packages/cli/src/ingest-query-executor.ts index 197119be..f8b6880d 100644 --- a/packages/cli/src/ingest-query-executor.ts +++ b/packages/cli/src/ingest-query-executor.ts @@ -1,6 +1,6 @@ -import type { KtxSqlQueryExecutionInput, KtxSqlQueryExecutorPort } from '@ktx/context/connections'; -import type { KtxLocalProject } from '@ktx/context/project'; -import type { KtxScanConnector, KtxScanContext } from '@ktx/context/scan'; +import type { KtxSqlQueryExecutionInput, KtxSqlQueryExecutorPort } from './context/connections/query-executor.js'; +import type { KtxLocalProject } from './context/project/project.js'; +import type { KtxScanConnector, KtxScanContext } from './context/scan/types.js'; import { createKtxCliScanConnector } from './local-scan-connectors.js'; type CreateConnector = typeof createKtxCliScanConnector; diff --git a/packages/cli/src/ingest-report-file.ts b/packages/cli/src/ingest-report-file.ts index cd29706b..7d545429 100644 --- a/packages/cli/src/ingest-report-file.ts +++ b/packages/cli/src/ingest-report-file.ts @@ -1,5 +1,6 @@ import { readFile } from 'node:fs/promises'; -import { parseIngestReportSnapshot, type IngestReportSnapshot } from '@ktx/context/ingest'; +import { parseIngestReportSnapshot } from './context/ingest/report-snapshot.js'; +import type { IngestReportSnapshot } from './context/ingest/reports.js'; export async function readIngestReportSnapshotFile(reportFile: string): Promise { const raw = await readFile(reportFile, 'utf-8'); diff --git a/packages/cli/src/ingest-viz.test.ts b/packages/cli/src/ingest-viz.test.ts index 7d790ab7..17b35f75 100644 --- a/packages/cli/src/ingest-viz.test.ts +++ b/packages/cli/src/ingest-viz.test.ts @@ -1,11 +1,8 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { - type LocalIngestResult, - type MemoryFlowReplayInput, - type RunLocalIngestOptions, -} from '@ktx/context/ingest'; +import type { LocalIngestResult, RunLocalIngestOptions } from './context/ingest/local-ingest.js'; +import type { MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runKtxIngest } from './ingest.js'; import { diff --git a/packages/cli/src/ingest.test-utils.ts b/packages/cli/src/ingest.test-utils.ts index d64dd7e1..9b3f16fa 100644 --- a/packages/cli/src/ingest.test-utils.ts +++ b/packages/cli/src/ingest.test-utils.ts @@ -1,29 +1,19 @@ import { EventEmitter } from 'node:events'; import { mkdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; -import type { AgentRunnerPort, RunLoopParams } from '@ktx/context'; -import { - KtxYamlMetabaseSourceStateReader, - LocalMetabaseDiscoveryCache, - MetabaseSourceAdapter, - getLocalIngestStatus, - type ChunkResult, - type FetchContext, - type IngestReportSnapshot, - type LocalIngestResult, - type LookerMappingClient, - type LookerRuntimeClient, - type LookerTableIdentifierParser, - type MemoryFlowEventSink, - type MetabaseCard, - type MetabaseCardSummary, - type MetabaseClientFactory, - type MetabaseRuntimeClient, - type RunLocalIngestOptions, - type SourceAdapter, - type SqliteBundleIngestStore, -} from '@ktx/context/ingest'; -import { ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project'; +import type { AgentRunnerPort, RunLoopParams } from './context/llm/runtime-port.js'; +import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './context/ingest/adapters/metabase/local-source-state-store.js'; +import { MetabaseSourceAdapter } from './context/ingest/adapters/metabase/metabase.adapter.js'; +import { getLocalIngestStatus, type LocalIngestResult, type RunLocalIngestOptions } from './context/ingest/local-ingest.js'; +import type { ChunkResult, FetchContext, SourceAdapter } from './context/ingest/types.js'; +import type { IngestReportSnapshot } from './context/ingest/reports.js'; +import type { LookerMappingClient, LookerTableIdentifierParser } from './context/ingest/adapters/looker/mapping.js'; +import type { LookerRuntimeClient } from './context/ingest/adapters/looker/fetch.js'; +import type { MemoryFlowEventSink } from './context/ingest/memory-flow/types.js'; +import type { MetabaseCard, MetabaseCardSummary, MetabaseClientFactory, MetabaseRuntimeClient } from './context/ingest/adapters/metabase/client-port.js'; +import type { SqliteBundleIngestStore } from './context/ingest/sqlite-bundle-ingest-store.js'; +import { ktxLocalStateDbPath } from './context/project/local-state-db.js'; +import { loadKtxProject } from './context/project/project.js'; import { expect, vi } from 'vitest'; import { runKtxIngest } from './ingest.js'; @@ -685,7 +675,7 @@ export function localFakeBundleReport( } export async function localBundleStore(projectDir: string, ids: [string, string]): Promise { - const { SqliteBundleIngestStore } = await import('@ktx/context/ingest'); + const { SqliteBundleIngestStore } = await import('./context/ingest/sqlite-bundle-ingest-store.js');; const project = await loadKtxProject({ projectDir }); return new SqliteBundleIngestStore({ dbPath: ktxLocalStateDbPath(project), diff --git a/packages/cli/src/ingest.test.ts b/packages/cli/src/ingest.test.ts index 03f04d8e..33eecfba 100644 --- a/packages/cli/src/ingest.test.ts +++ b/packages/cli/src/ingest.test.ts @@ -1,15 +1,12 @@ import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { - LocalLookerRuntimeStore, - LocalMetabaseDiscoveryCache, - type LocalIngestResult, - type LocalMetabaseFanoutProgress, - type RunLocalIngestOptions, - type SourceAdapter, -} from '@ktx/context/ingest'; -import { initKtxProject, ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project'; +import { LocalLookerRuntimeStore } from './context/ingest/adapters/looker/local-runtime-store.js'; +import { LocalMetabaseDiscoveryCache } from './context/ingest/adapters/metabase/local-source-state-store.js'; +import type { LocalIngestResult, LocalMetabaseFanoutProgress, RunLocalIngestOptions } from './context/ingest/local-ingest.js'; +import type { SourceAdapter } from './context/ingest/types.js'; +import { initKtxProject, loadKtxProject } from './context/project/project.js'; +import { ktxLocalStateDbPath } from './context/project/local-state-db.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type KtxIngestArgs, type KtxIngestDeps, runKtxIngest } from './ingest.js'; import type { KtxCliLocalIngestAdaptersOptions } from './local-adapters.js'; @@ -1418,6 +1415,52 @@ describe('runKtxIngest', () => { ); }); + it('uses runtime IO when resolving managed embedding runtime', async () => { + const projectDir = join(tempDir, 'managed-embedding-ingest-project'); + await initKtxProject({ projectDir }); + await writeWarehouseConfig(projectDir); + const createdAdapters: SourceAdapter[] = [ + { source: 'fake', skillNames: [], detect: async () => true, chunk: async () => ({ workUnits: [] }) }, + ]; + const createAdapters = vi.fn(() => createdAdapters as never); + const runLocal = vi.fn(async (input: RunLocalIngestOptions) => + completedLocalBundleRun(input, input.jobId ?? 'local-job-1'), + ); + const resolveEmbeddingProvider = vi.fn(async () => ({ kind: 'disabled' as const })); + const io = makeIo(); + const runtimeIo = makeIo({ isTTY: true }); + + await expect( + runKtxIngest( + { + command: 'run', + projectDir, + connectionId: 'warehouse', + adapter: 'fake', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + outputMode: 'plain', + } satisfies KtxIngestArgs, + io.io, + { + createAdapters, + runLocalIngest: runLocal, + jobIdFactory: () => 'local-job-1', + runtimeIo: runtimeIo.io, + resolveEmbeddingProvider, + }, + ), + ).resolves.toBe(0); + + expect(resolveEmbeddingProvider).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + installPolicy: 'auto', + io: runtimeIo.io, + }), + ); + }); + it('passes the target connection id when constructing local historic-sql adapters', async () => { const projectDir = join(tempDir, 'historic-sql-project'); await writeWarehouseConfig(projectDir); diff --git a/packages/cli/src/ingest.ts b/packages/cli/src/ingest.ts index b2b7bd0e..3615f401 100644 --- a/packages/cli/src/ingest.ts +++ b/packages/cli/src/ingest.ts @@ -1,24 +1,14 @@ -import { - buildMemoryFlowViewModel, - createMemoryFlowLiveBuffer, - formatMemoryFlowFinalSummary, - getLatestLocalIngestStatus, - getLocalIngestStatus, - type IngestReportSnapshot, - ingestReportToMemoryFlowReplay, - type LocalMetabaseFanoutResult, - type LocalMetabaseFanoutProgress, - type MemoryFlowEvent, - type MemoryFlowReplayInput, - type RunLocalIngestOptions, - renderMemoryFlowReplay, - runLocalIngest, - runLocalMetabaseIngest, - savedMemoryCountsForReport, - sanitizeMemoryFlowError, -} from '@ktx/context/ingest'; -import type { KtxSqlQueryExecutorPort } from '@ktx/context/connections'; -import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project'; +import { buildMemoryFlowViewModel } from './context/ingest/memory-flow/view-model.js'; +import { createMemoryFlowLiveBuffer, sanitizeMemoryFlowError } from './context/ingest/memory-flow/live-buffer.js'; +import { formatMemoryFlowFinalSummary } from './context/ingest/memory-flow/summary.js'; +import { getLatestLocalIngestStatus, getLocalIngestStatus, type LocalMetabaseFanoutResult, type LocalMetabaseFanoutProgress, type RunLocalIngestOptions, runLocalIngest, runLocalMetabaseIngest } from './context/ingest/local-ingest.js'; +import { type IngestReportSnapshot, savedMemoryCountsForReport } from './context/ingest/reports.js'; +import { ingestReportToMemoryFlowReplay } from './context/ingest/memory-flow/events.js'; +import type { MemoryFlowEvent, MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; +import { renderMemoryFlowReplay } from './context/ingest/memory-flow/render.js'; +import type { KtxSqlQueryExecutorPort } from './context/connections/query-executor.js'; +import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; +import { getKtxCliPackageInfo } from './cli-runtime.js'; import { resolveProjectEmbeddingProvider } from './embedding-resolution.js'; import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js'; import { readIngestReportSnapshotFile } from './ingest-report-file.js'; @@ -82,6 +72,7 @@ export interface KtxIngestDeps { now?: () => Date; createAdapters?: typeof createKtxCliLocalIngestAdapters; createQueryExecutor?: (project: KtxLocalProject) => KtxSqlQueryExecutorPort; + resolveEmbeddingProvider?: typeof resolveProjectEmbeddingProvider; runLocalIngest?: typeof runLocalIngest; runLocalMetabaseIngest?: typeof runLocalMetabaseIngest; readReportFile?: typeof readIngestReportSnapshotFile; @@ -685,11 +676,12 @@ export async function runKtxIngest( const project = await loadKtxProject({ projectDir: args.projectDir }); const env = deps.env ?? process.env; if (args.command === 'run') { - const resolution = await resolveProjectEmbeddingProvider(project, { + const resolveEmbeddingProvider = deps.resolveEmbeddingProvider ?? resolveProjectEmbeddingProvider; + const resolution = await resolveEmbeddingProvider(project, { mode: 'ensure', installPolicy: args.runtimeInstallPolicy ?? 'never', - cliVersion: args.cliVersion ?? '0.0.0-private', - io, + cliVersion: args.cliVersion ?? getKtxCliPackageInfo().version, + io: deps.runtimeIo ?? io, }); const embeddingProvider = resolution.kind === 'disabled' || resolution.kind === 'managed-unavailable' ? null : resolution.provider; diff --git a/packages/cli/src/io/logger.ts b/packages/cli/src/io/logger.ts index e9952254..fc84a559 100644 --- a/packages/cli/src/io/logger.ts +++ b/packages/cli/src/io/logger.ts @@ -14,6 +14,7 @@ function writeLine(io: KtxCliIo, message: string): void { io.stderr.write(message.endsWith('\n') ? message : `${message}\n`); } +/** @internal */ export function createNoopOperationalLogger(): KtxOperationalLogger { return { log: () => undefined, diff --git a/packages/cli/src/knowledge.test.ts b/packages/cli/src/knowledge.test.ts index d1872221..9fc92a31 100644 --- a/packages/cli/src/knowledge.test.ts +++ b/packages/cli/src/knowledge.test.ts @@ -2,9 +2,9 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { stripVTControlCharacters } from 'node:util'; -import { initKtxProject, loadKtxProject } from '@ktx/context/project'; -import type { KtxEmbeddingPort } from '@ktx/context'; -import { writeLocalKnowledgePage } from '@ktx/context/wiki'; +import { initKtxProject, loadKtxProject } from './context/project/project.js'; +import type { KtxEmbeddingPort } from './context/core/embedding.js'; +import { writeLocalKnowledgePage } from './context/wiki/local-knowledge.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runKtxKnowledge } from './knowledge.js'; diff --git a/packages/cli/src/knowledge.ts b/packages/cli/src/knowledge.ts index 07d68381..9eb35b12 100644 --- a/packages/cli/src/knowledge.ts +++ b/packages/cli/src/knowledge.ts @@ -1,11 +1,7 @@ -import { KtxIngestEmbeddingPortAdapter, type KtxEmbeddingPort } from '@ktx/context'; -import { loadKtxProject } from '@ktx/context/project'; -import { - type LocalKnowledgeSearchResult, - type LocalKnowledgeSummary, - listLocalKnowledgePages, - searchLocalKnowledgePages as defaultSearchLocalKnowledgePages, -} from '@ktx/context/wiki'; +import { KtxIngestEmbeddingPortAdapter } from './context/llm/embedding-port.js'; +import type { KtxEmbeddingPort } from './context/core/embedding.js'; +import { loadKtxProject } from './context/project/project.js'; +import { type LocalKnowledgeSearchResult, type LocalKnowledgeSummary, listLocalKnowledgePages, searchLocalKnowledgePages as defaultSearchLocalKnowledgePages } from './context/wiki/local-knowledge.js'; import { resolveProjectEmbeddingProvider, type EmbeddingProviderResolution, diff --git a/packages/llm/src/embedding-health.test.ts b/packages/cli/src/llm/embedding-health.test.ts similarity index 100% rename from packages/llm/src/embedding-health.test.ts rename to packages/cli/src/llm/embedding-health.test.ts diff --git a/packages/llm/src/embedding-health.ts b/packages/cli/src/llm/embedding-health.ts similarity index 100% rename from packages/llm/src/embedding-health.ts rename to packages/cli/src/llm/embedding-health.ts diff --git a/packages/llm/src/embedding-provider.test.ts b/packages/cli/src/llm/embedding-provider.test.ts similarity index 100% rename from packages/llm/src/embedding-provider.test.ts rename to packages/cli/src/llm/embedding-provider.test.ts diff --git a/packages/llm/src/embedding-provider.ts b/packages/cli/src/llm/embedding-provider.ts similarity index 100% rename from packages/llm/src/embedding-provider.ts rename to packages/cli/src/llm/embedding-provider.ts diff --git a/packages/llm/src/message-builder.test.ts b/packages/cli/src/llm/message-builder.test.ts similarity index 100% rename from packages/llm/src/message-builder.test.ts rename to packages/cli/src/llm/message-builder.test.ts diff --git a/packages/llm/src/message-builder.ts b/packages/cli/src/llm/message-builder.ts similarity index 100% rename from packages/llm/src/message-builder.ts rename to packages/cli/src/llm/message-builder.ts diff --git a/packages/llm/src/model-health.test.ts b/packages/cli/src/llm/model-health.test.ts similarity index 100% rename from packages/llm/src/model-health.test.ts rename to packages/cli/src/llm/model-health.test.ts diff --git a/packages/llm/src/model-health.ts b/packages/cli/src/llm/model-health.ts similarity index 95% rename from packages/llm/src/model-health.ts rename to packages/cli/src/llm/model-health.ts index abbc2735..91920478 100644 --- a/packages/llm/src/model-health.ts +++ b/packages/cli/src/llm/model-health.ts @@ -4,7 +4,7 @@ import type { KtxLlmConfig } from './types.js'; export type KtxLlmHealthCheckResult = { ok: true } | { ok: false; message: string }; -export interface KtxLlmHealthCheckDeps extends Omit { +interface KtxLlmHealthCheckDeps extends Omit { generateText?: (options: Parameters[0]) => Promise; } diff --git a/packages/llm/src/model-provider.test.ts b/packages/cli/src/llm/model-provider.test.ts similarity index 100% rename from packages/llm/src/model-provider.test.ts rename to packages/cli/src/llm/model-provider.test.ts diff --git a/packages/llm/src/model-provider.ts b/packages/cli/src/llm/model-provider.ts similarity index 98% rename from packages/llm/src/model-provider.ts rename to packages/cli/src/llm/model-provider.ts index 86b9270b..207faa93 100644 --- a/packages/llm/src/model-provider.ts +++ b/packages/cli/src/llm/model-provider.ts @@ -55,7 +55,7 @@ function resolveDevtoolsEnabled(override: boolean | undefined): boolean { return value === 'true' || value === '1' || value === 'yes'; } -export function modelIdFromLanguageModel(model: LanguageModel | string): string { +function modelIdFromLanguageModel(model: LanguageModel | string): string { return typeof model === 'string' ? model : ((model as { modelId?: string }).modelId ?? ''); } diff --git a/packages/llm/src/repair.test.ts b/packages/cli/src/llm/repair.test.ts similarity index 100% rename from packages/llm/src/repair.test.ts rename to packages/cli/src/llm/repair.test.ts diff --git a/packages/llm/src/repair.ts b/packages/cli/src/llm/repair.ts similarity index 100% rename from packages/llm/src/repair.ts rename to packages/cli/src/llm/repair.ts diff --git a/packages/llm/src/types.ts b/packages/cli/src/llm/types.ts similarity index 91% rename from packages/llm/src/types.ts rename to packages/cli/src/llm/types.ts index bc928e08..3f7f67e2 100644 --- a/packages/llm/src/types.ts +++ b/packages/cli/src/llm/types.ts @@ -3,10 +3,10 @@ import type { LanguageModel, TelemetrySettings, ToolCallRepairFunction, ToolSet export const KTX_MODEL_ROLES = ['default', 'triage', 'candidateExtraction', 'curator', 'reconcile', 'repair'] as const; export type KtxModelRole = (typeof KTX_MODEL_ROLES)[number]; -export type KtxLlmBackend = 'anthropic' | 'vertex' | 'gateway' | 'claude-code'; +type KtxLlmBackend = 'anthropic' | 'vertex' | 'gateway' | 'claude-code'; export type KtxPromptCacheTtl = '5m' | '1h'; -export type KtxJsonValue = +type KtxJsonValue = | null | string | number @@ -27,7 +27,7 @@ export interface KtxPromptCachingConfig { vertexFallbackTo5m: boolean; } -export interface KtxTokenUsageEvent { +interface KtxTokenUsageEvent { source?: string; modelId?: string; inputTokens?: number; @@ -62,9 +62,9 @@ export interface KtxLlmProvider { activeBackend(): KtxLlmBackend; } -export type KtxEmbeddingBackend = 'openai' | 'sentence-transformers'; +type KtxEmbeddingBackend = 'openai' | 'sentence-transformers'; -export interface KtxEmbeddingTokenUsageEvent { +interface KtxEmbeddingTokenUsageEvent { backend: KtxEmbeddingBackend; model: string; inputCount: number; diff --git a/packages/cli/src/local-adapters.test.ts b/packages/cli/src/local-adapters.test.ts index e1b4b014..ac8c3c41 100644 --- a/packages/cli/src/local-adapters.test.ts +++ b/packages/cli/src/local-adapters.test.ts @@ -1,7 +1,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { loadKtxProject } from '@ktx/context/project'; +import { loadKtxProject } from './context/project/project.js'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { createKtxCliLocalIngestAdapters } from './local-adapters.js'; diff --git a/packages/cli/src/local-adapters.ts b/packages/cli/src/local-adapters.ts index daa8f63b..d5b7817e 100644 --- a/packages/cli/src/local-adapters.ts +++ b/packages/cli/src/local-adapters.ts @@ -1,33 +1,28 @@ -import { - createBigQueryLiveDatabaseIntrospection, - isKtxBigQueryConnectionConfig, - KtxBigQueryScanConnector, - type KtxBigQueryConnectionConfig, -} from '@ktx/connector-bigquery'; -import { createClickHouseLiveDatabaseIntrospection, isKtxClickHouseConnectionConfig } from '@ktx/connector-clickhouse'; -import { createMysqlLiveDatabaseIntrospection, isKtxMysqlConnectionConfig } from '@ktx/connector-mysql'; -import { - createPostgresLiveDatabaseIntrospection, - isKtxPostgresConnectionConfig, - type KtxPostgresConnectionConfig, - KtxPostgresHistoricSqlQueryClient, -} from '@ktx/connector-postgres'; -import { createSqliteLiveDatabaseIntrospection, isKtxSqliteConnectionConfig } from '@ktx/connector-sqlite'; -import { createSqlServerLiveDatabaseIntrospection, isKtxSqlServerConnectionConfig } from '@ktx/connector-sqlserver'; -import { - BigQueryHistoricSqlQueryHistoryReader, - createDaemonLiveDatabaseIntrospection, - createDefaultLocalIngestAdapters, - type DefaultLocalIngestAdaptersOptions, - type HistoricSqlReader, - type LiveDatabaseIntrospectionPort, - LiveDatabaseSourceAdapter, - PostgresPgssReader, - SnowflakeHistoricSqlQueryHistoryReader, - type SourceAdapter, -} from '@ktx/context/ingest'; -import type { KtxLocalProject } from '@ktx/context/project'; -import { createHttpSqlAnalysisPort, type SqlAnalysisPort } from '@ktx/context/sql-analysis'; +import { createBigQueryLiveDatabaseIntrospection } from './connectors/bigquery/live-database-introspection.js'; +import { isKtxBigQueryConnectionConfig, KtxBigQueryScanConnector, type KtxBigQueryConnectionConfig } from './connectors/bigquery/connector.js'; +import { createClickHouseLiveDatabaseIntrospection } from './connectors/clickhouse/live-database-introspection.js'; +import { isKtxClickHouseConnectionConfig } from './connectors/clickhouse/connector.js'; +import { createMysqlLiveDatabaseIntrospection } from './connectors/mysql/live-database-introspection.js'; +import { isKtxMysqlConnectionConfig } from './connectors/mysql/connector.js'; +import { createPostgresLiveDatabaseIntrospection } from './connectors/postgres/live-database-introspection.js'; +import { isKtxPostgresConnectionConfig, type KtxPostgresConnectionConfig } from './connectors/postgres/connector.js'; +import { KtxPostgresHistoricSqlQueryClient } from './connectors/postgres/historic-sql-query-client.js'; +import { createSqliteLiveDatabaseIntrospection } from './connectors/sqlite/live-database-introspection.js'; +import { isKtxSqliteConnectionConfig } from './connectors/sqlite/connector.js'; +import { createSqlServerLiveDatabaseIntrospection } from './connectors/sqlserver/live-database-introspection.js'; +import { isKtxSqlServerConnectionConfig } from './connectors/sqlserver/connector.js'; +import { BigQueryHistoricSqlQueryHistoryReader } from './context/ingest/adapters/historic-sql/bigquery-query-history-reader.js'; +import { createDaemonLiveDatabaseIntrospection } from './context/ingest/adapters/live-database/daemon-introspection.js'; +import { createDefaultLocalIngestAdapters, type DefaultLocalIngestAdaptersOptions } from './context/ingest/local-adapters.js'; +import type { HistoricSqlReader } from './context/ingest/adapters/historic-sql/types.js'; +import type { LiveDatabaseIntrospectionPort } from './context/ingest/adapters/live-database/types.js'; +import { LiveDatabaseSourceAdapter } from './context/ingest/adapters/live-database/live-database.adapter.js'; +import { PostgresPgssReader } from './context/ingest/adapters/historic-sql/postgres-pgss-reader.js'; +import { SnowflakeHistoricSqlQueryHistoryReader } from './context/ingest/adapters/historic-sql/snowflake-query-history-reader.js'; +import type { SourceAdapter } from './context/ingest/types.js'; +import type { KtxLocalProject } from './context/project/project.js'; +import { createHttpSqlAnalysisPort } from './context/sql-analysis/http-sql-analysis-port.js'; +import type { SqlAnalysisPort } from './context/sql-analysis/ports.js'; import { createManagedDaemonLookerTableIdentifierParser, createManagedDaemonSqlAnalysisPort, @@ -44,7 +39,7 @@ function hasSnowflakeDriver(connection: unknown): boolean { ); } -type SnowflakeConnectorModule = typeof import('@ktx/connector-snowflake'); +type SnowflakeConnectorModule = typeof import('./connectors/snowflake/connector.js'); function ktxCliDaemonDatabaseIntrospectionOptions( options: KtxCliLocalIngestAdaptersOptions, @@ -141,9 +136,8 @@ function createKtxCliLiveDatabaseIntrospection( return bigquery.extractSchema(connectionId); } if (hasSnowflakeDriver(connection)) { - const { createSnowflakeLiveDatabaseIntrospection, isKtxSnowflakeConnectionConfig } = await import( - '@ktx/connector-snowflake' - ); + const { createSnowflakeLiveDatabaseIntrospection } = await import('./connectors/snowflake/live-database-introspection.js'); + const { isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/connector.js');; if (!isKtxSnowflakeConnectionConfig(connection)) { return daemon.extractSchema(connectionId); } @@ -342,7 +336,7 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli reader: new SnowflakeHistoricSqlQueryHistoryReader() satisfies HistoricSqlReader, queryClient: { async executeQuery(query: string) { - const connectorModule = await import('@ktx/connector-snowflake'); + const connectorModule = await import('./connectors/snowflake/connector.js'); const client = await createEphemeralSnowflakeHistoricSqlClient(project, connectionId, connectorModule); return client.executeQuery(query); }, diff --git a/packages/cli/src/local-scan-connectors.test.ts b/packages/cli/src/local-scan-connectors.test.ts index b9672bfa..a993faa2 100644 --- a/packages/cli/src/local-scan-connectors.test.ts +++ b/packages/cli/src/local-scan-connectors.test.ts @@ -1,7 +1,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject, loadKtxProject } from '@ktx/context/project'; +import { initKtxProject, loadKtxProject } from './context/project/project.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { createKtxCliScanConnector } from './local-scan-connectors.js'; @@ -12,7 +12,7 @@ const bigQueryMock = vi.hoisted(() => ({ }>, })); -vi.mock('@ktx/connector-bigquery', () => ({ +vi.mock('./connectors/bigquery/connector.js', () => ({ isKtxBigQueryConnectionConfig: (connection: { driver?: unknown } | undefined) => String(connection?.driver ?? '').toLowerCase() === 'bigquery', KtxBigQueryScanConnector: class { diff --git a/packages/cli/src/local-scan-connectors.ts b/packages/cli/src/local-scan-connectors.ts index 885ee5a7..10b2dd05 100644 --- a/packages/cli/src/local-scan-connectors.ts +++ b/packages/cli/src/local-scan-connectors.ts @@ -1,5 +1,5 @@ -import type { KtxLocalProject } from '@ktx/context/project'; -import type { KtxScanConnector } from '@ktx/context/scan'; +import type { KtxLocalProject } from './context/project/project.js'; +import type { KtxScanConnector } from './context/scan/types.js'; const SUPPORTED_DRIVERS = 'sqlite, postgres, mysql, clickhouse, sqlserver, bigquery, snowflake'; @@ -18,49 +18,49 @@ export async function createKtxCliScanConnector( ); } if (driver === 'sqlite' || driver === 'sqlite3') { - const { KtxSqliteScanConnector, isKtxSqliteConnectionConfig } = await import('@ktx/connector-sqlite'); + const { KtxSqliteScanConnector, isKtxSqliteConnectionConfig } = await import('./connectors/sqlite/connector.js');; if (!isKtxSqliteConnectionConfig(connection)) { throw invalidConnectionConfigError(connectionId, driver); } return new KtxSqliteScanConnector({ connectionId, connection, projectDir: project.projectDir }); } if (driver === 'postgres' || driver === 'postgresql') { - const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('@ktx/connector-postgres'); + const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('./connectors/postgres/connector.js');; if (!isKtxPostgresConnectionConfig(connection)) { throw invalidConnectionConfigError(connectionId, driver); } return new KtxPostgresScanConnector({ connectionId, connection }); } if (driver === 'mysql') { - const { KtxMysqlScanConnector, isKtxMysqlConnectionConfig } = await import('@ktx/connector-mysql'); + const { KtxMysqlScanConnector, isKtxMysqlConnectionConfig } = await import('./connectors/mysql/connector.js');; if (!isKtxMysqlConnectionConfig(connection)) { throw invalidConnectionConfigError(connectionId, driver); } return new KtxMysqlScanConnector({ connectionId, connection }); } if (driver === 'clickhouse') { - const { KtxClickHouseScanConnector, isKtxClickHouseConnectionConfig } = await import('@ktx/connector-clickhouse'); + const { KtxClickHouseScanConnector, isKtxClickHouseConnectionConfig } = await import('./connectors/clickhouse/connector.js');; if (!isKtxClickHouseConnectionConfig(connection)) { throw invalidConnectionConfigError(connectionId, driver); } return new KtxClickHouseScanConnector({ connectionId, connection }); } if (driver === 'sqlserver') { - const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('@ktx/connector-sqlserver'); + const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('./connectors/sqlserver/connector.js');; if (!isKtxSqlServerConnectionConfig(connection)) { throw invalidConnectionConfigError(connectionId, driver); } return new KtxSqlServerScanConnector({ connectionId, connection }); } if (driver === 'bigquery') { - const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('@ktx/connector-bigquery'); + const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('./connectors/bigquery/connector.js');; if (!isKtxBigQueryConnectionConfig(connection)) { throw invalidConnectionConfigError(connectionId, driver); } return new KtxBigQueryScanConnector({ connectionId, connection }); } if (driver === 'snowflake') { - const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('@ktx/connector-snowflake'); + const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/connector.js');; if (!isKtxSnowflakeConnectionConfig(connection)) { throw invalidConnectionConfigError(connectionId, driver); } diff --git a/packages/cli/src/managed-local-embeddings.ts b/packages/cli/src/managed-local-embeddings.ts index 48e83c86..b178be47 100644 --- a/packages/cli/src/managed-local-embeddings.ts +++ b/packages/cli/src/managed-local-embeddings.ts @@ -1,4 +1,4 @@ -import type { KtxEmbeddingConfig } from '@ktx/llm'; +import type { KtxEmbeddingConfig } from './llm/types.js'; import type { KtxCliIo } from './cli-runtime.js'; import { ensureManagedPythonCommandRuntime, diff --git a/packages/cli/src/managed-mcp-daemon.ts b/packages/cli/src/managed-mcp-daemon.ts index dd3fb821..881574de 100644 --- a/packages/cli/src/managed-mcp-daemon.ts +++ b/packages/cli/src/managed-mcp-daemon.ts @@ -17,6 +17,7 @@ export interface KtxMcpDaemonState { logPath: string; } +/** @internal */ export interface KtxMcpDaemonChild { pid?: number; unref(): void; diff --git a/packages/cli/src/managed-python-command.ts b/packages/cli/src/managed-python-command.ts index 83953602..ecad702f 100644 --- a/packages/cli/src/managed-python-command.ts +++ b/packages/cli/src/managed-python-command.ts @@ -1,4 +1,4 @@ -import { createPythonSemanticLayerComputePort, type KtxSemanticLayerComputePort } from '@ktx/context/daemon'; +import { createPythonSemanticLayerComputePort, type KtxSemanticLayerComputePort } from './context/daemon/semantic-layer-compute.js'; import type { KtxCliIo } from './cli-runtime.js'; import { createClackPromptAdapter, createStaticCliSpinner, type KtxCliSpinner } from './clack.js'; import { @@ -33,7 +33,7 @@ export interface ManagedPythonCommandRuntime { manifest: InstalledKtxRuntimeManifest; } -export interface ManagedPythonCommandDeps { +interface ManagedPythonCommandDeps { readStatus?: (options: ManagedPythonRuntimeLayoutOptions) => Promise; installRuntime?: (options: ManagedPythonRuntimeInstallOptions) => Promise; confirmInstall?: (message: string, io: KtxCliIo) => Promise; @@ -51,6 +51,7 @@ export interface ManagedPythonSemanticLayerComputeOptions extends ManagedPythonC createPythonCompute?: typeof createPythonSemanticLayerComputePort; } +/** @internal */ export function managedRuntimeInstallCommand(feature: KtxRuntimeFeature): string { return feature === 'local-embeddings' ? 'ktx admin runtime install --feature local-embeddings --yes' diff --git a/packages/cli/src/managed-python-daemon.ts b/packages/cli/src/managed-python-daemon.ts index e36435c3..7bc92e14 100644 --- a/packages/cli/src/managed-python-daemon.ts +++ b/packages/cli/src/managed-python-daemon.ts @@ -51,7 +51,7 @@ export interface ManagedPythonDaemonProcessInfo { command: string; } -export type ManagedPythonDaemonStopAllSource = 'state' | 'process'; +type ManagedPythonDaemonStopAllSource = 'state' | 'process'; export interface ManagedPythonDaemonStopAllEntry { pid: number; @@ -74,11 +74,13 @@ export interface ManagedPythonDaemonStopAllResult { scanErrors: string[]; } +/** @internal */ export interface ManagedPythonDaemonChild { pid?: number; unref(): void; } +/** @internal */ export type ManagedPythonDaemonSpawn = ( command: string, args: string[], @@ -89,6 +91,7 @@ export type ManagedPythonDaemonSpawn = ( }, ) => ManagedPythonDaemonChild; +/** @internal */ export type ManagedPythonDaemonFetch = ( url: string, ) => Promise<{ @@ -98,7 +101,7 @@ export type ManagedPythonDaemonFetch = ( text(): Promise; }>; -export type ManagedPythonDaemonKillProcess = (pid: number, signal?: NodeJS.Signals) => void; +type ManagedPythonDaemonKillProcess = (pid: number, signal?: NodeJS.Signals) => void; export interface ManagedPythonDaemonStartOptions extends ManagedPythonDaemonLayoutOptions { features: KtxRuntimeFeature[]; diff --git a/packages/cli/src/managed-python-http.ts b/packages/cli/src/managed-python-http.ts index e45caabb..0c9b24b3 100644 --- a/packages/cli/src/managed-python-http.ts +++ b/packages/cli/src/managed-python-http.ts @@ -1,18 +1,11 @@ import { request as httpRequest } from 'node:http'; import { request as httpsRequest } from 'node:https'; import { URL } from 'node:url'; -import { - createDaemonLookerTableIdentifierParser, - type DaemonLiveDatabaseIntrospectionOptions, - type KtxDaemonDatabaseHttpJsonRunner, - type KtxDaemonTableIdentifierHttpJsonRunner, - type LookerTableIdentifierParser, -} from '@ktx/context/ingest'; -import { - createHttpSqlAnalysisPort, - type KtxSqlAnalysisHttpJsonRunner, - type SqlAnalysisPort, -} from '@ktx/context/sql-analysis'; +import { createDaemonLookerTableIdentifierParser, type KtxDaemonTableIdentifierHttpJsonRunner } from './context/ingest/adapters/looker/daemon-table-identifier-parser.js'; +import type { DaemonLiveDatabaseIntrospectionOptions, KtxDaemonDatabaseHttpJsonRunner } from './context/ingest/adapters/live-database/daemon-introspection.js'; +import type { LookerTableIdentifierParser } from './context/ingest/adapters/looker/mapping.js'; +import { createHttpSqlAnalysisPort, type KtxSqlAnalysisHttpJsonRunner } from './context/sql-analysis/http-sql-analysis-port.js'; +import type { SqlAnalysisPort } from './context/sql-analysis/ports.js'; import type { KtxCliIo } from './cli-runtime.js'; import { ensureManagedPythonCommandRuntime, @@ -21,12 +14,13 @@ import { } from './managed-python-command.js'; import { startManagedPythonDaemon, type ManagedPythonDaemonStartResult } from './managed-python-daemon.js'; +/** @internal */ export type ManagedPythonHttpJsonRunner = ( path: string, payload: Record, ) => Promise>; -export type ManagedPythonHttpPostJson = ( +type ManagedPythonHttpPostJson = ( baseUrl: string, path: string, payload: Record, @@ -75,7 +69,7 @@ function parseJsonObject(raw: string, path: string): Record { return parsed as Record; } -export async function postManagedDaemonJson( +async function postManagedDaemonJson( baseUrl: string, path: string, payload: Record, @@ -117,6 +111,7 @@ export async function postManagedDaemonJson( }); } +/** @internal */ export function createManagedPythonDaemonBaseUrlResolver( options: ManagedPythonCoreDaemonOptions, ): () => Promise { @@ -158,6 +153,7 @@ function isResolveBaseUrlOnly( return 'resolveBaseUrl' in options; } +/** @internal */ export function createManagedDaemonHttpJsonRunner( options: ManagedPythonDaemonHttpOptions, ): ManagedPythonHttpJsonRunner { diff --git a/packages/cli/src/managed-python-runtime.ts b/packages/cli/src/managed-python-runtime.ts index 68272840..5aef2b32 100644 --- a/packages/cli/src/managed-python-runtime.ts +++ b/packages/cli/src/managed-python-runtime.ts @@ -25,7 +25,7 @@ const runtimeAssetManifestSchema = z.object({ }), }); -export type KtxRuntimeAssetManifest = z.infer; +type KtxRuntimeAssetManifest = z.infer; const installedRuntimeManifestSchema = z.object({ schemaVersion: z.literal(1), @@ -76,6 +76,7 @@ export interface ManagedPythonDaemonLayout extends ManagedPythonRuntimeLayout { daemonStderrPath: string; } +/** @internal */ export interface ManagedRuntimeAsset { manifest: KtxRuntimeAssetManifest; wheelPath: string; @@ -104,7 +105,7 @@ export interface ManagedPythonRuntimeInstallResult { manifest: InstalledKtxRuntimeManifest; } -export type ManagedPythonRuntimeStatusKind = 'missing' | 'ready' | 'mismatched' | 'broken'; +type ManagedPythonRuntimeStatusKind = 'missing' | 'ready' | 'mismatched' | 'broken'; export interface ManagedPythonRuntimeStatus { kind: ManagedPythonRuntimeStatusKind; @@ -121,6 +122,7 @@ export interface ManagedPythonRuntimeDoctorCheck { fix?: string; } +/** @internal */ export const MISSING_UV_RUNTIME_INSTALL_MESSAGE = 'uv is required to install the KTX Python runtime. KTX does not download uv automatically. Install uv, make sure it is on PATH, and retry: ktx admin runtime install --yes'; @@ -142,6 +144,7 @@ function executablePath(venvDir: string, platform: NodeJS.Platform, name: string return join(venvDir, 'bin', name); } +/** @internal */ export function managedPythonRuntimeLayout(options: ManagedPythonRuntimeLayoutOptions): ManagedPythonRuntimeLayout { const platform = options.platform ?? process.platform; const env = options.env ?? process.env; @@ -235,6 +238,7 @@ function parseRequiresPythonFromWheel(input: { wheelPath: string; contents: Buff }; } +/** @internal */ export async function verifyRuntimeAsset(input: { assetDir: string }): Promise { const manifestPath = join(input.assetDir, 'manifest.json'); let manifestData: unknown; diff --git a/packages/cli/src/mcp-http-server.ts b/packages/cli/src/mcp-http-server.ts index 0d4607f0..a95dcbbe 100644 --- a/packages/cli/src/mcp-http-server.ts +++ b/packages/cli/src/mcp-http-server.ts @@ -1,10 +1,10 @@ import { randomUUID } from 'node:crypto'; import { createServer, type IncomingHttpHeaders, type IncomingMessage, type Server, type ServerResponse } from 'node:http'; -import { loadKtxProject } from '@ktx/context/project'; +import { loadKtxProject } from './context/project/project.js'; import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; import { isInitializeRequest } from '@modelcontextprotocol/sdk/types.js'; -import type { KtxCliIo } from './cli-runtime.js'; +import { getKtxCliPackageInfo, type KtxCliIo } from './cli-runtime.js'; import { createKtxMcpServerFactory } from './mcp-server-factory.js'; const DEFAULT_ALLOWED_HOSTS = ['localhost', '127.0.0.1', '::1'] as const; @@ -25,6 +25,7 @@ export interface McpSecurityConfig { allowedOrigins: string[]; } +/** @internal */ export type McpAuthorizationResult = | { ok: true } | { ok: false; status: 401 | 403; message: string }; @@ -34,6 +35,7 @@ function isLoopbackHost(host: string): boolean { return normalized === 'localhost' || normalized === '127.0.0.1' || normalized === '::1'; } +/** @internal */ export function normalizeHostHeader(value: string): string { const trimmed = value.trim().toLowerCase(); if (trimmed.startsWith('[')) { @@ -85,6 +87,7 @@ function headerValue(headers: IncomingHttpHeaders | Record }, config: McpSecurityConfig, @@ -175,7 +178,7 @@ export async function runKtxMcpHttpServer(options: RunKtxMcpHttpServerOptions): (await createKtxMcpServerFactory({ project: project!, projectDir: options.projectDir, - cliVersion: options.cliVersion ?? '0.0.0-private', + cliVersion: options.cliVersion ?? getKtxCliPackageInfo().version, io: options.io, })); const sessions = new Map(); diff --git a/packages/cli/src/mcp-server-factory.test.ts b/packages/cli/src/mcp-server-factory.test.ts index 779e008a..64c7275d 100644 --- a/packages/cli/src/mcp-server-factory.test.ts +++ b/packages/cli/src/mcp-server-factory.test.ts @@ -1,6 +1,7 @@ -import { createDefaultKtxMcpServer, createLocalProjectMcpContextPorts } from '@ktx/context/mcp'; -import { createLocalProjectMemoryIngest } from '@ktx/context/memory'; import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { createDefaultKtxMcpServer } from './context/mcp/server.js'; +import { createLocalProjectMcpContextPorts } from './context/mcp/local-project-ports.js'; +import { createLocalProjectMemoryIngest } from './context/memory/local-memory.js'; import { resolveProjectEmbeddingProvider } from './embedding-resolution.js'; import { createKtxCliScanConnector } from './local-scan-connectors.js'; import { createKtxMcpServerFactory } from './mcp-server-factory.js'; @@ -18,7 +19,7 @@ const mocks = vi.hoisted(() => ({ memoryIngest: { ingest: vi.fn(), status: vi.fn(), waitForRun: vi.fn() }, })); -vi.mock('@ktx/context', () => ({ +vi.mock('./context/llm/embedding-port.js', () => ({ KtxIngestEmbeddingPortAdapter: class { readonly maxBatchSize: number; @@ -36,12 +37,15 @@ vi.mock('@ktx/context', () => ({ }, })); -vi.mock('@ktx/context/mcp', () => ({ +vi.mock('./context/mcp/server.js', () => ({ createDefaultKtxMcpServer: vi.fn(() => ({ kind: 'mcp-server' })), +})); + +vi.mock('./context/mcp/local-project-ports.js', () => ({ createLocalProjectMcpContextPorts: vi.fn(() => ({ context_tool: { name: 'context_tool' } })), })); -vi.mock('@ktx/context/memory', () => ({ +vi.mock('./context/memory/local-memory.js', () => ({ createLocalProjectMemoryIngest: vi.fn(() => mocks.memoryIngest), })); diff --git a/packages/cli/src/mcp-server-factory.ts b/packages/cli/src/mcp-server-factory.ts index 1528a350..e6d4887f 100644 --- a/packages/cli/src/mcp-server-factory.ts +++ b/packages/cli/src/mcp-server-factory.ts @@ -1,7 +1,8 @@ -import { KtxIngestEmbeddingPortAdapter } from '@ktx/context'; -import { createDefaultKtxMcpServer, createLocalProjectMcpContextPorts } from '@ktx/context/mcp'; -import { createLocalProjectMemoryIngest } from '@ktx/context/memory'; -import type { KtxLocalProject } from '@ktx/context/project'; +import { KtxIngestEmbeddingPortAdapter } from './context/llm/embedding-port.js'; +import { createDefaultKtxMcpServer } from './context/mcp/server.js'; +import { createLocalProjectMcpContextPorts } from './context/mcp/local-project-ports.js'; +import { createLocalProjectMemoryIngest } from './context/memory/local-memory.js'; +import type { KtxLocalProject } from './context/project/project.js'; import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import type { KtxCliIo } from './cli-runtime.js'; import { resolveProjectEmbeddingProvider } from './embedding-resolution.js'; diff --git a/packages/cli/src/mcp-stdio-server.ts b/packages/cli/src/mcp-stdio-server.ts index 6a7cc996..93b5a155 100644 --- a/packages/cli/src/mcp-stdio-server.ts +++ b/packages/cli/src/mcp-stdio-server.ts @@ -1,9 +1,9 @@ import process from 'node:process'; import type { Readable, Writable } from 'node:stream'; -import { loadKtxProject } from '@ktx/context/project'; +import { loadKtxProject } from './context/project/project.js'; import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; -import type { KtxCliIo } from './cli-runtime.js'; +import { getKtxCliPackageInfo, type KtxCliIo } from './cli-runtime.js'; import { createKtxMcpServerFactory } from './mcp-server-factory.js'; export interface RunKtxMcpStdioServerOptions { @@ -30,7 +30,7 @@ export async function runKtxMcpStdioServer(options: RunKtxMcpStdioServerOptions) (await createKtxMcpServerFactory({ project: project!, projectDir: options.projectDir, - cliVersion: options.cliVersion ?? '0.0.0-private', + cliVersion: options.cliVersion ?? getKtxCliPackageInfo().version, io: protocolIo, })); const stdin = options.stdin ?? process.stdin; diff --git a/packages/cli/src/memory-flow-hud.tsx b/packages/cli/src/memory-flow-hud.tsx index 9a9b3d96..8b044122 100644 --- a/packages/cli/src/memory-flow-hud.tsx +++ b/packages/cli/src/memory-flow-hud.tsx @@ -1,5 +1,5 @@ /* @jsxImportSource react */ -import type { MemoryFlowEvent, MemoryFlowReplayInput } from '@ktx/context/ingest/memory-flow'; +import type { MemoryFlowEvent, MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; import { Box, Text } from 'ink'; import { type ReactNode } from 'react'; import { buildDemoMetrics, formatCost, formatDuration } from './demo-metrics.js'; diff --git a/packages/cli/src/memory-flow-interactive.test.ts b/packages/cli/src/memory-flow-interactive.test.ts index d7fe8bd8..d6976a03 100644 --- a/packages/cli/src/memory-flow-interactive.test.ts +++ b/packages/cli/src/memory-flow-interactive.test.ts @@ -1,5 +1,5 @@ import { EventEmitter } from 'node:events'; -import type { MemoryFlowReplayInput } from '@ktx/context/ingest'; +import type { MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; import { describe, expect, it, vi } from 'vitest'; import { memoryFlowCommandForKey, renderMemoryFlowInteractively } from './memory-flow-interactive.js'; diff --git a/packages/cli/src/memory-flow-interactive.ts b/packages/cli/src/memory-flow-interactive.ts index 7998a978..b445f025 100644 --- a/packages/cli/src/memory-flow-interactive.ts +++ b/packages/cli/src/memory-flow-interactive.ts @@ -1,13 +1,8 @@ import { emitKeypressEvents } from 'node:readline'; -import { - buildMemoryFlowViewModel, - createInitialMemoryFlowInteractionState, - reduceMemoryFlowInteractionState, - renderMemoryFlowInteractive, - type MemoryFlowInteractionCommand, - type MemoryFlowInteractionState, - type MemoryFlowReplayInput, -} from '@ktx/context/ingest'; +import { buildMemoryFlowViewModel } from './context/ingest/memory-flow/view-model.js'; +import { createInitialMemoryFlowInteractionState, reduceMemoryFlowInteractionState } from './context/ingest/memory-flow/interaction.js'; +import { renderMemoryFlowInteractive } from './context/ingest/memory-flow/interactive-render.js'; +import type { MemoryFlowInteractionCommand, MemoryFlowInteractionState, MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; interface KtxMemoryFlowKey { name?: string; @@ -42,6 +37,7 @@ function defaultPrepareKeypressEvents(stdin: KtxMemoryFlowStdin): void { emitKeypressEvents(stdin as Parameters[0]); } +/** @internal */ export function memoryFlowCommandForKey( chunk: string, search: MemoryFlowInteractionState['search'], diff --git a/packages/cli/src/memory-flow-tui.test.tsx b/packages/cli/src/memory-flow-tui.test.tsx index 405fa18c..09d50125 100644 --- a/packages/cli/src/memory-flow-tui.test.tsx +++ b/packages/cli/src/memory-flow-tui.test.tsx @@ -1,5 +1,5 @@ /* @jsxImportSource react */ -import type { MemoryFlowReplayInput } from '@ktx/context/ingest'; +import type { MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; import { render as renderInkTest } from 'ink-testing-library'; import React, { type ReactNode } from 'react'; import { describe, expect, it, vi } from 'vitest'; diff --git a/packages/cli/src/memory-flow-tui.tsx b/packages/cli/src/memory-flow-tui.tsx index 37b9f2af..5e5176cb 100644 --- a/packages/cli/src/memory-flow-tui.tsx +++ b/packages/cli/src/memory-flow-tui.tsx @@ -1,17 +1,19 @@ /* @jsxImportSource react */ +import { buildMemoryFlowViewModel } from './context/ingest/memory-flow/view-model.js'; import { - buildMemoryFlowViewModel, createInitialMemoryFlowInteractionState, findMemoryFlowSearchMatches, - type MemoryFlowColumnId, - type MemoryFlowInteractionCommand, - type MemoryFlowInteractionState, - type MemoryFlowReplayInput, - type MemoryFlowViewModel, reduceMemoryFlowInteractionState, selectedMemoryFlowColumn, selectedMemoryFlowDetails, -} from '@ktx/context/ingest'; +} from './context/ingest/memory-flow/interaction.js'; +import type { + MemoryFlowColumnId, + MemoryFlowInteractionCommand, + MemoryFlowInteractionState, + MemoryFlowReplayInput, + MemoryFlowViewModel, +} from './context/ingest/memory-flow/types.js'; import { Box, Text, render as renderInkRuntime, useApp, useInput } from 'ink'; import { type ReactNode, useEffect, useMemo, useRef, useState } from 'react'; import { @@ -66,6 +68,7 @@ export interface MemoryFlowTuiLiveSession { isClosed(): boolean; } +/** @internal */ export interface MemoryFlowInkInstance { rerender(tree: ReactNode): void; unmount(): void; @@ -73,7 +76,7 @@ export interface MemoryFlowInkInstance { clear?(): void; } -export interface MemoryFlowInkRenderOptions { +interface MemoryFlowInkRenderOptions { stdin?: KtxMemoryFlowTuiIo['stdin']; stdout: KtxMemoryFlowTuiIo['stdout']; stderr: KtxMemoryFlowTuiIo['stderr']; @@ -157,6 +160,7 @@ export function sanitizeMemoryFlowTuiError(error: unknown): string { .replace(/\b(api[_-]?key|password|token|secret)=\S+/gi, '[redacted]'); } +/** @internal */ export function memoryFlowCommandForInkInput( input: string, key: InkKey, @@ -285,6 +289,7 @@ function TrustIssues(props: { view: MemoryFlowViewModel; theme: MemoryFlowTuiThe ); } +/** @internal */ export function MemoryFlowTuiApp(props: MemoryFlowTuiAppProps): ReactNode { const app = useApp(); const totalEvents = props.input.events.length; diff --git a/packages/cli/src/next-steps.ts b/packages/cli/src/next-steps.ts index 5410eee8..80a1b441 100644 --- a/packages/cli/src/next-steps.ts +++ b/packages/cli/src/next-steps.ts @@ -1,3 +1,4 @@ +/** @internal */ export const KTX_CONTEXT_BUILD_COMMANDS = [ { command: 'ktx ingest', @@ -24,9 +25,10 @@ export const KTX_NEXT_STEP_DIRECT_COMMANDS = [ }, ] as const; +/** @internal */ export const KTX_NEXT_STEP_COMMANDS = [...KTX_NEXT_STEP_DIRECT_COMMANDS] as const; -export const KTX_NEXT_STEP_COMMAND_WIDTH = Math.max( +const KTX_NEXT_STEP_COMMAND_WIDTH = Math.max( ...[...KTX_CONTEXT_BUILD_COMMANDS, ...KTX_NEXT_STEP_COMMANDS].map((step) => step.command.length), ); diff --git a/packages/cli/src/notion-page-picker.ts b/packages/cli/src/notion-page-picker.ts index 26e561f5..066acec1 100644 --- a/packages/cli/src/notion-page-picker.ts +++ b/packages/cli/src/notion-page-picker.ts @@ -1,6 +1,6 @@ -import { resolveNotionConnectionAuthToken } from '@ktx/context/connections'; -import { type NotionApi, type NotionBotInfo, NotionClient } from '@ktx/context/ingest'; -import type { KtxProjectConnectionConfig } from '@ktx/context/project'; +import { resolveNotionConnectionAuthToken } from './context/connections/notion-config.js'; +import { type NotionApi, type NotionBotInfo, NotionClient } from './context/ingest/adapters/notion/notion-client.js'; +import type { KtxProjectConnectionConfig } from './context/project/config.js'; import type { KtxCliIo } from './cli-runtime.js'; import { profileMark } from './startup-profile.js'; import { @@ -24,6 +24,7 @@ export interface PickNotionRootPagesArgs { connection: KtxProjectConnectionConfig; } +/** @internal */ export type NotionPickerApi = Pick; export type NotionRootPagePickResult = | { kind: 'selected'; rootPageIds: string[] } @@ -50,6 +51,7 @@ function assertSafeNotionPickerConnectionId(connectionId: string): void { } } +/** @internal */ export function normalizeNotionPageId(value: string): string { const trimmed = value.trim(); const compact = trimmed.includes('-') ? trimmed.replace(/-/g, '') : trimmed; @@ -106,6 +108,7 @@ function extractParentPageId(page: Record): string | null { return normalizeNotionPageId(parent.page_id); } +/** @internal */ export function notionPickerPageFromSearchResult(result: Record): TreePickerNodeInput { const id = typeof result.id === 'string' ? normalizeNotionPageId(result.id) : ''; if (!id) { @@ -119,6 +122,7 @@ export function notionPickerPageFromSearchResult(result: Record }; } +/** @internal */ export async function discoverNotionPickerPages( api: NotionPickerApi, options: { cap?: number } = {}, @@ -161,6 +165,7 @@ export async function discoverNotionPickerPages( return { pages, cappedAtCount: cap, warnings }; } +/** @internal */ export async function resolveNotionWorkspaceLabel(api: NotionPickerApi, connectionId: string): Promise { try { const bot = (await api.retrieveBotUser()) as NotionBotInfo; diff --git a/packages/cli/src/print-command-tree.ts b/packages/cli/src/print-command-tree.ts index be643546..389891b1 100644 --- a/packages/cli/src/print-command-tree.ts +++ b/packages/cli/src/print-command-tree.ts @@ -1,6 +1,6 @@ import { fileURLToPath } from 'node:url'; import { buildKtxProgram } from './cli-program.js'; -import type { KtxCliIo, KtxCliPackageInfo } from './cli-runtime.js'; +import { getKtxCliPackageInfo, type KtxCliIo } from './cli-runtime.js'; import { formatCommandTree, walkCommandTree } from './command-tree.js'; function silentIo(): KtxCliIo { @@ -10,19 +10,11 @@ function silentIo(): KtxCliIo { }; } -function stubPackageInfo(): KtxCliPackageInfo { - return { - name: '@ktx/cli', - version: '0.0.0-docs', - contextPackageName: '@ktx/context', - }; -} - export function renderKtxCommandTree(): string { const program = buildKtxProgram({ io: silentIo(), deps: {}, - packageInfo: stubPackageInfo(), + packageInfo: getKtxCliPackageInfo(), runInit: async () => 0, }); return formatCommandTree(walkCommandTree(program)); diff --git a/packages/cli/src/prompt-navigation.ts b/packages/cli/src/prompt-navigation.ts index c3644338..619ee027 100644 --- a/packages/cli/src/prompt-navigation.ts +++ b/packages/cli/src/prompt-navigation.ts @@ -38,6 +38,7 @@ function withTextInputBodySpacing(message: string): string { return `${title}\n\n${bodyLines.join('\n')}`; } +/** @internal */ export function withMenuOptionSpacing(message: string): string { if (!message.includes('\n') || message.endsWith('\n')) { return message; diff --git a/packages/context/prompts/memory_agent_backfill.md b/packages/cli/src/prompts/memory_agent_backfill.md similarity index 100% rename from packages/context/prompts/memory_agent_backfill.md rename to packages/cli/src/prompts/memory_agent_backfill.md diff --git a/packages/context/prompts/memory_agent_bundle_ingest_reconcile.md b/packages/cli/src/prompts/memory_agent_bundle_ingest_reconcile.md similarity index 100% rename from packages/context/prompts/memory_agent_bundle_ingest_reconcile.md rename to packages/cli/src/prompts/memory_agent_bundle_ingest_reconcile.md diff --git a/packages/context/prompts/memory_agent_bundle_ingest_work_unit.md b/packages/cli/src/prompts/memory_agent_bundle_ingest_work_unit.md similarity index 100% rename from packages/context/prompts/memory_agent_bundle_ingest_work_unit.md rename to packages/cli/src/prompts/memory_agent_bundle_ingest_work_unit.md diff --git a/packages/context/prompts/memory_agent_external_ingest.md b/packages/cli/src/prompts/memory_agent_external_ingest.md similarity index 100% rename from packages/context/prompts/memory_agent_external_ingest.md rename to packages/cli/src/prompts/memory_agent_external_ingest.md diff --git a/packages/context/prompts/memory_agent_research.md b/packages/cli/src/prompts/memory_agent_research.md similarity index 100% rename from packages/context/prompts/memory_agent_research.md rename to packages/cli/src/prompts/memory_agent_research.md diff --git a/packages/context/prompts/skills/light_extraction.md b/packages/cli/src/prompts/skills/light_extraction.md similarity index 100% rename from packages/context/prompts/skills/light_extraction.md rename to packages/cli/src/prompts/skills/light_extraction.md diff --git a/packages/context/prompts/skills/page_triage_classifier.md b/packages/cli/src/prompts/skills/page_triage_classifier.md similarity index 100% rename from packages/context/prompts/skills/page_triage_classifier.md rename to packages/cli/src/prompts/skills/page_triage_classifier.md diff --git a/packages/cli/src/public-ingest.test.ts b/packages/cli/src/public-ingest.test.ts index 47b80074..ac2560cd 100644 --- a/packages/cli/src/public-ingest.test.ts +++ b/packages/cli/src/public-ingest.test.ts @@ -1,4 +1,4 @@ -import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from '@ktx/context/project'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; import { describe, expect, it, vi } from 'vitest'; import { buildPublicIngestPlan, @@ -801,6 +801,63 @@ describe('runKtxPublicIngest', () => { ); }); + it('preflights foreground managed embeddings runtime before starting the context-build view', async () => { + const io = makeIo({ isTTY: true, interactive: true }); + const config = buildDefaultKtxProjectConfig(); + const project: KtxPublicIngestProject = { + projectDir: '/tmp/project', + config: { + ...config, + connections: { + warehouse: { driver: 'postgres' }, + }, + ingest: { + ...config.ingest, + embeddings: { + backend: 'sentence-transformers', + model: 'all-MiniLM-L6-v2', + dimensions: 384, + }, + }, + }, + }; + const ensureRuntime = vi.fn(async (): Promise => { + return {} as ManagedPythonCommandRuntime; + }); + const runContextBuild = vi.fn(async () => ({ exitCode: 0 })); + + await expect( + runKtxPublicIngest( + { + command: 'run', + projectDir: '/tmp/project', + targetConnectionId: 'warehouse', + all: false, + json: false, + inputMode: 'auto', + queryHistory: 'default', + cliVersion: '0.2.0', + runtimeInstallPolicy: 'prompt', + }, + io.io, + { + loadProject: vi.fn(async () => project), + ensureRuntime, + runContextBuild, + }, + ), + ).resolves.toBe(0); + + expect(ensureRuntime).toHaveBeenCalledWith( + expect.objectContaining({ + cliVersion: '0.2.0', + installPolicy: 'prompt', + feature: 'local-embeddings', + }), + ); + expect(runContextBuild).toHaveBeenCalled(); + }); + it('runs all independent targets and reports partial failures', async () => { const io = makeIo(); const project = projectWithConnections({ diff --git a/packages/cli/src/public-ingest.ts b/packages/cli/src/public-ingest.ts index eb5a47fd..60b9622c 100644 --- a/packages/cli/src/public-ingest.ts +++ b/packages/cli/src/public-ingest.ts @@ -1,5 +1,7 @@ -import { loadKtxProject, type KtxLocalProject, type KtxProjectConnectionConfig } from '@ktx/context/project'; -import type { KtxProgressPort } from '@ktx/context/scan'; +import { getKtxCliPackageInfo } from './cli-runtime.js'; +import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; +import type { KtxProjectConnectionConfig } from './context/project/config.js'; +import type { KtxProgressPort } from './context/scan/types.js'; import type { KtxCliIo } from './index.js'; import type { KtxIngestArgs, KtxIngestDeps, KtxIngestProgressUpdate } from './ingest.js'; import { @@ -872,12 +874,15 @@ export async function runKtxPublicIngest( const project = await loadProject({ projectDir: args.projectDir }); if (shouldUseForegroundContextBuildView(args, io)) { const plan = buildPublicIngestPlan(project, args); - const requirements = resolvePublicIngestRuntimeRequirements(plan, { env: deps.env ?? process.env }); + const requirements = resolvePublicIngestRuntimeRequirements(plan, { + config: project.config, + env: deps.env ?? process.env, + }); const ensureRuntime = deps.ensureRuntime ?? ensureManagedPythonCommandRuntime; for (const feature of requirements.features) { try { await ensureRuntime({ - cliVersion: args.cliVersion ?? '0.0.0-private', + cliVersion: args.cliVersion ?? getKtxCliPackageInfo().version, installPolicy: args.runtimeInstallPolicy ?? 'prompt', io, feature, diff --git a/packages/cli/src/runtime-requirements.test.ts b/packages/cli/src/runtime-requirements.test.ts index 2d86ed89..5f8831cf 100644 --- a/packages/cli/src/runtime-requirements.test.ts +++ b/packages/cli/src/runtime-requirements.test.ts @@ -1,4 +1,4 @@ -import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from '@ktx/context/project'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; import { describe, expect, it } from 'vitest'; import { resolveProjectRuntimeRequirements, @@ -60,21 +60,36 @@ describe('runtime requirement detection', () => { }); it('detects foreground ingest runtime needs from selected query-history targets', () => { + const config: KtxProjectConfig = { + ...buildDefaultKtxProjectConfig(), + ingest: { + ...buildDefaultKtxProjectConfig().ingest, + embeddings: { + backend: 'sentence-transformers' as const, + model: 'all-MiniLM-L6-v2', + dimensions: 384, + }, + }, + }; + expect( - resolvePublicIngestRuntimeRequirements({ - projectDir: '/tmp/project', - warnings: [], - targets: [ - { - connectionId: 'warehouse', - driver: 'postgres', - operation: 'database-ingest', - debugCommand: 'ktx ingest warehouse --debug', - steps: ['database-schema', 'query-history'], - queryHistory: { enabled: true }, - }, - ], - }).features, - ).toEqual(['core']); + resolvePublicIngestRuntimeRequirements( + { + projectDir: '/tmp/project', + warnings: [], + targets: [ + { + connectionId: 'warehouse', + driver: 'postgres', + operation: 'database-ingest', + debugCommand: 'ktx ingest warehouse --debug', + steps: ['database-schema', 'query-history'], + queryHistory: { enabled: true }, + }, + ], + }, + { config }, + ).features, + ).toEqual(['core', 'local-embeddings']); }); }); diff --git a/packages/cli/src/runtime-requirements.ts b/packages/cli/src/runtime-requirements.ts index ca95e12e..31ad1be0 100644 --- a/packages/cli/src/runtime-requirements.ts +++ b/packages/cli/src/runtime-requirements.ts @@ -1,8 +1,4 @@ -import type { - KtxProjectConfig, - KtxProjectConnectionConfig, - KtxProjectEmbeddingConfig, -} from '@ktx/context/project'; +import type { KtxProjectConfig, KtxProjectConnectionConfig, KtxProjectEmbeddingConfig } from './context/project/config.js'; import type { KtxRuntimeFeature } from './managed-python-runtime.js'; import type { KtxPublicIngestPlan } from './public-ingest.js'; @@ -29,6 +25,7 @@ export interface KtxProjectRuntimeRequirementOptions { } export interface KtxPublicIngestRuntimeRequirementOptions { + config?: KtxProjectConfig; env?: NodeJS.ProcessEnv | Record; } @@ -153,5 +150,13 @@ export function resolvePublicIngestRuntimeRequirements( } } + if (options.config && requiresManagedLocalEmbeddings(options.config.ingest.embeddings)) { + requirements.push({ + feature: 'local-embeddings', + reason: 'local-embeddings', + detail: 'Local sentence-transformers embeddings use the managed Python runtime.', + }); + } + return uniqueRequirements(requirements); } diff --git a/packages/cli/src/scan.test.ts b/packages/cli/src/scan.test.ts index 9863be12..0d2bcdc9 100644 --- a/packages/cli/src/scan.test.ts +++ b/packages/cli/src/scan.test.ts @@ -1,13 +1,10 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import type { SourceAdapter } from '@ktx/context/ingest'; -import { initKtxProject } from '@ktx/context/project'; -import type { - KtxScanReport, - LocalScanRunResult, - RunLocalScanOptions, -} from '@ktx/context/scan'; +import type { SourceAdapter } from './context/ingest/types.js'; +import { initKtxProject } from './context/project/project.js'; +import type { KtxScanReport } from './context/scan/types.js'; +import type { LocalScanRunResult, RunLocalScanOptions } from './context/scan/local-scan.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { createCliScanProgress, runKtxScan, type KtxScanDeps } from './scan.js'; @@ -138,29 +135,37 @@ const KtxPostgresScanConnector = vi.hoisted( }, ); -vi.mock('@ktx/connector-sqlserver', () => ({ - createSqlServerLiveDatabaseIntrospection, +vi.mock('./connectors/sqlserver/connector.js', () => ({ isKtxSqlServerConnectionConfig, KtxSqlServerScanConnector, })); +vi.mock('./connectors/sqlserver/live-database-introspection.js', () => ({ + createSqlServerLiveDatabaseIntrospection, +})); -vi.mock('@ktx/connector-bigquery', () => ({ - createBigQueryLiveDatabaseIntrospection, +vi.mock('./connectors/bigquery/connector.js', () => ({ isKtxBigQueryConnectionConfig, KtxBigQueryScanConnector, })); +vi.mock('./connectors/bigquery/live-database-introspection.js', () => ({ + createBigQueryLiveDatabaseIntrospection, +})); -vi.mock('@ktx/connector-snowflake', () => ({ - createSnowflakeLiveDatabaseIntrospection, +vi.mock('./connectors/snowflake/connector.js', () => ({ isKtxSnowflakeConnectionConfig, KtxSnowflakeScanConnector, })); +vi.mock('./connectors/snowflake/live-database-introspection.js', () => ({ + createSnowflakeLiveDatabaseIntrospection, +})); -vi.mock('@ktx/connector-postgres', () => ({ - createPostgresLiveDatabaseIntrospection, +vi.mock('./connectors/postgres/connector.js', () => ({ isKtxPostgresConnectionConfig, KtxPostgresScanConnector, })); +vi.mock('./connectors/postgres/live-database-introspection.js', () => ({ + createPostgresLiveDatabaseIntrospection, +})); function makeIo(options: { isTTY?: boolean } = {}) { let stdout = ''; @@ -423,6 +428,55 @@ describe('runKtxScan', () => { }); }); + it('uses runtime IO when resolving managed embedding runtime', async () => { + await initKtxProject({ projectDir: tempDir }); + const runLocalScan = vi.fn( + async (_input: RunLocalScanOptions): Promise => ({ + runId: 'scan-run-1', + status: 'done', + done: true, + connectionId: 'warehouse', + mode: 'structural', + dryRun: false, + syncId: 'sync-1', + report, + }), + ); + const resolveEmbeddingProvider = vi.fn(async () => ({ kind: 'disabled' as const })); + const io = makeIo(); + const runtimeIo = makeIo({ isTTY: true }); + + await expect( + runKtxScan( + { + command: 'run', + projectDir: tempDir, + connectionId: 'warehouse', + mode: 'structural', + detectRelationships: false, + dryRun: false, + cliVersion: '0.2.0', + runtimeInstallPolicy: 'auto', + }, + io.io, + { + runLocalScan, + createLocalIngestAdapters: noLocalIngestAdapters, + runtimeIo: runtimeIo.io, + resolveEmbeddingProvider, + }, + ), + ).resolves.toBe(0); + + expect(resolveEmbeddingProvider).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + installPolicy: 'auto', + io: runtimeIo.io, + }), + ); + }); + it('explains warnings, capability gaps, and relationships in human scan summaries', async () => { await initKtxProject({ projectDir: tempDir }); const runLocalScan = vi.fn( diff --git a/packages/cli/src/scan.ts b/packages/cli/src/scan.ts index 9583d3bf..a92aaa62 100644 --- a/packages/cli/src/scan.ts +++ b/packages/cli/src/scan.ts @@ -1,11 +1,7 @@ -import { - type KtxProgressPort, - type KtxScanMode, - type KtxScanReport, - type KtxScanWarning, - runLocalScan, -} from '@ktx/context/scan'; -import { loadKtxProject } from '@ktx/context/project'; +import type { KtxProgressPort, KtxScanMode, KtxScanReport, KtxScanWarning } from './context/scan/types.js'; +import { runLocalScan } from './context/scan/local-scan.js'; +import { loadKtxProject } from './context/project/project.js'; +import { getKtxCliPackageInfo } from './cli-runtime.js'; import { resolveProjectEmbeddingProvider } from './embedding-resolution.js'; import type { KtxCliIo } from './index.js'; import { createKtxCliLocalIngestAdapters } from './local-adapters.js'; @@ -30,6 +26,7 @@ export interface KtxScanArgs { export interface KtxScanDeps { runLocalScan?: typeof runLocalScan; createLocalIngestAdapters?: typeof createKtxCliLocalIngestAdapters; + resolveEmbeddingProvider?: typeof resolveProjectEmbeddingProvider; progress?: KtxProgressPort; runtimeIo?: KtxCliIo; } @@ -274,6 +271,7 @@ interface KtxCliScanProgress extends Omit { flush(): void; } +/** @internal */ export function createCliScanProgress( io: KtxCliIo, state: KtxCliScanProgressState = { progress: 0, hasPendingTransient: false }, @@ -315,11 +313,12 @@ export function createCliScanProgress( export async function runKtxScan(args: KtxScanArgs, io: KtxCliIo = process, deps: KtxScanDeps = {}): Promise { try { const project = await loadKtxProject({ projectDir: args.projectDir }); - const resolution = await resolveProjectEmbeddingProvider(project, { + const resolveEmbeddingProvider = deps.resolveEmbeddingProvider ?? resolveProjectEmbeddingProvider; + const resolution = await resolveEmbeddingProvider(project, { mode: 'ensure', installPolicy: args.runtimeInstallPolicy ?? 'never', - cliVersion: args.cliVersion ?? '0.0.0-private', - io, + cliVersion: args.cliVersion ?? getKtxCliPackageInfo().version, + io: deps.runtimeIo ?? io, }); const embeddingProvider = resolution.kind === 'disabled' || resolution.kind === 'managed-unavailable' ? null : resolution.provider; diff --git a/packages/cli/src/setup-agents.test.ts b/packages/cli/src/setup-agents.test.ts index e6ca39ed..a5f488d5 100644 --- a/packages/cli/src/setup-agents.test.ts +++ b/packages/cli/src/setup-agents.test.ts @@ -1,7 +1,7 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { readKtxSetupState } from '@ktx/context/project'; +import { readKtxSetupState } from './context/project/setup-config.js'; import { strFromU8, unzipSync } from 'fflate'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { @@ -380,7 +380,7 @@ describe('setup agents', () => { }); }); - it('prompts for MCP-first client agent connection mode in interactive setup', async () => { + it('prompts for MCP-first agent client connection mode in interactive setup', async () => { const io = makeIo(); const prompts = { select: vi.fn(async ({ message }: { message: string }) => (message.startsWith('Where') ? 'project' : 'mcp')), diff --git a/packages/cli/src/setup-agents.ts b/packages/cli/src/setup-agents.ts index ecac2917..240622f6 100644 --- a/packages/cli/src/setup-agents.ts +++ b/packages/cli/src/setup-agents.ts @@ -5,11 +5,9 @@ import type { Writable } from 'node:stream'; import { fileURLToPath } from 'node:url'; import { styleText } from 'node:util'; import { log, outro } from '@clack/prompts'; -import { - loadKtxProject, - markKtxSetupStateStepComplete, - serializeKtxProjectConfig, -} from '@ktx/context/project'; +import { loadKtxProject } from './context/project/project.js'; +import { markKtxSetupStateStepComplete } from './context/project/setup-config.js'; +import { serializeKtxProjectConfig } from './context/project/config.js'; import { strToU8, zipSync } from 'fflate'; import type { KtxCliIo } from './cli-runtime.js'; import { @@ -21,6 +19,7 @@ import { readKtxMcpDaemonStatus } from './managed-mcp-daemon.js'; export type KtxAgentTarget = 'claude-code' | 'claude-desktop' | 'codex' | 'cursor' | 'opencode' | 'universal'; export type KtxAgentScope = 'project' | 'global' | 'local'; +/** @internal */ export type KtxAgentInstallMode = 'mcp' | 'mcp-cli'; export interface KtxSetupAgentsArgs { @@ -118,6 +117,7 @@ function writeSetupOutro(io: KtxCliIo, message: string): void { const STEP_HEADING_RE = /^(\d+)\. (.+)$/; const ACTION_MARKER_RE = /^(RUN|PASTE|USE|OPEN):$/; +/** @internal */ export function createAgentNextActionsLineFormatter( stdout: KtxCliIo['stdout'], ): (line: string) => string { @@ -301,7 +301,7 @@ function claudeDesktopConfigPath(): { path: string; jsonPath: string[] } { const CLAUDE_DESKTOP_FORWARDED_ENV_KEYS = ['OPENAI_API_KEY', 'ANTHROPIC_API_KEY'] as const; -export function collectClaudeDesktopForwardedEnv(source: NodeJS.ProcessEnv): Record { +function collectClaudeDesktopForwardedEnv(source: NodeJS.ProcessEnv): Record { const captured: Record = {}; for (const [key, value] of Object.entries(source)) { if (value === undefined || value === '') continue; @@ -394,7 +394,7 @@ function plannedMcpJsonEntries(input: { return []; } -export function agentInstallManifestPath(projectDir: string): string { +function agentInstallManifestPath(projectDir: string): string { return join(resolve(projectDir), '.ktx/agents/install-manifest.json'); } @@ -410,6 +410,7 @@ function claudeDesktopLauncherPath(projectDir: string): string { return join(resolve(projectDir), '.ktx/agents/claude/ktx-plugin-runner.sh'); } +/** @internal */ export function plannedKtxAgentFiles(input: { projectDir: string; target: KtxAgentTarget; @@ -750,6 +751,7 @@ function mergeManifest( }; } +/** @internal */ export async function removeKtxAgentInstall(projectDir: string, io: KtxCliIo): Promise { const manifest = await readKtxAgentInstallManifest(projectDir); if (!manifest) { @@ -765,7 +767,7 @@ export async function removeKtxAgentInstall(projectDir: string, io: KtxCliIo): P return 0; } -export interface KtxSetupAgentsPromptAdapter { +interface KtxSetupAgentsPromptAdapter { select(options: { message: string; options: KtxSetupPromptOption[] }): Promise; multiselect(options: { message: string; @@ -853,6 +855,7 @@ function hasAdminCliEntries(entries: InstallEntry[]): boolean { ); } +/** @internal */ export interface InstallSummaryEntry { title: string; lines: string[]; @@ -869,6 +872,7 @@ function formatInlinePath(path: string): string { return path; } +/** @internal */ export function formatInstallSummaryLines( installs: Array<{ target: KtxAgentTarget; scope: KtxAgentScope; mode: KtxAgentInstallMode }>, entries: InstallEntry[], diff --git a/packages/cli/src/setup-context.test.ts b/packages/cli/src/setup-context.test.ts index cb6581a8..7bcad93e 100644 --- a/packages/cli/src/setup-context.test.ts +++ b/packages/cli/src/setup-context.test.ts @@ -1,14 +1,8 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { - buildDefaultKtxProjectConfig, - parseKtxProjectConfig, - readKtxSetupState, - serializeKtxProjectConfig, - type KtxProjectConfig, - writeKtxSetupState, -} from '@ktx/context/project'; +import { buildDefaultKtxProjectConfig, parseKtxProjectConfig, serializeKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; +import { readKtxSetupState, writeKtxSetupState } from './context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { diff --git a/packages/cli/src/setup-context.ts b/packages/cli/src/setup-context.ts index 476a11e0..aa519111 100644 --- a/packages/cli/src/setup-context.ts +++ b/packages/cli/src/setup-context.ts @@ -1,13 +1,9 @@ import { mkdirSync, writeFileSync } from 'node:fs'; import { access, mkdir, readdir, readFile, writeFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; -import { - type KtxLocalProject, - loadKtxProject, - markKtxSetupStateStepComplete, - readKtxSetupState, - serializeKtxProjectConfig, -} from '@ktx/context/project'; +import { type KtxLocalProject, loadKtxProject } from './context/project/project.js'; +import { markKtxSetupStateStepComplete, readKtxSetupState } from './context/project/setup-config.js'; +import { serializeKtxProjectConfig } from './context/project/config.js'; import type { KtxCliIo } from './cli-runtime.js'; import { buildPublicIngestPlan } from './public-ingest.js'; import { @@ -25,12 +21,13 @@ import { type KtxSetupPromptOption, } from './setup-prompts.js'; -export type KtxSetupContextBuildStatus = +type KtxSetupContextBuildStatus = | 'not_started' | 'completed' | 'failed' | 'stale'; +/** @internal */ export interface KtxSetupContextCommands { build: string; status: string; @@ -61,7 +58,7 @@ export interface KtxSetupContextStatusSummary { detail?: string; } -export interface KtxSetupContextReadiness { +interface KtxSetupContextReadiness { ready: boolean; agentContextReady: boolean; semanticSearchReady: boolean; @@ -86,7 +83,7 @@ export interface KtxSetupContextStepArgs { runtimeInstallPolicy?: KtxManagedPythonInstallPolicy; } -export interface KtxSetupContextPromptAdapter { +interface KtxSetupContextPromptAdapter { select(options: { message: string; options: KtxSetupPromptOption[] }): Promise; cancel(message: string): void; } @@ -125,6 +122,7 @@ async function pathExists(path: string): Promise { } } +/** @internal */ export function contextBuildCommands(projectDir: string): KtxSetupContextCommands { const resolvedProjectDir = resolve(projectDir); return { @@ -236,6 +234,7 @@ export async function readKtxSetupContextState(projectDir: string): Promise { const resolvedProjectDir = resolve(projectDir); await mkdir(join(resolvedProjectDir, '.ktx', 'setup'), { recursive: true }); diff --git a/packages/cli/src/setup-database-context-depth.ts b/packages/cli/src/setup-database-context-depth.ts index 39e75f71..20df813c 100644 --- a/packages/cli/src/setup-database-context-depth.ts +++ b/packages/cli/src/setup-database-context-depth.ts @@ -1,10 +1,6 @@ import { writeFile } from 'node:fs/promises'; -import { - type KtxLocalProject, - type KtxProjectConnectionConfig, - loadKtxProject, - serializeKtxProjectConfig, -} from '@ktx/context/project'; +import { type KtxLocalProject, loadKtxProject } from './context/project/project.js'; +import { type KtxProjectConnectionConfig, serializeKtxProjectConfig } from './context/project/config.js'; import { type KtxDatabaseContextDepth, databaseContextDepth, diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/src/setup-databases.test.ts index 86a73fa3..178ed076 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/src/setup-databases.test.ts @@ -1,7 +1,9 @@ import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; -import { initKtxProject, parseKtxProjectConfig, readKtxSetupState, writeKtxSetupState } from '@ktx/context/project'; +import { initKtxProject } from './context/project/project.js'; +import { parseKtxProjectConfig } from './context/project/config.js'; +import { readKtxSetupState, writeKtxSetupState } from './context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type KtxSetupDatabaseDriver, diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index 1d41b4da..c8e735c5 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -3,15 +3,11 @@ import { readFile, writeFile } from 'node:fs/promises'; import { delimiter, dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; -import type { HistoricSqlDialect } from '@ktx/context/ingest'; -import { - type KtxProjectConnectionConfig, - loadKtxProject, - markKtxSetupStateStepComplete, - serializeKtxProjectConfig, - setKtxSetupDatabaseConnectionIds, -} from '@ktx/context/project'; -import type { KtxTableListEntry } from '@ktx/context/scan'; +import type { HistoricSqlDialect } from './context/ingest/adapters/historic-sql/types.js'; +import { type KtxProjectConnectionConfig, serializeKtxProjectConfig } from './context/project/config.js'; +import { loadKtxProject } from './context/project/project.js'; +import { markKtxSetupStateStepComplete, setKtxSetupDatabaseConnectionIds } from './context/project/setup-config.js'; +import type { KtxTableListEntry } from './context/scan/types.js'; import type { KtxCliIo } from './cli-runtime.js'; import { runKtxConnection } from './connection.js'; import { @@ -64,6 +60,7 @@ export type KtxSetupDatabasesResult = | { status: 'missing-input'; projectDir: string } | { status: 'failed'; projectDir: string }; +/** @internal */ export interface KtxSetupDatabasesPromptAdapter { multiselect(options: { message: string; @@ -327,8 +324,12 @@ async function defaultHistoricSqlProbe(input: KtxSetupHistoricSqlProbeInput): Pr const project = await loadKtxProject({ projectDir: input.projectDir }); const connection = project.config.connections[input.connectionId]; - const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient, isKtxPostgresConnectionConfig }] = - await Promise.all([import('@ktx/context/ingest'), import('@ktx/connector-postgres')]); + const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient }, { isKtxPostgresConnectionConfig }] = + await Promise.all([ + import('./context/ingest/adapters/historic-sql/postgres-pgss-reader.js'), + import('./connectors/postgres/historic-sql-query-client.js'), + import('./connectors/postgres/connector.js'), + ]); const postgresConnection = connection as Parameters[0]; if (!isKtxPostgresConnectionConfig(postgresConnection)) { @@ -364,7 +365,7 @@ async function defaultListSchemas(projectDir: string, connectionId: string): Pro const driver = normalizeDriver(connection?.driver); if (driver === 'postgres') { - const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('@ktx/connector-postgres'); + const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('./connectors/postgres/connector.js');; if (!isKtxPostgresConnectionConfig(connection)) return []; const connector = new KtxPostgresScanConnector({ connectionId, connection }); try { @@ -375,7 +376,7 @@ async function defaultListSchemas(projectDir: string, connectionId: string): Pro } if (driver === 'sqlserver') { - const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('@ktx/connector-sqlserver'); + const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('./connectors/sqlserver/connector.js');; if (!isKtxSqlServerConnectionConfig(connection)) return []; const connector = new KtxSqlServerScanConnector({ connectionId, connection }); try { @@ -386,7 +387,7 @@ async function defaultListSchemas(projectDir: string, connectionId: string): Pro } if (driver === 'bigquery') { - const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('@ktx/connector-bigquery'); + const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('./connectors/bigquery/connector.js');; if (!isKtxBigQueryConnectionConfig(connection)) return []; const connector = new KtxBigQueryScanConnector({ connectionId, connection }); try { @@ -397,7 +398,7 @@ async function defaultListSchemas(projectDir: string, connectionId: string): Pro } if (driver === 'snowflake') { - const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('@ktx/connector-snowflake'); + const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/connector.js');; if (!isKtxSnowflakeConnectionConfig(connection)) return []; const connector = new KtxSnowflakeScanConnector({ connectionId, connection }); try { @@ -429,7 +430,7 @@ async function defaultListTables( const schemas = schemasOverride ?? (driver ? configuredSchemas(connection, driver) : undefined); if (driver === 'postgres') { - const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('@ktx/connector-postgres'); + const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('./connectors/postgres/connector.js');; if (!isKtxPostgresConnectionConfig(connection)) return []; const connector = new KtxPostgresScanConnector({ connectionId, connection }); try { @@ -440,7 +441,7 @@ async function defaultListTables( } if (driver === 'mysql') { - const { KtxMysqlScanConnector, isKtxMysqlConnectionConfig } = await import('@ktx/connector-mysql'); + const { KtxMysqlScanConnector, isKtxMysqlConnectionConfig } = await import('./connectors/mysql/connector.js');; if (!isKtxMysqlConnectionConfig(connection)) return []; const connector = new KtxMysqlScanConnector({ connectionId, connection }); try { @@ -451,7 +452,7 @@ async function defaultListTables( } if (driver === 'sqlserver') { - const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('@ktx/connector-sqlserver'); + const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('./connectors/sqlserver/connector.js');; if (!isKtxSqlServerConnectionConfig(connection)) return []; const connector = new KtxSqlServerScanConnector({ connectionId, connection }); try { @@ -462,7 +463,7 @@ async function defaultListTables( } if (driver === 'bigquery') { - const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('@ktx/connector-bigquery'); + const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('./connectors/bigquery/connector.js');; if (!isKtxBigQueryConnectionConfig(connection)) return []; const connector = new KtxBigQueryScanConnector({ connectionId, connection }); try { @@ -473,7 +474,7 @@ async function defaultListTables( } if (driver === 'snowflake') { - const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('@ktx/connector-snowflake'); + const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/connector.js');; if (!isKtxSnowflakeConnectionConfig(connection)) return []; const connector = new KtxSnowflakeScanConnector({ connectionId, connection }); try { @@ -484,7 +485,7 @@ async function defaultListTables( } if (driver === 'clickhouse') { - const { KtxClickHouseScanConnector, isKtxClickHouseConnectionConfig } = await import('@ktx/connector-clickhouse'); + const { KtxClickHouseScanConnector, isKtxClickHouseConnectionConfig } = await import('./connectors/clickhouse/connector.js');; if (!isKtxClickHouseConnectionConfig(connection)) return []; const connector = new KtxClickHouseScanConnector({ connectionId, connection }); try { diff --git a/packages/cli/src/setup-demo-tour.ts b/packages/cli/src/setup-demo-tour.ts index b2acb9f1..79f71fbe 100644 --- a/packages/cli/src/setup-demo-tour.ts +++ b/packages/cli/src/setup-demo-tour.ts @@ -64,6 +64,7 @@ function createTargetState(target: KtxPublicIngestPlanTarget): ContextBuildTarge // Pure rendering functions // --------------------------------------------------------------------------- +/** @internal */ export function renderDemoBanner(projectDir?: string): string { const lines = [ '', @@ -76,6 +77,7 @@ export function renderDemoBanner(projectDir?: string): string { return lines.join('\n'); } +/** @internal */ export function renderDemoCardContent(title: string, selections: string[]): string { const lines = [ `┌ ${title}`, @@ -88,6 +90,7 @@ export function renderDemoCardContent(title: string, selections: string[]): stri return lines.join('\n'); } +/** @internal */ export function renderDemoAgentTransition(): string { const lines = [ '┌ Demo project is ready — let\'s connect your agent', @@ -99,6 +102,7 @@ export function renderDemoAgentTransition(): string { return lines.join('\n'); } +/** @internal */ export function renderDemoCompletionSummary(projectDir: string, agentInstalled: boolean): string { const lines: string[] = [ '', @@ -129,7 +133,7 @@ export function renderDemoCompletionSummary(projectDir: string, agentInstalled: // Keypress navigation // --------------------------------------------------------------------------- -export async function waitForDemoNavigation( +async function waitForDemoNavigation( stdin?: NodeJS.ReadStream, ): Promise<'forward' | 'back'> { const input = stdin ?? process.stdin; @@ -169,7 +173,7 @@ export async function waitForDemoNavigation( // Interactive card // --------------------------------------------------------------------------- -export async function renderDemoCard( +async function renderDemoCard( title: string, selections: string[], io: KtxCliIo, @@ -186,6 +190,7 @@ export async function renderDemoCard( // Context build replay // --------------------------------------------------------------------------- +/** @internal */ export interface DemoReplayEvent { delayMs: number; connectionId: string; @@ -194,6 +199,7 @@ export interface DemoReplayEvent { summaryText: string | null; } +/** @internal */ export const DEMO_REPLAY_TARGETS = { primarySources: [ createDemoTarget('postgres-warehouse', 'database-ingest', 'postgres'), @@ -205,6 +211,7 @@ export const DEMO_REPLAY_TARGETS = { ], } as const; +/** @internal */ export function buildDemoReplayTimeline(): DemoReplayEvent[] { return [ // postgres-warehouse: database schema context @@ -239,7 +246,7 @@ function renderDemoContextCompletionSummary(): string { return lines.join('\n'); } -export async function runDemoContextReplay( +async function runDemoContextReplay( io: KtxCliIo, stdin?: NodeJS.ReadStream, ): Promise<'forward' | 'back'> { diff --git a/packages/cli/src/setup-embeddings.test.ts b/packages/cli/src/setup-embeddings.test.ts index 2fd6c541..bf9e2b2d 100644 --- a/packages/cli/src/setup-embeddings.test.ts +++ b/packages/cli/src/setup-embeddings.test.ts @@ -1,7 +1,9 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject, parseKtxProjectConfig, readKtxSetupState, writeKtxSetupState } from '@ktx/context/project'; +import { initKtxProject } from './context/project/project.js'; +import { parseKtxProjectConfig } from './context/project/config.js'; +import { readKtxSetupState, writeKtxSetupState } from './context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type KtxSetupEmbeddingsPromptAdapter, runKtxSetupEmbeddingsStep } from './setup-embeddings.js'; diff --git a/packages/cli/src/setup-embeddings.ts b/packages/cli/src/setup-embeddings.ts index 5fb8b8a0..0aedd264 100644 --- a/packages/cli/src/setup-embeddings.ts +++ b/packages/cli/src/setup-embeddings.ts @@ -1,14 +1,10 @@ import { readFile, writeFile } from 'node:fs/promises'; -import { resolveKtxConfigReference } from '@ktx/context/core'; -import { - type KtxProjectConfig, - type KtxProjectEmbeddingConfig, - loadKtxProject, - markKtxSetupStateStepComplete, - readKtxSetupState, - serializeKtxProjectConfig, -} from '@ktx/context/project'; -import { type KtxEmbeddingConfig, type KtxEmbeddingHealthCheckResult, runKtxEmbeddingHealthCheck } from '@ktx/llm'; +import { resolveKtxConfigReference } from './context/core/config-reference.js'; +import { type KtxProjectConfig, type KtxProjectEmbeddingConfig, serializeKtxProjectConfig } from './context/project/config.js'; +import { loadKtxProject } from './context/project/project.js'; +import { markKtxSetupStateStepComplete, readKtxSetupState } from './context/project/setup-config.js'; +import type { KtxEmbeddingConfig } from './llm/types.js'; +import { type KtxEmbeddingHealthCheckResult, runKtxEmbeddingHealthCheck } from './llm/embedding-health.js'; import type { KtxCliIo } from './cli-runtime.js'; import { createStaticCliSpinner, type KtxCliSpinner } from './clack.js'; import { @@ -46,6 +42,7 @@ export type KtxSetupEmbeddingsResult = | { status: 'missing-input'; projectDir: string } | { status: 'failed'; projectDir: string }; +/** @internal */ export interface KtxSetupEmbeddingsPromptAdapter { select(options: { message: string; options: KtxSetupPromptOption[] }): Promise; password(options: { message: string }): Promise; diff --git a/packages/cli/src/setup-interrupt.ts b/packages/cli/src/setup-interrupt.ts index 9baa0f1f..782abbf5 100644 --- a/packages/cli/src/setup-interrupt.ts +++ b/packages/cli/src/setup-interrupt.ts @@ -9,6 +9,7 @@ export class KtxSetupExitError extends Error { } } +/** @internal */ export interface SetupInterruptTracker { track(run: () => Promise): Promise; wasCtrlC(): boolean; diff --git a/packages/cli/src/setup-models.test.ts b/packages/cli/src/setup-models.test.ts index 82fc9461..ba9260be 100644 --- a/packages/cli/src/setup-models.test.ts +++ b/packages/cli/src/setup-models.test.ts @@ -1,7 +1,9 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject, parseKtxProjectConfig, readKtxSetupState, writeKtxSetupState } from '@ktx/context/project'; +import { initKtxProject } from './context/project/project.js'; +import { parseKtxProjectConfig } from './context/project/config.js'; +import { readKtxSetupState, writeKtxSetupState } from './context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { BUNDLED_ANTHROPIC_MODELS, diff --git a/packages/cli/src/setup-models.ts b/packages/cli/src/setup-models.ts index e24800e4..7f7385b1 100644 --- a/packages/cli/src/setup-models.ts +++ b/packages/cli/src/setup-models.ts @@ -1,16 +1,14 @@ import { execFile } from 'node:child_process'; import { writeFile } from 'node:fs/promises'; import { promisify } from 'node:util'; -import { resolveLocalKtxLlmConfig, runClaudeCodeAuthProbe } from '@ktx/context'; -import { resolveKtxConfigReference } from '@ktx/context/core'; -import { - type KtxProjectConfig, - type KtxProjectLlmConfig, - loadKtxProject, - markKtxSetupStateStepComplete, - serializeKtxProjectConfig, -} from '@ktx/context/project'; -import { type KtxLlmConfig, type KtxLlmHealthCheckResult, runKtxLlmHealthCheck } from '@ktx/llm'; +import { resolveLocalKtxLlmConfig } from './context/llm/local-config.js'; +import { runClaudeCodeAuthProbe } from './context/llm/claude-code-runtime.js'; +import { resolveKtxConfigReference } from './context/core/config-reference.js'; +import { type KtxProjectConfig, type KtxProjectLlmConfig, serializeKtxProjectConfig } from './context/project/config.js'; +import { loadKtxProject } from './context/project/project.js'; +import { markKtxSetupStateStepComplete } from './context/project/setup-config.js'; +import type { KtxLlmConfig } from './llm/types.js'; +import { type KtxLlmHealthCheckResult, runKtxLlmHealthCheck } from './llm/model-health.js'; import { formatClaudeCodePromptCachingWarning, ignoredClaudeCodePromptCachingFields, @@ -51,6 +49,7 @@ export type KtxSetupModelResult = | { status: 'missing-input'; projectDir: string } | { status: 'failed'; projectDir: string }; +/** @internal */ export interface AnthropicModelChoice { id: string; label: string; @@ -59,6 +58,7 @@ export interface AnthropicModelChoice { export type KtxSetupLlmBackend = 'anthropic' | 'vertex' | 'claude-code'; +/** @internal */ export interface KtxSetupModelPromptAdapter { select(options: { message: string; options: KtxSetupPromptOption[] }): Promise; text(options: { message: string; placeholder?: string }): Promise; @@ -82,8 +82,7 @@ export interface KtxSetupModelDeps { spinner?: () => KtxCliSpinner; } -export const BUNDLED_ANTHROPIC_MODEL_REGISTRY_VERSION = '2026-05-07'; - +/** @internal */ export const BUNDLED_ANTHROPIC_MODELS: AnthropicModelChoice[] = [ { id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: true }, { id: 'claude-opus-4-6', label: 'Claude Opus 4.6', recommended: false }, @@ -131,7 +130,7 @@ const execFileAsync = promisify(execFile); type AnthropicModelDiscoveryErrorReason = 'authentication' | 'http' | 'empty-response'; -export class AnthropicModelDiscoveryError extends Error { +class AnthropicModelDiscoveryError extends Error { constructor( message: string, public readonly reason: AnthropicModelDiscoveryErrorReason, @@ -212,6 +211,7 @@ async function defaultListGcloudProjects(): Promise { .filter((project): project is GcloudProjectChoice => Boolean(project)); } +/** @internal */ export async function fetchAnthropicModels( apiKey: string, fetchFn: typeof fetch = fetch, diff --git a/packages/cli/src/setup-project.test.ts b/packages/cli/src/setup-project.test.ts index 14cf18f7..26b1e35b 100644 --- a/packages/cli/src/setup-project.test.ts +++ b/packages/cli/src/setup-project.test.ts @@ -1,7 +1,9 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject, parseKtxProjectConfig, readKtxSetupState } from '@ktx/context/project'; +import { initKtxProject } from './context/project/project.js'; +import { parseKtxProjectConfig } from './context/project/config.js'; +import { readKtxSetupState } from './context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { gray } from './io/symbols.js'; import { type KtxSetupProjectPromptAdapter, runKtxSetupProjectStep } from './setup-project.js'; diff --git a/packages/cli/src/setup-project.ts b/packages/cli/src/setup-project.ts index 04bd54f5..d7d189e1 100644 --- a/packages/cli/src/setup-project.ts +++ b/packages/cli/src/setup-project.ts @@ -2,14 +2,9 @@ import { existsSync } from 'node:fs'; import { mkdir, readdir, readFile, stat, writeFile } from 'node:fs/promises'; import { homedir } from 'node:os'; import { join, resolve } from 'node:path'; -import { - initKtxProject, - type KtxLocalProject, - loadKtxProject, - markKtxSetupStateStepComplete, - mergeKtxSetupGitignoreEntries, - serializeKtxProjectConfig, -} from '@ktx/context/project'; +import { initKtxProject, type KtxLocalProject, loadKtxProject } from './context/project/project.js'; +import { markKtxSetupStateStepComplete, mergeKtxSetupGitignoreEntries } from './context/project/setup-config.js'; +import { serializeKtxProjectConfig } from './context/project/config.js'; import type { KtxCliIo } from './cli-runtime.js'; import { gray } from './io/symbols.js'; import { withTextInputNavigation } from './prompt-navigation.js'; @@ -18,8 +13,8 @@ import { type KtxSetupPromptOption, } from './setup-prompts.js'; -export type KtxSetupProjectMode = 'auto' | 'prompt-new'; -export type KtxSetupInputMode = 'auto' | 'disabled'; +type KtxSetupProjectMode = 'auto' | 'prompt-new'; +type KtxSetupInputMode = 'auto' | 'disabled'; export interface KtxSetupProjectArgs { projectDir: string; @@ -45,6 +40,7 @@ export type KtxSetupProjectResult = | { status: 'cancelled'; projectDir: string } | { status: 'missing-input'; projectDir: string }; +/** @internal */ export interface KtxSetupProjectPromptAdapter { select(options: { message: string; options: KtxSetupPromptOption[] }): Promise; text(options: { message: string; placeholder?: string }): Promise; diff --git a/packages/cli/src/setup-ready-menu.ts b/packages/cli/src/setup-ready-menu.ts index afb89a0f..f1f736e4 100644 --- a/packages/cli/src/setup-ready-menu.ts +++ b/packages/cli/src/setup-ready-menu.ts @@ -14,7 +14,7 @@ export type KtxSetupReadyAction = | 'agents' | 'exit'; -export interface KtxSetupReadyMenuPromptAdapter { +interface KtxSetupReadyMenuPromptAdapter { select(options: { message: string; options: KtxSetupPromptOption[] }): Promise; cancel(message: string): void; } diff --git a/packages/cli/src/setup-runtime.test.ts b/packages/cli/src/setup-runtime.test.ts index 32342702..2fb1f1f2 100644 --- a/packages/cli/src/setup-runtime.test.ts +++ b/packages/cli/src/setup-runtime.test.ts @@ -1,7 +1,8 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { buildDefaultKtxProjectConfig, readKtxSetupState, type KtxProjectConfig } from '@ktx/context/project'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; +import { readKtxSetupState } from './context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runKtxSetupRuntimeStep } from './setup-runtime.js'; diff --git a/packages/cli/src/setup-runtime.ts b/packages/cli/src/setup-runtime.ts index cedcd3d0..25612065 100644 --- a/packages/cli/src/setup-runtime.ts +++ b/packages/cli/src/setup-runtime.ts @@ -1,8 +1,5 @@ -import { - loadKtxProject, - markKtxSetupStateStepComplete, - type KtxLocalProject, -} from '@ktx/context/project'; +import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; +import { markKtxSetupStateStepComplete } from './context/project/setup-config.js'; import type { KtxCliIo } from './cli-runtime.js'; import { ensureManagedLocalEmbeddingsDaemon, diff --git a/packages/cli/src/setup-sources-notion.test.ts b/packages/cli/src/setup-sources-notion.test.ts index b85eefff..84fce4aa 100644 --- a/packages/cli/src/setup-sources-notion.test.ts +++ b/packages/cli/src/setup-sources-notion.test.ts @@ -1,12 +1,8 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { - initKtxProject, - type KtxProjectConnectionConfig, - parseKtxProjectConfig, - serializeKtxProjectConfig, -} from '@ktx/context/project'; +import { initKtxProject } from './context/project/project.js'; +import { type KtxProjectConnectionConfig, parseKtxProjectConfig, serializeKtxProjectConfig } from './context/project/config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runKtxSetupSourcesStep, @@ -19,8 +15,8 @@ const notionMocks = vi.hoisted(() => ({ retrievePage: vi.fn(async () => ({ id: 'page-1' })), })); -vi.mock('@ktx/context/ingest', async (importOriginal) => { - const actual = await importOriginal(); +vi.mock('./context/ingest/adapters/notion/notion-client.js', async (importOriginal) => { + const actual = await importOriginal(); return { ...actual, NotionClient: vi.fn().mockImplementation(function NotionClient(token: string) { diff --git a/packages/cli/src/setup-sources.test.ts b/packages/cli/src/setup-sources.test.ts index bfcb54bc..a9de4436 100644 --- a/packages/cli/src/setup-sources.test.ts +++ b/packages/cli/src/setup-sources.test.ts @@ -1,13 +1,9 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { - initKtxProject, - type KtxProjectConnectionConfig, - parseKtxProjectConfig, - readKtxSetupState, - serializeKtxProjectConfig, -} from '@ktx/context/project'; +import { initKtxProject } from './context/project/project.js'; +import { type KtxProjectConnectionConfig, parseKtxProjectConfig, serializeKtxProjectConfig } from './context/project/config.js'; +import { readKtxSetupState } from './context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { KtxCliIo } from './cli-runtime.js'; import { diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index 164ec2e8..ff8eb420 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -2,32 +2,20 @@ import { mkdtemp, readdir, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, relative, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; -import { - localConnectionTypeForConfig, - resolveNotionConnectionAuthToken, -} from '@ktx/context/connections'; -import { resolveKtxConfigReference } from '@ktx/context/core'; -import { - cloneOrPull, - DEFAULT_METABASE_CLIENT_CONFIG, - discoverMetabaseDatabases, - type DiscoveredMetabaseDatabase, - loadDbtSchemaFiles, - loadProjectInfo, - MetabaseClient, - type NotionApi, - NotionClient, - parseLookmlStagedDir, - parseMetricflowFiles, - testRepoConnection, -} from '@ktx/context/ingest'; -import { - type KtxProjectConfig, - type KtxProjectConnectionConfig, - loadKtxProject, - markKtxSetupStateStepComplete, - serializeKtxProjectConfig, -} from '@ktx/context/project'; +import { localConnectionTypeForConfig } from './context/connections/local-warehouse-descriptor.js'; +import { resolveNotionConnectionAuthToken } from './context/connections/notion-config.js'; +import { resolveKtxConfigReference } from './context/core/config-reference.js'; +import { cloneOrPull, testRepoConnection } from './context/ingest/repo-fetch.js'; +import { DEFAULT_METABASE_CLIENT_CONFIG, MetabaseClient } from './context/ingest/adapters/metabase/client.js'; +import { discoverMetabaseDatabases, type DiscoveredMetabaseDatabase } from './context/ingest/adapters/metabase/mapping.js'; +import { loadDbtSchemaFiles } from './context/ingest/dbt-shared/schema-files.js'; +import { loadProjectInfo } from './context/ingest/dbt-shared/project-vars.js'; +import { type NotionApi, NotionClient } from './context/ingest/adapters/notion/notion-client.js'; +import { parseLookmlStagedDir } from './context/ingest/adapters/lookml/parse.js'; +import { parseMetricflowFiles } from './context/ingest/adapters/metricflow/deep-parse.js'; +import { type KtxProjectConfig, type KtxProjectConnectionConfig, serializeKtxProjectConfig } from './context/project/config.js'; +import { loadKtxProject } from './context/project/project.js'; +import { markKtxSetupStateStepComplete } from './context/project/setup-config.js'; import type { KtxCliIo } from './cli-runtime.js'; import { pickNotionRootPages } from './notion-page-picker.js'; import { runKtxSourceMapping } from './source-mapping.js'; @@ -89,7 +77,7 @@ export interface KtxSetupSourcesPromptAdapter { log?(message: string): void; } -export type SourceValidationResult = { ok: true; detail?: string } | { ok: false; message: string }; +type SourceValidationResult = { ok: true; detail?: string } | { ok: false; message: string }; export interface KtxSetupSourcesDeps { prompts?: KtxSetupSourcesPromptAdapter; diff --git a/packages/cli/src/setup.test.ts b/packages/cli/src/setup.test.ts index 4ada92e3..9c4a3b58 100644 --- a/packages/cli/src/setup.test.ts +++ b/packages/cli/src/setup.test.ts @@ -3,7 +3,7 @@ import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; -import { writeKtxSetupState } from '@ktx/context/project'; +import { writeKtxSetupState } from './context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { localFakeBundleReport, persistLocalBundleReport } from './ingest.test-utils.js'; diff --git a/packages/cli/src/setup.ts b/packages/cli/src/setup.ts index fbf40aa8..9b4c3a24 100644 --- a/packages/cli/src/setup.ts +++ b/packages/cli/src/setup.ts @@ -1,13 +1,11 @@ import { existsSync } from 'node:fs'; import { rm } from 'node:fs/promises'; import { basename, join, resolve } from 'node:path'; -import { getLatestLocalIngestStatus, savedMemoryCountsForReport } from '@ktx/context/ingest'; -import { - ktxLocalStateDbPath, - loadKtxProject, - readKtxSetupState, - type KtxLocalProject, -} from '@ktx/context/project'; +import { getLatestLocalIngestStatus } from './context/ingest/local-ingest.js'; +import { savedMemoryCountsForReport } from './context/ingest/reports.js'; +import { ktxLocalStateDbPath } from './context/project/local-state-db.js'; +import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; +import { readKtxSetupState } from './context/project/setup-config.js'; import type { KtxCliIo } from './cli-runtime.js'; import { formatSetupNextStepLines } from './next-steps.js'; import { runtimeInstallPolicyFromFlags } from './managed-python-command.js'; diff --git a/packages/context/skills/_shared/identifier-verification.md b/packages/cli/src/skills/_shared/identifier-verification.md similarity index 100% rename from packages/context/skills/_shared/identifier-verification.md rename to packages/cli/src/skills/_shared/identifier-verification.md diff --git a/packages/context/skills/dbt_ingest/SKILL.md b/packages/cli/src/skills/dbt_ingest/SKILL.md similarity index 100% rename from packages/context/skills/dbt_ingest/SKILL.md rename to packages/cli/src/skills/dbt_ingest/SKILL.md diff --git a/packages/context/skills/historic_sql_patterns/SKILL.md b/packages/cli/src/skills/historic_sql_patterns/SKILL.md similarity index 100% rename from packages/context/skills/historic_sql_patterns/SKILL.md rename to packages/cli/src/skills/historic_sql_patterns/SKILL.md diff --git a/packages/context/skills/historic_sql_table_digest/SKILL.md b/packages/cli/src/skills/historic_sql_table_digest/SKILL.md similarity index 100% rename from packages/context/skills/historic_sql_table_digest/SKILL.md rename to packages/cli/src/skills/historic_sql_table_digest/SKILL.md diff --git a/packages/context/skills/ingest_triage/SKILL.md b/packages/cli/src/skills/ingest_triage/SKILL.md similarity index 100% rename from packages/context/skills/ingest_triage/SKILL.md rename to packages/cli/src/skills/ingest_triage/SKILL.md diff --git a/packages/context/skills/live_database_ingest/SKILL.md b/packages/cli/src/skills/live_database_ingest/SKILL.md similarity index 100% rename from packages/context/skills/live_database_ingest/SKILL.md rename to packages/cli/src/skills/live_database_ingest/SKILL.md diff --git a/packages/context/skills/looker_ingest/SKILL.md b/packages/cli/src/skills/looker_ingest/SKILL.md similarity index 100% rename from packages/context/skills/looker_ingest/SKILL.md rename to packages/cli/src/skills/looker_ingest/SKILL.md diff --git a/packages/context/skills/lookml_ingest/SKILL.md b/packages/cli/src/skills/lookml_ingest/SKILL.md similarity index 100% rename from packages/context/skills/lookml_ingest/SKILL.md rename to packages/cli/src/skills/lookml_ingest/SKILL.md diff --git a/packages/context/skills/metabase_ingest/SKILL.md b/packages/cli/src/skills/metabase_ingest/SKILL.md similarity index 100% rename from packages/context/skills/metabase_ingest/SKILL.md rename to packages/cli/src/skills/metabase_ingest/SKILL.md diff --git a/packages/context/skills/metricflow_ingest/SKILL.md b/packages/cli/src/skills/metricflow_ingest/SKILL.md similarity index 100% rename from packages/context/skills/metricflow_ingest/SKILL.md rename to packages/cli/src/skills/metricflow_ingest/SKILL.md diff --git a/packages/context/skills/notion_synthesize/SKILL.md b/packages/cli/src/skills/notion_synthesize/SKILL.md similarity index 100% rename from packages/context/skills/notion_synthesize/SKILL.md rename to packages/cli/src/skills/notion_synthesize/SKILL.md diff --git a/packages/context/skills/sl/SKILL.md b/packages/cli/src/skills/sl/SKILL.md similarity index 100% rename from packages/context/skills/sl/SKILL.md rename to packages/cli/src/skills/sl/SKILL.md diff --git a/packages/context/skills/sl_capture/SKILL.md b/packages/cli/src/skills/sl_capture/SKILL.md similarity index 100% rename from packages/context/skills/sl_capture/SKILL.md rename to packages/cli/src/skills/sl_capture/SKILL.md diff --git a/packages/context/skills/wiki_capture/SKILL.md b/packages/cli/src/skills/wiki_capture/SKILL.md similarity index 100% rename from packages/context/skills/wiki_capture/SKILL.md rename to packages/cli/src/skills/wiki_capture/SKILL.md diff --git a/packages/cli/src/sl.test.ts b/packages/cli/src/sl.test.ts index b78dfcba..f1fe0b89 100644 --- a/packages/cli/src/sl.test.ts +++ b/packages/cli/src/sl.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { stripVTControlCharacters } from 'node:util'; import Database from 'better-sqlite3'; -import { initKtxProject } from '@ktx/context/project'; +import { initKtxProject } from './context/project/project.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runKtxSl } from './sl.js'; diff --git a/packages/cli/src/sl.ts b/packages/cli/src/sl.ts index 530aef03..0e5e0fc0 100644 --- a/packages/cli/src/sl.ts +++ b/packages/cli/src/sl.ts @@ -1,18 +1,13 @@ import { readFile } from 'node:fs/promises'; -import { createDefaultLocalQueryExecutor, type KtxSqlQueryExecutorPort } from '@ktx/context/connections'; -import { KtxIngestEmbeddingPortAdapter, type KtxEmbeddingPort } from '@ktx/context'; -import type { KtxSemanticLayerComputePort } from '@ktx/context/daemon'; -import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project'; -import { - compileLocalSlQuery, - listLocalSlSources, - readLocalSlSource, - searchLocalSlSources as defaultSearchLocalSlSources, - validateLocalSlSource, - type LocalSlSourceSearchResult, - type LocalSlSourceSummary, - type SemanticLayerQueryInput, -} from '@ktx/context/sl'; +import { createDefaultLocalQueryExecutor } from './context/connections/local-query-executor.js'; +import type { KtxSqlQueryExecutorPort } from './context/connections/query-executor.js'; +import { KtxIngestEmbeddingPortAdapter } from './context/llm/embedding-port.js'; +import type { KtxEmbeddingPort } from './context/core/embedding.js'; +import type { KtxSemanticLayerComputePort } from './context/daemon/semantic-layer-compute.js'; +import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; +import { compileLocalSlQuery } from './context/sl/local-query.js'; +import { listLocalSlSources, readLocalSlSource, searchLocalSlSources as defaultSearchLocalSlSources, validateLocalSlSource, type LocalSlSourceSearchResult, type LocalSlSourceSummary } from './context/sl/local-sl.js'; +import type { SemanticLayerQueryInput } from './context/sl/types.js'; import { resolveProjectEmbeddingProvider, type EmbeddingProviderResolution, diff --git a/packages/cli/src/source-mapping.ts b/packages/cli/src/source-mapping.ts index 13fa15fb..5227ceab 100644 --- a/packages/cli/src/source-mapping.ts +++ b/packages/cli/src/source-mapping.ts @@ -1,26 +1,17 @@ -import { localConnectionToWarehouseDescriptor } from '@ktx/context/connections'; -import { - DEFAULT_METABASE_CLIENT_CONFIG, - DefaultLookerConnectionClientFactory, - DefaultMetabaseConnectionClientFactory, - KtxYamlMetabaseSourceStateReader, - LocalLookerRuntimeStore, - LocalMetabaseDiscoveryCache, - computeLookerMappingDrift, - computeMetabaseMappingDrift, - discoverLookerConnections, - discoverMetabaseDatabases, - lookerCredentialsFromLocalConnection, - metabaseRuntimeConfigFromLocalConnection, - planMetabaseFanoutChildren, - seedLocalMappingStateFromKtxYaml, - validateLookerMappings, - validateMappingPhysicalMatch, - type LookerMappingClient, - type LocalMetabaseMappingListRow, - type MetabaseRuntimeClient, -} from '@ktx/context/ingest'; -import { type KtxLocalProject, ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project'; +import { localConnectionToWarehouseDescriptor } from './context/connections/local-warehouse-descriptor.js'; +import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory } from './context/ingest/adapters/metabase/client.js'; +import { DefaultLookerConnectionClientFactory } from './context/ingest/adapters/looker/factory.js'; +import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache, type LocalMetabaseMappingListRow } from './context/ingest/adapters/metabase/local-source-state-store.js'; +import { LocalLookerRuntimeStore } from './context/ingest/adapters/looker/local-runtime-store.js'; +import { computeLookerMappingDrift, discoverLookerConnections, validateLookerMappings, type LookerMappingClient } from './context/ingest/adapters/looker/mapping.js'; +import { computeMetabaseMappingDrift, discoverMetabaseDatabases, validateMappingPhysicalMatch } from './context/ingest/adapters/metabase/mapping.js'; +import { lookerCredentialsFromLocalConnection } from './context/ingest/adapters/looker/local-looker.adapter.js'; +import { metabaseRuntimeConfigFromLocalConnection } from './context/ingest/adapters/metabase/local-metabase.adapter.js'; +import { planMetabaseFanoutChildren } from './context/ingest/adapters/metabase/fanout-planner.js'; +import { seedLocalMappingStateFromKtxYaml } from './context/ingest/local-mapping-reconcile.js'; +import type { MetabaseRuntimeClient } from './context/ingest/adapters/metabase/client-port.js'; +import { type KtxLocalProject, loadKtxProject } from './context/project/project.js'; +import { ktxLocalStateDbPath } from './context/project/local-state-db.js'; import type { KtxCliIo } from './cli-runtime.js'; import { profileMark } from './startup-profile.js'; diff --git a/packages/cli/src/sql.test.ts b/packages/cli/src/sql.test.ts index 7be128cc..5317b2a8 100644 --- a/packages/cli/src/sql.test.ts +++ b/packages/cli/src/sql.test.ts @@ -1,9 +1,10 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject, parseKtxProjectConfig, serializeKtxProjectConfig } from '@ktx/context/project'; -import type { KtxScanConnector } from '@ktx/context/scan'; -import type { SqlAnalysisPort } from '@ktx/context/sql-analysis'; +import { initKtxProject } from './context/project/project.js'; +import { parseKtxProjectConfig, serializeKtxProjectConfig } from './context/project/config.js'; +import type { KtxScanConnector } from './context/scan/types.js'; +import type { SqlAnalysisPort } from './context/sql-analysis/ports.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runKtxSql } from './sql.js'; diff --git a/packages/cli/src/sql.ts b/packages/cli/src/sql.ts index a04c7959..833df78e 100644 --- a/packages/cli/src/sql.ts +++ b/packages/cli/src/sql.ts @@ -1,6 +1,6 @@ -import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project'; -import type { KtxQueryResult, KtxScanConnector } from '@ktx/context/scan'; -import type { SqlAnalysisDialect, SqlAnalysisPort } from '@ktx/context/sql-analysis'; +import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; +import type { KtxQueryResult, KtxScanConnector } from './context/scan/types.js'; +import type { SqlAnalysisDialect, SqlAnalysisPort } from './context/sql-analysis/ports.js'; import type { KtxCliIo } from './cli-runtime.js'; import { type KtxOutputMode, resolveOutputMode } from './io/mode.js'; import { createKtxCliScanConnector } from './local-scan-connectors.js'; diff --git a/packages/cli/src/standalone-smoke.test.ts b/packages/cli/src/standalone-smoke.test.ts index 3bfc74f2..7e6ed56e 100644 --- a/packages/cli/src/standalone-smoke.test.ts +++ b/packages/cli/src/standalone-smoke.test.ts @@ -3,7 +3,7 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { promisify } from 'node:util'; -import { parseKtxProjectConfig } from '@ktx/context/project'; +import { parseKtxProjectConfig } from './context/project/config.js'; import Database from 'better-sqlite3'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; diff --git a/packages/cli/src/status-project.test.ts b/packages/cli/src/status-project.test.ts index 84f7b48f..9f8c879e 100644 --- a/packages/cli/src/status-project.test.ts +++ b/packages/cli/src/status-project.test.ts @@ -3,7 +3,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import Database from 'better-sqlite3'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { buildDefaultKtxProjectConfig, type KtxLocalProject, type KtxProjectConfig } from '@ktx/context/project'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; +import type { KtxLocalProject } from './context/project/project.js'; import { buildLocalStatsStatus, buildProjectStatus, @@ -267,9 +268,9 @@ describe('buildLocalStatsStatus', () => { embedding_json TEXT ); INSERT INTO knowledge_pages VALUES - ('a.md', 'a', 'GLOBAL', NULL, '', '', '[]', '', NULL), + ('a.md', 'a', 'GLOBAL', NULL, '', '', '[]', '', '[0.1,0.2]'), ('b.md', 'b', 'GLOBAL', NULL, '', '', '[]', '', NULL), - ('c.md', 'c', 'PROJECT', NULL, '', '', '[]', '', NULL); + ('c.md', 'c', 'PROJECT', NULL, '', '', '[]', '', '[0.3,0.4]'); CREATE TABLE local_sl_sources ( connection_id TEXT NOT NULL, @@ -281,7 +282,7 @@ describe('buildLocalStatsStatus', () => { PRIMARY KEY (connection_id, source_name) ); INSERT INTO local_sl_sources VALUES - ('analytics', 'orders', '', NULL, NULL, '2026-05-10T10:00:00Z'), + ('analytics', 'orders', '', '[0.1,0.2]', NULL, '2026-05-10T10:00:00Z'), ('analytics', 'users', '', NULL, NULL, '2026-05-10T10:00:00Z'); CREATE TABLE local_sl_dictionary_values ( @@ -307,18 +308,21 @@ describe('buildLocalStatsStatus', () => { { connectionId: 'analytics', adapter: 'live-database', lastCompletedAt: '2026-05-10T10:00:00Z' }, { connectionId: 'docs', adapter: 'notion', lastCompletedAt: '2026-05-01T10:00:00Z' }, ]); - expect(stats.knowledgePages).toEqual([ - { scope: 'GLOBAL', count: 2 }, - { scope: 'PROJECT', count: 1 }, + expect(stats.wikiPages).toEqual([ + { scope: 'GLOBAL', count: 2, embeddedCount: 1 }, + { scope: 'PROJECT', count: 1, embeddedCount: 1 }, ]); expect(stats.semanticLayer).toEqual([ - { connectionId: 'analytics', sourceCount: 2, dictionaryValueCount: 2 }, + { + connectionId: 'analytics', + sourceCount: 2, + embeddedSourceCount: 1, + dictionaryValueCount: 2, + }, ]); expect(stats.projectDir.dbSqliteBytes).toBeGreaterThan(0); expect(stats.projectDir.ktxCacheBytes).toBe(2048); expect(stats.projectDir.rawSources).toEqual({ fileCount: 2, bytes: 612 }); - expect(stats.projectDir.wikiGlobalMarkdownCount).toBe(2); - expect(stats.projectDir.semanticLayerYamlCount).toBe(2); }); it('tolerates a SQLite DB missing some tables', async () => { @@ -343,7 +347,7 @@ describe('buildLocalStatsStatus', () => { const stats = await buildLocalStatsStatus(projectIn(tempDir)); expect(stats.unavailable).toBeUndefined(); expect(stats.ingest.totalCompletedRuns).toBe(1); - expect(stats.knowledgePages).toEqual([]); + expect(stats.wikiPages).toEqual([]); expect(stats.semanticLayer).toEqual([]); }); }); @@ -359,31 +363,36 @@ describe('renderProjectStatus Local data', () => { { connectionId: 'analytics', adapter: 'live-database', lastCompletedAt: new Date(Date.now() - 60 * 60 * 1000).toISOString() }, ], }, - knowledgePages: [ - { scope: 'GLOBAL', count: 2 }, - { scope: 'PROJECT', count: 1 }, + wikiPages: [ + { scope: 'GLOBAL', count: 2, embeddedCount: 2 }, + { scope: 'PROJECT', count: 1, embeddedCount: 0 }, ], semanticLayer: [ - { connectionId: 'analytics', sourceCount: 12, dictionaryValueCount: 200 }, + { + connectionId: 'analytics', + sourceCount: 12, + embeddedSourceCount: 10, + dictionaryValueCount: 200, + }, ], projectDir: { dbSqliteBytes: 4096, ktxCacheBytes: 1_048_576, rawSources: { fileCount: 5, bytes: 200 }, - wikiGlobalMarkdownCount: 7, - semanticLayerYamlCount: 3, }, }; const rendered = renderProjectStatus(status, { useColor: false }); expect(rendered).toContain('Local data'); + expect(rendered).toContain('Wiki'); + expect(rendered).not.toContain('Knowledge'); expect(rendered).toContain('3 completed runs'); - expect(rendered).toContain('GLOBAL=2'); - expect(rendered).toContain('PROJECT=1'); - expect(rendered).toContain('12 sources · 200 dictionary values'); + expect(rendered).toContain('GLOBAL=2 (2 embedded)'); + expect(rendered).toContain('PROJECT=1 (0 embedded)'); + expect(rendered).toContain('12 sources (10 embedded) · 200 dictionary values'); expect(rendered).toContain('db=4.00 KiB'); expect(rendered).toContain('cache=1.00 MiB'); - expect(rendered).toContain('wiki=7 md'); - expect(rendered).toContain('semantic-layer=3 yaml'); + expect(rendered).not.toMatch(/wiki=\d+ md/); + expect(rendered).not.toMatch(/semantic-layer=\d+ yaml/); }); it('renders unavailable note when DB is missing', async () => { @@ -391,14 +400,12 @@ describe('renderProjectStatus Local data', () => { const status = await buildProjectStatus(project, { claudeCodeAuthProbe: stubClaudeCodeAuthProbe }); status.localStats = { ingest: { totalCompletedRuns: 0, perConnection: [] }, - knowledgePages: [], + wikiPages: [], semanticLayer: [], projectDir: { dbSqliteBytes: null, ktxCacheBytes: 0, rawSources: { fileCount: 0, bytes: 0 }, - wikiGlobalMarkdownCount: 0, - semanticLayerYamlCount: 0, }, unavailable: 'no .ktx/db.sqlite yet', }; diff --git a/packages/cli/src/status-project.ts b/packages/cli/src/status-project.ts index cf85cf13..9e257157 100644 --- a/packages/cli/src/status-project.ts +++ b/packages/cli/src/status-project.ts @@ -1,17 +1,10 @@ -import type { Dirent } from 'node:fs'; import { stat as statAsync, readdir as readdirAsync } from 'node:fs/promises'; import { basename, join } from 'node:path'; -import { runClaudeCodeAuthProbe } from '@ktx/context'; -import type { - KtxConfigIssue, - KtxLocalProject, - KtxProjectConfig, - KtxProjectConnectionConfig, - KtxProjectEmbeddingConfig, - KtxProjectLlmConfig, -} from '@ktx/context/project'; -import { ktxLocalStateDbPath } from '@ktx/context/project'; -import type { PostgresPgssProbeResult } from '@ktx/context/ingest'; +import { runClaudeCodeAuthProbe } from './context/llm/claude-code-runtime.js'; +import type { KtxConfigIssue, KtxProjectConfig, KtxProjectConnectionConfig, KtxProjectEmbeddingConfig, KtxProjectLlmConfig } from './context/project/config.js'; +import type { KtxLocalProject } from './context/project/project.js'; +import { ktxLocalStateDbPath } from './context/project/local-state-db.js'; +import type { PostgresPgssProbeResult } from './context/ingest/adapters/historic-sql/types.js'; import { formatClaudeCodePromptCachingFix, formatClaudeCodePromptCachingWarning, @@ -111,28 +104,29 @@ interface LocalStatsIngestPerConnection { interface LocalStatsSemanticLayerEntry { connectionId: string; sourceCount: number; + embeddedSourceCount: number; dictionaryValueCount: number; } -interface LocalStatsKnowledgeEntry { +interface LocalStatsWikiEntry { scope: string; count: number; + embeddedCount: number; } interface LocalStatsProjectDir { dbSqliteBytes: number | null; ktxCacheBytes: number; rawSources: { fileCount: number; bytes: number }; - wikiGlobalMarkdownCount: number; - semanticLayerYamlCount: number; } +/** @internal */ export interface LocalStatsStatus { ingest: { totalCompletedRuns: number; perConnection: LocalStatsIngestPerConnection[]; }; - knowledgePages: LocalStatsKnowledgeEntry[]; + wikiPages: LocalStatsWikiEntry[]; semanticLayer: LocalStatsSemanticLayerEntry[]; projectDir: LocalStatsProjectDir; unavailable?: string; @@ -470,8 +464,12 @@ function readinessDetail(result: PostgresPgssProbeResult): string { async function defaultPostgresQueryHistoryProbe( input: PostgresQueryHistoryProbeInput, ): Promise { - const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient, isKtxPostgresConnectionConfig }] = - await Promise.all([import('@ktx/context/ingest'), import('@ktx/connector-postgres')]); + const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient }, { isKtxPostgresConnectionConfig }] = + await Promise.all([ + import('./context/ingest/adapters/historic-sql/postgres-pgss-reader.js'), + import('./connectors/postgres/historic-sql-query-client.js'), + import('./connectors/postgres/connector.js'), + ]); const inputDriver = input.connection.driver ?? 'unknown'; if (!isKtxPostgresConnectionConfig(input.connection)) { @@ -775,16 +773,12 @@ interface DirSummary { bytes: number; } -async function summarizeDir( - dir: string, - filter?: (entry: Dirent, fullPath: string) => boolean, - maxDepth = 10, -): Promise { +async function summarizeDir(dir: string, maxDepth = 10): Promise { let fileCount = 0; let bytes = 0; const walk = async (current: string, depth: number): Promise => { if (depth > maxDepth) return; - let entries: Dirent[]; + let entries; try { entries = await readdirAsync(current, { withFileTypes: true }); } catch { @@ -797,7 +791,6 @@ async function summarizeDir( continue; } if (!entry.isFile()) continue; - if (filter && !filter(entry, full)) continue; try { const s = await statAsync(full); fileCount += 1; @@ -811,14 +804,6 @@ async function summarizeDir( return { fileCount, bytes }; } -function isMarkdownEntry(entry: Dirent): boolean { - return entry.isFile() && /\.mdx?$/i.test(entry.name); -} - -function isYamlEntry(entry: Dirent): boolean { - return entry.isFile() && /\.ya?ml$/i.test(entry.name); -} - async function fileSizeOrNull(filePath: string): Promise { try { const s = await statAsync(filePath); @@ -836,6 +821,7 @@ function tryQuery(run: () => T, fallback: T): T { } } +/** @internal */ export async function buildLocalStatsStatus(project: KtxLocalProject): Promise { const dbPath = ktxLocalStateDbPath(project); const dbSqliteBytes = await fileSizeOrNull(dbPath); @@ -844,18 +830,12 @@ export async function buildLocalStatsStatus(project: KtxLocalProject): Promise db .prepare( - `SELECT scope, COUNT(*) AS n FROM knowledge_pages GROUP BY scope ORDER BY scope`, + `SELECT scope, COUNT(*) AS n, SUM(CASE WHEN embedding_json IS NOT NULL THEN 1 ELSE 0 END) AS embedded + FROM knowledge_pages + GROUP BY scope + ORDER BY scope`, ) - .all() as Array<{ scope: string; n: number }>, - [] as Array<{ scope: string; n: number }>, + .all() as Array<{ scope: string; n: number; embedded: number | null }>, + [] as Array<{ scope: string; n: number; embedded: number | null }>, ); - const knowledgePages: LocalStatsKnowledgeEntry[] = knowledgeRows.map((row) => ({ + const wikiPages: LocalStatsWikiEntry[] = wikiRows.map((row) => ({ scope: row.scope, count: row.n, + embeddedCount: row.embedded ?? 0, })); const sourceRows = tryQuery( () => db .prepare( - `SELECT connection_id, COUNT(*) AS n FROM local_sl_sources GROUP BY connection_id`, + `SELECT connection_id, COUNT(*) AS n, SUM(CASE WHEN embedding_json IS NOT NULL THEN 1 ELSE 0 END) AS embedded + FROM local_sl_sources + GROUP BY connection_id`, ) - .all() as Array<{ connection_id: string; n: number }>, - [] as Array<{ connection_id: string; n: number }>, + .all() as Array<{ connection_id: string; n: number; embedded: number | null }>, + [] as Array<{ connection_id: string; n: number; embedded: number | null }>, ); const dictionaryRows = tryQuery( () => @@ -942,6 +928,7 @@ export async function buildLocalStatsStatus(project: KtxLocalProject): Promise `${entry.scope}=${entry.count}`) + const wikiText = stats.wikiPages + .map((entry) => `${entry.scope}=${entry.count} ${dim(`(${entry.embeddedCount} embedded)`)}`) .join(` ${dim('·')} `); - lines.push(` ${lLabel('Knowledge')} ${knowledgeText}`); + lines.push(` ${lLabel('Wiki')} ${wikiText}`); } if (stats.semanticLayer.length === 0) { @@ -1155,8 +1142,10 @@ function renderLocalStats( let firstLine = true; for (const entry of stats.semanticLayer) { const prefix = firstLine ? lLabel('Semantic layer') : ' '.repeat(localLabelWidth); + const sourcesText = `${entry.sourceCount} source${entry.sourceCount === 1 ? '' : 's'} (${entry.embeddedSourceCount} embedded)`; + const dictText = `${entry.dictionaryValueCount} dictionary value${entry.dictionaryValueCount === 1 ? '' : 's'}`; lines.push( - ` ${prefix} ${entry.connectionId.padEnd(nameWidth)} ${dim(`${entry.sourceCount} source${entry.sourceCount === 1 ? '' : 's'} · ${entry.dictionaryValueCount} dictionary value${entry.dictionaryValueCount === 1 ? '' : 's'}`)}`, + ` ${prefix} ${entry.connectionId.padEnd(nameWidth)} ${dim(`${sourcesText} · ${dictText}`)}`, ); firstLine = false; } @@ -1169,8 +1158,6 @@ function renderLocalStats( diskBits.push( `raw-sources=${disk.rawSources.fileCount} file${disk.rawSources.fileCount === 1 ? '' : 's'} (${formatBytes(disk.rawSources.bytes)})`, ); - diskBits.push(`wiki=${disk.wikiGlobalMarkdownCount} md`); - diskBits.push(`semantic-layer=${disk.semanticLayerYamlCount} yaml`); lines.push(` ${lLabel('Disk')} ${dim(diskBits.join(` ${dim('·')} `))}`); lines.push(''); } diff --git a/packages/context/test/fixtures/lookml/extends-chain/orders.model.lkml b/packages/cli/src/test/fixtures/lookml/extends-chain/orders.model.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/extends-chain/orders.model.lkml rename to packages/cli/src/test/fixtures/lookml/extends-chain/orders.model.lkml diff --git a/packages/context/test/fixtures/lookml/extends-chain/views/base.view.lkml b/packages/cli/src/test/fixtures/lookml/extends-chain/views/base.view.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/extends-chain/views/base.view.lkml rename to packages/cli/src/test/fixtures/lookml/extends-chain/views/base.view.lkml diff --git a/packages/context/test/fixtures/lookml/extends-chain/views/orders.view.lkml b/packages/cli/src/test/fixtures/lookml/extends-chain/views/orders.view.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/extends-chain/views/orders.view.lkml rename to packages/cli/src/test/fixtures/lookml/extends-chain/views/orders.view.lkml diff --git a/packages/context/test/fixtures/lookml/extends-chain/views/orders_ext.view.lkml b/packages/cli/src/test/fixtures/lookml/extends-chain/views/orders_ext.view.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/extends-chain/views/orders_ext.view.lkml rename to packages/cli/src/test/fixtures/lookml/extends-chain/views/orders_ext.view.lkml diff --git a/packages/context/test/fixtures/lookml/multi-model/marketing.model.lkml b/packages/cli/src/test/fixtures/lookml/multi-model/marketing.model.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/multi-model/marketing.model.lkml rename to packages/cli/src/test/fixtures/lookml/multi-model/marketing.model.lkml diff --git a/packages/context/test/fixtures/lookml/multi-model/orders.model.lkml b/packages/cli/src/test/fixtures/lookml/multi-model/orders.model.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/multi-model/orders.model.lkml rename to packages/cli/src/test/fixtures/lookml/multi-model/orders.model.lkml diff --git a/packages/context/test/fixtures/lookml/multi-model/views/campaigns.view.lkml b/packages/cli/src/test/fixtures/lookml/multi-model/views/campaigns.view.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/multi-model/views/campaigns.view.lkml rename to packages/cli/src/test/fixtures/lookml/multi-model/views/campaigns.view.lkml diff --git a/packages/context/test/fixtures/lookml/multi-model/views/orders.view.lkml b/packages/cli/src/test/fixtures/lookml/multi-model/views/orders.view.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/multi-model/views/orders.view.lkml rename to packages/cli/src/test/fixtures/lookml/multi-model/views/orders.view.lkml diff --git a/packages/context/test/fixtures/lookml/multi-model/views/shared_dims.view.lkml b/packages/cli/src/test/fixtures/lookml/multi-model/views/shared_dims.view.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/multi-model/views/shared_dims.view.lkml rename to packages/cli/src/test/fixtures/lookml/multi-model/views/shared_dims.view.lkml diff --git a/packages/context/test/fixtures/lookml/single-model/orders.model.lkml b/packages/cli/src/test/fixtures/lookml/single-model/orders.model.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/single-model/orders.model.lkml rename to packages/cli/src/test/fixtures/lookml/single-model/orders.model.lkml diff --git a/packages/context/test/fixtures/lookml/single-model/views/customers.view.lkml b/packages/cli/src/test/fixtures/lookml/single-model/views/customers.view.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/single-model/views/customers.view.lkml rename to packages/cli/src/test/fixtures/lookml/single-model/views/customers.view.lkml diff --git a/packages/context/test/fixtures/lookml/single-model/views/orders.view.lkml b/packages/cli/src/test/fixtures/lookml/single-model/views/orders.view.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/single-model/views/orders.view.lkml rename to packages/cli/src/test/fixtures/lookml/single-model/views/orders.view.lkml diff --git a/packages/context/test/fixtures/lookml/three-churn/billing.model.lkml b/packages/cli/src/test/fixtures/lookml/three-churn/billing.model.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/three-churn/billing.model.lkml rename to packages/cli/src/test/fixtures/lookml/three-churn/billing.model.lkml diff --git a/packages/context/test/fixtures/lookml/three-churn/customers.model.lkml b/packages/cli/src/test/fixtures/lookml/three-churn/customers.model.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/three-churn/customers.model.lkml rename to packages/cli/src/test/fixtures/lookml/three-churn/customers.model.lkml diff --git a/packages/context/test/fixtures/lookml/three-churn/support.model.lkml b/packages/cli/src/test/fixtures/lookml/three-churn/support.model.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/three-churn/support.model.lkml rename to packages/cli/src/test/fixtures/lookml/three-churn/support.model.lkml diff --git a/packages/context/test/fixtures/lookml/three-churn/views/billing/billing_churn_risk.view.lkml b/packages/cli/src/test/fixtures/lookml/three-churn/views/billing/billing_churn_risk.view.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/three-churn/views/billing/billing_churn_risk.view.lkml rename to packages/cli/src/test/fixtures/lookml/three-churn/views/billing/billing_churn_risk.view.lkml diff --git a/packages/context/test/fixtures/lookml/three-churn/views/customers/customer_churn_risk.view.lkml b/packages/cli/src/test/fixtures/lookml/three-churn/views/customers/customer_churn_risk.view.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/three-churn/views/customers/customer_churn_risk.view.lkml rename to packages/cli/src/test/fixtures/lookml/three-churn/views/customers/customer_churn_risk.view.lkml diff --git a/packages/context/test/fixtures/lookml/three-churn/views/support/support_churn_risk.view.lkml b/packages/cli/src/test/fixtures/lookml/three-churn/views/support/support_churn_risk.view.lkml similarity index 100% rename from packages/context/test/fixtures/lookml/three-churn/views/support/support_churn_risk.view.lkml rename to packages/cli/src/test/fixtures/lookml/three-churn/views/support/support_churn_risk.view.lkml diff --git a/packages/context/test/fixtures/metabase/card-ref/cards/10.json b/packages/cli/src/test/fixtures/metabase/card-ref/cards/10.json similarity index 100% rename from packages/context/test/fixtures/metabase/card-ref/cards/10.json rename to packages/cli/src/test/fixtures/metabase/card-ref/cards/10.json diff --git a/packages/context/test/fixtures/metabase/card-ref/cards/11.json b/packages/cli/src/test/fixtures/metabase/card-ref/cards/11.json similarity index 100% rename from packages/context/test/fixtures/metabase/card-ref/cards/11.json rename to packages/cli/src/test/fixtures/metabase/card-ref/cards/11.json diff --git a/packages/context/test/fixtures/metabase/card-ref/collections/5.json b/packages/cli/src/test/fixtures/metabase/card-ref/collections/5.json similarity index 100% rename from packages/context/test/fixtures/metabase/card-ref/collections/5.json rename to packages/cli/src/test/fixtures/metabase/card-ref/collections/5.json diff --git a/packages/context/test/fixtures/metabase/card-ref/databases/42.json b/packages/cli/src/test/fixtures/metabase/card-ref/databases/42.json similarity index 100% rename from packages/context/test/fixtures/metabase/card-ref/databases/42.json rename to packages/cli/src/test/fixtures/metabase/card-ref/databases/42.json diff --git a/packages/context/test/fixtures/metabase/card-ref/sync-config.json b/packages/cli/src/test/fixtures/metabase/card-ref/sync-config.json similarity index 100% rename from packages/context/test/fixtures/metabase/card-ref/sync-config.json rename to packages/cli/src/test/fixtures/metabase/card-ref/sync-config.json diff --git a/packages/context/test/fixtures/metabase/multi-collection/cards/1.json b/packages/cli/src/test/fixtures/metabase/multi-collection/cards/1.json similarity index 100% rename from packages/context/test/fixtures/metabase/multi-collection/cards/1.json rename to packages/cli/src/test/fixtures/metabase/multi-collection/cards/1.json diff --git a/packages/context/test/fixtures/metabase/multi-collection/cards/2.json b/packages/cli/src/test/fixtures/metabase/multi-collection/cards/2.json similarity index 100% rename from packages/context/test/fixtures/metabase/multi-collection/cards/2.json rename to packages/cli/src/test/fixtures/metabase/multi-collection/cards/2.json diff --git a/packages/context/test/fixtures/metabase/multi-collection/cards/3.json b/packages/cli/src/test/fixtures/metabase/multi-collection/cards/3.json similarity index 100% rename from packages/context/test/fixtures/metabase/multi-collection/cards/3.json rename to packages/cli/src/test/fixtures/metabase/multi-collection/cards/3.json diff --git a/packages/context/test/fixtures/metabase/multi-collection/collections/5.json b/packages/cli/src/test/fixtures/metabase/multi-collection/collections/5.json similarity index 100% rename from packages/context/test/fixtures/metabase/multi-collection/collections/5.json rename to packages/cli/src/test/fixtures/metabase/multi-collection/collections/5.json diff --git a/packages/context/test/fixtures/metabase/multi-collection/collections/6.json b/packages/cli/src/test/fixtures/metabase/multi-collection/collections/6.json similarity index 100% rename from packages/context/test/fixtures/metabase/multi-collection/collections/6.json rename to packages/cli/src/test/fixtures/metabase/multi-collection/collections/6.json diff --git a/packages/context/test/fixtures/metabase/multi-collection/databases/42.json b/packages/cli/src/test/fixtures/metabase/multi-collection/databases/42.json similarity index 100% rename from packages/context/test/fixtures/metabase/multi-collection/databases/42.json rename to packages/cli/src/test/fixtures/metabase/multi-collection/databases/42.json diff --git a/packages/context/test/fixtures/metabase/multi-collection/sync-config.json b/packages/cli/src/test/fixtures/metabase/multi-collection/sync-config.json similarity index 100% rename from packages/context/test/fixtures/metabase/multi-collection/sync-config.json rename to packages/cli/src/test/fixtures/metabase/multi-collection/sync-config.json diff --git a/packages/context/test/fixtures/metabase/simple/cards/1.json b/packages/cli/src/test/fixtures/metabase/simple/cards/1.json similarity index 100% rename from packages/context/test/fixtures/metabase/simple/cards/1.json rename to packages/cli/src/test/fixtures/metabase/simple/cards/1.json diff --git a/packages/context/test/fixtures/metabase/simple/cards/2.json b/packages/cli/src/test/fixtures/metabase/simple/cards/2.json similarity index 100% rename from packages/context/test/fixtures/metabase/simple/cards/2.json rename to packages/cli/src/test/fixtures/metabase/simple/cards/2.json diff --git a/packages/context/test/fixtures/metabase/simple/collections/5.json b/packages/cli/src/test/fixtures/metabase/simple/collections/5.json similarity index 100% rename from packages/context/test/fixtures/metabase/simple/collections/5.json rename to packages/cli/src/test/fixtures/metabase/simple/collections/5.json diff --git a/packages/context/test/fixtures/metabase/simple/databases/42.json b/packages/cli/src/test/fixtures/metabase/simple/databases/42.json similarity index 100% rename from packages/context/test/fixtures/metabase/simple/databases/42.json rename to packages/cli/src/test/fixtures/metabase/simple/databases/42.json diff --git a/packages/context/test/fixtures/metabase/simple/sync-config.json b/packages/cli/src/test/fixtures/metabase/simple/sync-config.json similarity index 100% rename from packages/context/test/fixtures/metabase/simple/sync-config.json rename to packages/cli/src/test/fixtures/metabase/simple/sync-config.json diff --git a/packages/context/test/fixtures/metricflow/dbt-mixed/dbt_project.yml b/packages/cli/src/test/fixtures/metricflow/dbt-mixed/dbt_project.yml similarity index 100% rename from packages/context/test/fixtures/metricflow/dbt-mixed/dbt_project.yml rename to packages/cli/src/test/fixtures/metricflow/dbt-mixed/dbt_project.yml diff --git a/packages/context/test/fixtures/metricflow/dbt-mixed/models/orders.yml b/packages/cli/src/test/fixtures/metricflow/dbt-mixed/models/orders.yml similarity index 100% rename from packages/context/test/fixtures/metricflow/dbt-mixed/models/orders.yml rename to packages/cli/src/test/fixtures/metricflow/dbt-mixed/models/orders.yml diff --git a/packages/context/test/fixtures/metricflow/extends-chain/metrics/orders_final.yml b/packages/cli/src/test/fixtures/metricflow/extends-chain/metrics/orders_final.yml similarity index 100% rename from packages/context/test/fixtures/metricflow/extends-chain/metrics/orders_final.yml rename to packages/cli/src/test/fixtures/metricflow/extends-chain/metrics/orders_final.yml diff --git a/packages/context/test/fixtures/metricflow/extends-chain/models/orders.yml b/packages/cli/src/test/fixtures/metricflow/extends-chain/models/orders.yml similarity index 100% rename from packages/context/test/fixtures/metricflow/extends-chain/models/orders.yml rename to packages/cli/src/test/fixtures/metricflow/extends-chain/models/orders.yml diff --git a/packages/context/test/fixtures/metricflow/extends-chain/models/orders_ext.yml b/packages/cli/src/test/fixtures/metricflow/extends-chain/models/orders_ext.yml similarity index 100% rename from packages/context/test/fixtures/metricflow/extends-chain/models/orders_ext.yml rename to packages/cli/src/test/fixtures/metricflow/extends-chain/models/orders_ext.yml diff --git a/packages/context/test/fixtures/metricflow/multi-component/models/marketing/campaigns.yml b/packages/cli/src/test/fixtures/metricflow/multi-component/models/marketing/campaigns.yml similarity index 100% rename from packages/context/test/fixtures/metricflow/multi-component/models/marketing/campaigns.yml rename to packages/cli/src/test/fixtures/metricflow/multi-component/models/marketing/campaigns.yml diff --git a/packages/context/test/fixtures/metricflow/multi-component/models/sales/orders.yml b/packages/cli/src/test/fixtures/metricflow/multi-component/models/sales/orders.yml similarity index 100% rename from packages/context/test/fixtures/metricflow/multi-component/models/sales/orders.yml rename to packages/cli/src/test/fixtures/metricflow/multi-component/models/sales/orders.yml diff --git a/packages/context/test/fixtures/metricflow/single-model/models/orders.yml b/packages/cli/src/test/fixtures/metricflow/single-model/models/orders.yml similarity index 100% rename from packages/context/test/fixtures/metricflow/single-model/models/orders.yml rename to packages/cli/src/test/fixtures/metricflow/single-model/models/orders.yml diff --git a/packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/data.sqlite b/packages/cli/src/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/data.sqlite similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/data.sqlite rename to packages/cli/src/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/data.sqlite diff --git a/packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/snapshot.json b/packages/cli/src/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/snapshot.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/snapshot.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/snapshot.json diff --git a/packages/context/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/snapshot.json b/packages/cli/src/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/snapshot.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/snapshot.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/adventureworks_oltp_with_declared_metadata/snapshot.json diff --git a/packages/context/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/snapshot.json b/packages/cli/src/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/snapshot.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/snapshot.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/adventureworkslt_with_declared_metadata/snapshot.json diff --git a/packages/context/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/data.sqlite b/packages/cli/src/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/data.sqlite similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/data.sqlite rename to packages/cli/src/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/data.sqlite diff --git a/packages/context/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/snapshot.json b/packages/cli/src/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/snapshot.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/snapshot.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/analytical_warehouse_no_naming_convention/snapshot.json diff --git a/packages/context/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/snapshot.json b/packages/cli/src/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/snapshot.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/snapshot.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/chinook_with_declared_metadata/snapshot.json diff --git a/packages/context/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/data.sqlite b/packages/cli/src/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/data.sqlite similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/data.sqlite rename to packages/cli/src/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/data.sqlite diff --git a/packages/context/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/snapshot.json b/packages/cli/src/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/snapshot.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/snapshot.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/composite_keys_no_declared_constraints/snapshot.json diff --git a/packages/context/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/data.sqlite b/packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/data.sqlite similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/data.sqlite rename to packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/data.sqlite diff --git a/packages/context/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/snapshot.json b/packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/snapshot.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/snapshot.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_declared_metadata/snapshot.json diff --git a/packages/context/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/data.sqlite b/packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/data.sqlite similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/data.sqlite rename to packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/data.sqlite diff --git a/packages/context/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/snapshot.json b/packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/snapshot.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/snapshot.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/demo_b2b_no_declared_constraints/snapshot.json diff --git a/packages/context/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/data.sqlite b/packages/cli/src/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/data.sqlite similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/data.sqlite rename to packages/cli/src/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/data.sqlite diff --git a/packages/context/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/snapshot.json b/packages/cli/src/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/snapshot.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/snapshot.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/mixed_case_within_schema_no_declared_constraints/snapshot.json diff --git a/packages/context/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/data.sqlite b/packages/cli/src/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/data.sqlite similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/data.sqlite rename to packages/cli/src/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/data.sqlite diff --git a/packages/context/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/snapshot.json b/packages/cli/src/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/snapshot.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/snapshot.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/natural_keys_no_declared_constraints/snapshot.json diff --git a/packages/context/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/data.sqlite b/packages/cli/src/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/data.sqlite similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/data.sqlite rename to packages/cli/src/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/data.sqlite diff --git a/packages/context/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/snapshot.json b/packages/cli/src/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/snapshot.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/snapshot.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/non_english_naming_no_declared_constraints/snapshot.json diff --git a/packages/context/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/snapshot.json b/packages/cli/src/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/snapshot.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/snapshot.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/northwind_with_declared_metadata/snapshot.json diff --git a/packages/context/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/data.sqlite b/packages/cli/src/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/data.sqlite similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/data.sqlite rename to packages/cli/src/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/data.sqlite diff --git a/packages/context/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/snapshot.json b/packages/cli/src/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/snapshot.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/snapshot.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/snapshot.json diff --git a/packages/context/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/data.sqlite b/packages/cli/src/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/data.sqlite similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/data.sqlite rename to packages/cli/src/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/data.sqlite diff --git a/packages/context/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/snapshot.json b/packages/cli/src/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/snapshot.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/snapshot.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/plan_code_no_declared_constraints/snapshot.json diff --git a/packages/context/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/data.sqlite b/packages/cli/src/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/data.sqlite similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/data.sqlite rename to packages/cli/src/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/data.sqlite diff --git a/packages/context/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/snapshot.json b/packages/cli/src/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/snapshot.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/snapshot.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/polymorphic_partial_overlap_no_declared_constraints/snapshot.json diff --git a/packages/context/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/snapshot.json b/packages/cli/src/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/snapshot.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/snapshot.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/sakila_with_declared_metadata/snapshot.json diff --git a/packages/context/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/data.sqlite.gz b/packages/cli/src/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/data.sqlite.gz similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/data.sqlite.gz rename to packages/cli/src/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/data.sqlite.gz diff --git a/packages/context/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/snapshot.json.gz b/packages/cli/src/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/snapshot.json.gz similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/snapshot.json.gz rename to packages/cli/src/test/fixtures/relationship-benchmarks/scale_stress_no_declared_constraints/snapshot.json.gz diff --git a/packages/context/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/column-embeddings.json b/packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/column-embeddings.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/column-embeddings.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/column-embeddings.json diff --git a/packages/context/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/data.sqlite b/packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/data.sqlite similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/data.sqlite rename to packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/data.sqlite diff --git a/packages/context/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/expected-links.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/expected-links.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/fixture.yaml b/packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/fixture.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/fixture.yaml rename to packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/fixture.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/snapshot.json b/packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/snapshot.json similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/snapshot.json rename to packages/cli/src/test/fixtures/relationship-benchmarks/semantic_embedding_aliases_no_declared_constraints/snapshot.json diff --git a/packages/cli/src/text-ingest.test.ts b/packages/cli/src/text-ingest.test.ts index 04ff2e40..b7737f36 100644 --- a/packages/cli/src/text-ingest.test.ts +++ b/packages/cli/src/text-ingest.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import type { MemoryIngestStatus } from '@ktx/context/memory'; -import type { KtxLocalProject } from '@ktx/context/project'; +import type { MemoryIngestStatus } from './context/memory/memory-runs.js'; +import type { KtxLocalProject } from './context/project/project.js'; import { runKtxTextIngest, type TextMemoryIngestPort } from './text-ingest.js'; function makeIo(options: { isTTY?: boolean } = {}) { diff --git a/packages/cli/src/text-ingest.ts b/packages/cli/src/text-ingest.ts index fe15244e..388e58d6 100644 --- a/packages/cli/src/text-ingest.ts +++ b/packages/cli/src/text-ingest.ts @@ -1,7 +1,9 @@ import { readFile as fsReadFile } from 'node:fs/promises'; import { basename, resolve } from 'node:path'; -import { createLocalProjectMemoryIngest, type MemoryAgentInput, type MemoryIngestStatus } from '@ktx/context/memory'; -import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project'; +import { createLocalProjectMemoryIngest } from './context/memory/local-memory.js'; +import type { MemoryAgentInput } from './context/memory/types.js'; +import type { MemoryIngestStatus } from './context/memory/memory-runs.js'; +import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; import type { KtxCliIo } from './cli-runtime.js'; import { createRepainter, initViewState, renderContextBuildView, type ContextBuildTargetState } from './context-build-view.js'; import { formatDuration } from './demo-metrics.js'; @@ -17,6 +19,7 @@ export interface KtxTextIngestArgs { failFast: boolean; } +/** @internal */ export interface TextMemoryIngestPort { ingest(input: MemoryAgentInput): Promise<{ runId: string }>; waitForRun(runId: string): Promise; diff --git a/packages/cli/src/tree-picker-state.ts b/packages/cli/src/tree-picker-state.ts index 3f9d13bf..3e96e096 100644 --- a/packages/cli/src/tree-picker-state.ts +++ b/packages/cli/src/tree-picker-state.ts @@ -93,6 +93,7 @@ function transientHint(text: string, now: number): PickerState['transientHint'] return { text, expiresAt: now + TRANSIENT_HINT_DURATION_MS }; } +/** @internal */ export function clearExpiredTransientHint(state: PickerState, now = Date.now()): PickerState { if (!state.transientHint || state.transientHint.expiresAt > now) { return state; @@ -249,6 +250,7 @@ function checkedAncestor(nodeId: string, state: PickerState): TreePickerNode | n return null; } +/** @internal */ export function canToggle(nodeId: string, state: PickerState): { ok: true } | { ok: false; reason: string } { if (!state.byId.has(nodeId)) { return { ok: false, reason: 'Node not found' }; @@ -260,6 +262,7 @@ export function canToggle(nodeId: string, state: PickerState): { ok: true } | { return { ok: true }; } +/** @internal */ export function toggleChecked(state: PickerState, nodeId: string, now = Date.now()): PickerState { const toggle = canToggle(nodeId, state); if (!toggle.ok) { @@ -335,6 +338,7 @@ export function visibleNodeIds(state: PickerState): string[] { return result; } +/** @internal */ export function selectAllVisible(state: PickerState): PickerState { const candidates = state.search.query.trim().length > 0 ? matchingIds(state) : new Set(visibleNodeIds(state)); const checked = new Set(state.checked); @@ -358,6 +362,7 @@ export function selectAllVisible(state: PickerState): PickerState { }); } +/** @internal */ export function selectNone(state: PickerState): PickerState { return cloneState(state, { checked: new Set(), transientHint: null }); } @@ -383,6 +388,7 @@ function setExpanded(state: PickerState, nodeId: string, value: boolean | 'toggl return cloneState(state, { expanded }); } +/** @internal */ export function moveCursor(state: PickerState, dir: 'up' | 'down' | 'left' | 'right'): PickerState { const node = state.byId.get(state.cursorId); if (!node) { diff --git a/packages/cli/src/tree-picker-tui.tsx b/packages/cli/src/tree-picker-tui.tsx index a2c4a9ab..57525270 100644 --- a/packages/cli/src/tree-picker-tui.tsx +++ b/packages/cli/src/tree-picker-tui.tsx @@ -77,12 +77,14 @@ interface TreePickerAppProps extends TreePickerRenderInput { onExit(result: TreePickerResult): void; } +/** @internal */ export interface TreePickerInkInstance { rerender(tree: ReactNode): void; unmount(): void; waitUntilExit(): Promise; } +/** @internal */ export interface TreePickerInkRenderOptions { stdin?: TreePickerTuiIo['stdin']; stdout: TreePickerTuiIo['stdout']; @@ -97,6 +99,7 @@ function resolveTheme(env: NodeJS.ProcessEnv = process.env): TreePickerTheme { return env.NO_COLOR || env.TERM === 'dumb' ? NO_COLOR_THEME : COLOR_THEME; } +/** @internal */ export function resolveTreePickerWidth(columns: number | undefined): number { const resolvedColumns = columns ?? 100; return Math.max(60, Math.min(120, resolvedColumns - 4)); @@ -114,6 +117,7 @@ function rowMatchesSearch(state: PickerState, nodeId: string): boolean { return node.title.toLocaleLowerCase().includes(query) || node.path.toLocaleLowerCase().includes(query); } +/** @internal */ export function sanitizeTreePickerTuiError(error: unknown): string { const message = error instanceof Error ? error.message : String(error); return message @@ -121,11 +125,13 @@ export function sanitizeTreePickerTuiError(error: unknown): string { .replace(/\b(api[_-]?key|password|token|secret)=\S+/gi, '[redacted]'); } +/** @internal */ export function windowOffset(count: number, selected: number, visible: number): number { if (count <= visible) return 0; return Math.max(0, Math.min(count - visible, selected - Math.floor(visible / 2))); } +/** @internal */ export function windowItems(items: T[], selected: number, visible: number): { items: T[]; offset: number } { const offset = windowOffset(items.length, selected, visible); return { items: items.slice(offset, offset + visible), offset }; @@ -137,6 +143,7 @@ function truncateText(value: string, width: number): string { return `${value.slice(0, width - 3)}...`; } +/** @internal */ export function treePickerCommandForInkInput( input: string, key: InkKey, @@ -205,6 +212,7 @@ function PickerRow(props: { state: PickerState; nodeId: string; width: number; t ); } +/** @internal */ export function TreePickerApp(props: TreePickerAppProps): ReactNode { const app = useApp(); const [state, setState] = useState(props.initialState); diff --git a/packages/cli/src/viz-fallback.ts b/packages/cli/src/viz-fallback.ts index 140d700f..c0f9c281 100644 --- a/packages/cli/src/viz-fallback.ts +++ b/packages/cli/src/viz-fallback.ts @@ -88,6 +88,7 @@ export function warnVizFallbackOnce(io: KtxVizFallbackIo, decision: KtxVizFallba io.stderr.write(`Visualization requested but ${decision.message}; printing plain output.\n`); } +/** @internal */ export function resetVizFallbackWarningsForTest(): void { warnedFallbackReasons.clear(); } diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json index b527c8a1..6590987d 100644 --- a/packages/cli/tsconfig.json +++ b/packages/cli/tsconfig.json @@ -4,7 +4,7 @@ "outDir": "./dist", "rootDir": "./src", "jsx": "react-jsx", - "tsBuildInfoFile": "./node_modules/.cache/tsc.tsbuildinfo" + "tsBuildInfoFile": "./dist/.tsbuildinfo" }, "include": ["src/**/*.ts", "src/**/*.tsx"], "exclude": ["dist", "node_modules"] diff --git a/packages/connector-bigquery/package.json b/packages/connector-bigquery/package.json deleted file mode 100644 index 591e5193..00000000 --- a/packages/connector-bigquery/package.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "name": "@ktx/connector-bigquery", - "version": "0.0.0-private", - "description": "BigQuery connector package for KTX scan interfaces", - "private": true, - "type": "module", - "engines": { - "node": ">=22.0.0" - }, - "main": "dist/index.js", - "types": "dist/index.d.ts", - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - }, - "./package.json": "./package.json" - }, - "files": [ - "dist" - ], - "scripts": { - "build": "tsc -p tsconfig.json", - "test": "vitest run", - "type-check": "tsc -p tsconfig.json --noEmit" - }, - "dependencies": { - "@google-cloud/bigquery": "^8.3.1", - "@ktx/context": "workspace:*" - }, - "devDependencies": { - "@types/node": "^25.7.0", - "@vitest/coverage-v8": "^4.1.6", - "typescript": "^6.0.3", - "vitest": "^4.1.6" - }, - "license": "Apache-2.0", - "repository": { - "type": "git", - "url": "git+https://github.com/kaelio/ktx.git", - "directory": "packages/connector-bigquery" - }, - "bugs": { - "url": "https://github.com/kaelio/ktx/issues" - }, - "homepage": "https://github.com/kaelio/ktx#readme" -} diff --git a/packages/connector-bigquery/src/index.ts b/packages/connector-bigquery/src/index.ts deleted file mode 100644 index e4003530..00000000 --- a/packages/connector-bigquery/src/index.ts +++ /dev/null @@ -1,18 +0,0 @@ -export { KtxBigQueryDialect } from './dialect.js'; -export { - bigQueryConnectionConfigFromConfig, - isKtxBigQueryConnectionConfig, - KtxBigQueryScanConnector, - type KtxBigQueryClient, - type KtxBigQueryClientFactory, - type KtxBigQueryColumnDistinctValuesOptions, - type KtxBigQueryColumnDistinctValuesResult, - type KtxBigQueryConnectionConfig, - type KtxBigQueryDataset, - type KtxBigQueryQueryJob, - type KtxBigQueryReadOnlyQueryInput, - type KtxBigQueryResolvedConnectionConfig, - type KtxBigQueryScanConnectorOptions, - type KtxBigQueryTableRef, -} from './connector.js'; -export { createBigQueryLiveDatabaseIntrospection } from './live-database-introspection.js'; diff --git a/packages/connector-bigquery/src/package-exports.test.ts b/packages/connector-bigquery/src/package-exports.test.ts deleted file mode 100644 index 71508c5a..00000000 --- a/packages/connector-bigquery/src/package-exports.test.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import * as connector from './index.js'; - -describe('@ktx/connector-bigquery exports', () => { - it('exports public connector, dialect, and introspection APIs', () => { - expect(connector.KtxBigQueryDialect).toBeTypeOf('function'); - expect(connector.KtxBigQueryScanConnector).toBeTypeOf('function'); - expect(connector.bigQueryConnectionConfigFromConfig).toBeTypeOf('function'); - expect(connector.createBigQueryLiveDatabaseIntrospection).toBeTypeOf('function'); - }); -}); diff --git a/packages/connector-bigquery/tsconfig.json b/packages/connector-bigquery/tsconfig.json deleted file mode 100644 index 965e6978..00000000 --- a/packages/connector-bigquery/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*.ts"], - "exclude": ["dist", "node_modules"] -} diff --git a/packages/connector-clickhouse/package.json b/packages/connector-clickhouse/package.json deleted file mode 100644 index 568ed6b2..00000000 --- a/packages/connector-clickhouse/package.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "name": "@ktx/connector-clickhouse", - "version": "0.0.0-private", - "description": "ClickHouse connector package for KTX scan interfaces", - "private": true, - "type": "module", - "engines": { - "node": ">=22.0.0" - }, - "main": "dist/index.js", - "types": "dist/index.d.ts", - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - }, - "./package.json": "./package.json" - }, - "files": [ - "dist" - ], - "scripts": { - "build": "tsc -p tsconfig.json", - "test": "vitest run", - "type-check": "tsc -p tsconfig.json --noEmit" - }, - "dependencies": { - "@clickhouse/client": "^1.18.4", - "@ktx/context": "workspace:*" - }, - "devDependencies": { - "@types/node": "^25.7.0", - "@vitest/coverage-v8": "^4.1.6", - "typescript": "^6.0.3", - "vitest": "^4.1.6" - }, - "license": "Apache-2.0", - "repository": { - "type": "git", - "url": "git+https://github.com/kaelio/ktx.git", - "directory": "packages/connector-clickhouse" - }, - "bugs": { - "url": "https://github.com/kaelio/ktx/issues" - }, - "homepage": "https://github.com/kaelio/ktx#readme" -} diff --git a/packages/connector-clickhouse/src/index.ts b/packages/connector-clickhouse/src/index.ts deleted file mode 100644 index 106df237..00000000 --- a/packages/connector-clickhouse/src/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -export { KtxClickHouseDialect } from './dialect.js'; -export { - clickHouseClientConfigFromConfig, - isKtxClickHouseConnectionConfig, - KtxClickHouseScanConnector, - type KtxClickHouseClient, - type KtxClickHouseClientFactory, - type KtxClickHouseColumnDistinctValuesOptions, - type KtxClickHouseColumnDistinctValuesResult, - type KtxClickHouseConnectionConfig, - type KtxClickHouseEndpointResolver, - type KtxClickHouseReadOnlyQueryInput, - type KtxClickHouseResolvedClientConfig, - type KtxClickHouseScanConnectorOptions, -} from './connector.js'; -export { createClickHouseLiveDatabaseIntrospection } from './live-database-introspection.js'; diff --git a/packages/connector-clickhouse/src/package-exports.test.ts b/packages/connector-clickhouse/src/package-exports.test.ts deleted file mode 100644 index 644e6075..00000000 --- a/packages/connector-clickhouse/src/package-exports.test.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -describe('@ktx/connector-clickhouse package exports', () => { - it( - 'exports public connector APIs during package bootstrap', - async () => { - const connector = await import('./index.js'); - - expect(connector.KtxClickHouseDialect).toBeTypeOf('function'); - expect(connector.KtxClickHouseScanConnector).toBeTypeOf('function'); - expect(connector.clickHouseClientConfigFromConfig).toBeTypeOf('function'); - expect(connector.createClickHouseLiveDatabaseIntrospection).toBeTypeOf('function'); - }, - 20_000, - ); -}); diff --git a/packages/connector-clickhouse/tsconfig.json b/packages/connector-clickhouse/tsconfig.json deleted file mode 100644 index 965e6978..00000000 --- a/packages/connector-clickhouse/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*.ts"], - "exclude": ["dist", "node_modules"] -} diff --git a/packages/connector-mysql/package.json b/packages/connector-mysql/package.json deleted file mode 100644 index 16bbc782..00000000 --- a/packages/connector-mysql/package.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "name": "@ktx/connector-mysql", - "version": "0.0.0-private", - "description": "MySQL connector package for KTX scan interfaces", - "private": true, - "type": "module", - "engines": { - "node": ">=22.0.0" - }, - "main": "dist/index.js", - "types": "dist/index.d.ts", - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - }, - "./package.json": "./package.json" - }, - "files": [ - "dist" - ], - "scripts": { - "build": "tsc -p tsconfig.json", - "test": "vitest run", - "type-check": "tsc -p tsconfig.json --noEmit" - }, - "dependencies": { - "@ktx/context": "workspace:*", - "mysql2": "^3.22.3" - }, - "devDependencies": { - "@types/node": "^25.7.0", - "@vitest/coverage-v8": "^4.1.6", - "typescript": "^6.0.3", - "vitest": "^4.1.6" - }, - "license": "Apache-2.0", - "repository": { - "type": "git", - "url": "git+https://github.com/kaelio/ktx.git", - "directory": "packages/connector-mysql" - }, - "bugs": { - "url": "https://github.com/kaelio/ktx/issues" - }, - "homepage": "https://github.com/kaelio/ktx#readme" -} diff --git a/packages/connector-mysql/src/index.ts b/packages/connector-mysql/src/index.ts deleted file mode 100644 index ea5da962..00000000 --- a/packages/connector-mysql/src/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -export { KtxMysqlDialect } from './dialect.js'; -export { - isKtxMysqlConnectionConfig, - KtxMysqlScanConnector, - mysqlConnectionPoolConfigFromConfig, - type KtxMysqlColumnDistinctValuesOptions, - type KtxMysqlColumnDistinctValuesResult, - type KtxMysqlConnectionConfig, - type KtxMysqlEndpointResolver, - type KtxMysqlPoolConfig, - type KtxMysqlPoolFactory, - type KtxMysqlReadOnlyQueryInput, - type KtxMysqlScanConnectorOptions, -} from './connector.js'; -export { createMysqlLiveDatabaseIntrospection } from './live-database-introspection.js'; diff --git a/packages/connector-mysql/src/package-exports.test.ts b/packages/connector-mysql/src/package-exports.test.ts deleted file mode 100644 index 14bbd872..00000000 --- a/packages/connector-mysql/src/package-exports.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -describe('@ktx/connector-mysql package exports', () => { - it('exports the native MySQL scan surface', async () => { - const connector = await import('./index.js'); - - expect(connector.KtxMysqlDialect).toBeTypeOf('function'); - expect(connector.KtxMysqlScanConnector).toBeTypeOf('function'); - expect(connector.createMysqlLiveDatabaseIntrospection).toBeTypeOf('function'); - expect(connector.isKtxMysqlConnectionConfig).toBeTypeOf('function'); - expect(connector.mysqlConnectionPoolConfigFromConfig).toBeTypeOf('function'); - }); -}); diff --git a/packages/connector-mysql/tsconfig.json b/packages/connector-mysql/tsconfig.json deleted file mode 100644 index 965e6978..00000000 --- a/packages/connector-mysql/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*.ts"], - "exclude": ["dist", "node_modules"] -} diff --git a/packages/connector-postgres/package.json b/packages/connector-postgres/package.json deleted file mode 100644 index de7c5457..00000000 --- a/packages/connector-postgres/package.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "name": "@ktx/connector-postgres", - "version": "0.0.0-private", - "description": "PostgreSQL connector package for KTX scan interfaces", - "private": true, - "type": "module", - "engines": { - "node": ">=22.0.0" - }, - "main": "dist/index.js", - "types": "dist/index.d.ts", - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - }, - "./package.json": "./package.json" - }, - "files": [ - "dist" - ], - "scripts": { - "build": "tsc -p tsconfig.json", - "test": "vitest run", - "type-check": "tsc -p tsconfig.json --noEmit" - }, - "dependencies": { - "@ktx/context": "workspace:*", - "pg": "^8.20.0" - }, - "devDependencies": { - "@types/node": "^25.7.0", - "@types/pg": "^8.20.0", - "@vitest/coverage-v8": "^4.1.6", - "typescript": "^6.0.3", - "vitest": "^4.1.6" - }, - "license": "Apache-2.0", - "repository": { - "type": "git", - "url": "git+https://github.com/kaelio/ktx.git", - "directory": "packages/connector-postgres" - }, - "bugs": { - "url": "https://github.com/kaelio/ktx/issues" - }, - "homepage": "https://github.com/kaelio/ktx#readme" -} diff --git a/packages/connector-postgres/src/index.ts b/packages/connector-postgres/src/index.ts deleted file mode 100644 index b32e085d..00000000 --- a/packages/connector-postgres/src/index.ts +++ /dev/null @@ -1,21 +0,0 @@ -export { KtxPostgresDialect } from './dialect.js'; -export { - isKtxPostgresConnectionConfig, - KtxPostgresScanConnector, - postgresPoolConfigFromConfig, - type KtxPostgresColumnDistinctValuesOptions, - type KtxPostgresColumnDistinctValuesResult, - type KtxPostgresColumnStatisticsResult, - type KtxPostgresConnectionConfig, - type KtxPostgresEndpointResolver, - type KtxPostgresPoolConfig, - type KtxPostgresPoolFactory, - type KtxPostgresReadOnlyQueryInput, - type KtxPostgresScanConnectorOptions, - type KtxPostgresTableSampleResult, -} from './connector.js'; -export { - KtxPostgresHistoricSqlQueryClient, - type KtxPostgresHistoricSqlQueryClientOptions, -} from './historic-sql-query-client.js'; -export { createPostgresLiveDatabaseIntrospection } from './live-database-introspection.js'; diff --git a/packages/connector-postgres/src/package-exports.test.ts b/packages/connector-postgres/src/package-exports.test.ts deleted file mode 100644 index c216b736..00000000 --- a/packages/connector-postgres/src/package-exports.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -describe('@ktx/connector-postgres package exports', () => { - it('exports the connector, dialect, and live-database adapter', async () => { - const connector = await import('./index.js'); - expect(connector.KtxPostgresDialect).toBeTypeOf('function'); - expect(connector.KtxPostgresScanConnector).toBeTypeOf('function'); - expect(connector.KtxPostgresHistoricSqlQueryClient).toBeTypeOf('function'); - expect(connector.createPostgresLiveDatabaseIntrospection).toBeTypeOf('function'); - expect(connector.isKtxPostgresConnectionConfig).toBeTypeOf('function'); - expect(connector.postgresPoolConfigFromConfig).toBeTypeOf('function'); - }); -}); diff --git a/packages/connector-postgres/tsconfig.json b/packages/connector-postgres/tsconfig.json deleted file mode 100644 index 965e6978..00000000 --- a/packages/connector-postgres/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*.ts"], - "exclude": ["dist", "node_modules"] -} diff --git a/packages/connector-snowflake/package.json b/packages/connector-snowflake/package.json deleted file mode 100644 index 5135a457..00000000 --- a/packages/connector-snowflake/package.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "name": "@ktx/connector-snowflake", - "version": "0.0.0-private", - "description": "Snowflake connector package for KTX scan interfaces", - "private": true, - "type": "module", - "engines": { - "node": ">=22.0.0" - }, - "main": "dist/index.js", - "types": "dist/index.d.ts", - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - }, - "./package.json": "./package.json" - }, - "files": [ - "dist" - ], - "scripts": { - "build": "tsc -p tsconfig.json", - "test": "vitest run", - "type-check": "tsc -p tsconfig.json --noEmit" - }, - "dependencies": { - "@ktx/context": "workspace:*", - "snowflake-sdk": "^2.4.1" - }, - "devDependencies": { - "@types/node": "^25.7.0", - "@vitest/coverage-v8": "^4.1.6", - "typescript": "^6.0.3", - "vitest": "^4.1.6" - }, - "license": "Apache-2.0", - "repository": { - "type": "git", - "url": "git+https://github.com/kaelio/ktx.git", - "directory": "packages/connector-snowflake" - }, - "bugs": { - "url": "https://github.com/kaelio/ktx/issues" - }, - "homepage": "https://github.com/kaelio/ktx#readme" -} diff --git a/packages/connector-snowflake/src/index.ts b/packages/connector-snowflake/src/index.ts deleted file mode 100644 index ea96b0cb..00000000 --- a/packages/connector-snowflake/src/index.ts +++ /dev/null @@ -1,18 +0,0 @@ -export { KtxSnowflakeDialect } from './dialect.js'; -export { - isKtxSnowflakeConnectionConfig, - KtxSnowflakeScanConnector, - snowflakeConnectionConfigFromConfig, - type KtxSnowflakeColumnDistinctValuesOptions, - type KtxSnowflakeColumnDistinctValuesResult, - type KtxSnowflakeConnectionConfig, - type KtxSnowflakeDriver, - type KtxSnowflakeDriverFactory, - type KtxSnowflakeRawColumnMetadata, - type KtxSnowflakeRawTableMetadata, - type KtxSnowflakeReadOnlyQueryInput, - type KtxSnowflakeResolvedConnectionConfig, - type KtxSnowflakeScanConnectorOptions, - type KtxSnowflakeSdkOptionsProvider, -} from './connector.js'; -export { createSnowflakeLiveDatabaseIntrospection } from './live-database-introspection.js'; diff --git a/packages/connector-snowflake/src/package-exports.test.ts b/packages/connector-snowflake/src/package-exports.test.ts deleted file mode 100644 index 985068bf..00000000 --- a/packages/connector-snowflake/src/package-exports.test.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import * as connector from './index.js'; - -describe('@ktx/connector-snowflake package exports', () => { - it('exports public connector, dialect, and introspection APIs', () => { - expect(connector.KtxSnowflakeDialect).toBeTypeOf('function'); - expect(connector.KtxSnowflakeScanConnector).toBeTypeOf('function'); - expect(connector.snowflakeConnectionConfigFromConfig).toBeTypeOf('function'); - expect(connector.createSnowflakeLiveDatabaseIntrospection).toBeTypeOf('function'); - }); -}); diff --git a/packages/connector-snowflake/tsconfig.json b/packages/connector-snowflake/tsconfig.json deleted file mode 100644 index 965e6978..00000000 --- a/packages/connector-snowflake/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*.ts"], - "exclude": ["dist", "node_modules"] -} diff --git a/packages/connector-sqlite/package.json b/packages/connector-sqlite/package.json deleted file mode 100644 index 6dc0b725..00000000 --- a/packages/connector-sqlite/package.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "name": "@ktx/connector-sqlite", - "version": "0.0.0-private", - "description": "SQLite connector package for KTX scan interfaces", - "private": true, - "type": "module", - "engines": { - "node": ">=22.0.0" - }, - "main": "dist/index.js", - "types": "dist/index.d.ts", - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - }, - "./package.json": "./package.json" - }, - "files": [ - "dist" - ], - "scripts": { - "build": "tsc -p tsconfig.json", - "test": "vitest run", - "type-check": "tsc -p tsconfig.json --noEmit" - }, - "dependencies": { - "@ktx/context": "workspace:*", - "better-sqlite3": "^12.10.0" - }, - "devDependencies": { - "@types/better-sqlite3": "^7.6.13", - "@types/node": "^25.7.0", - "@vitest/coverage-v8": "^4.1.6", - "typescript": "^6.0.3", - "vitest": "^4.1.6" - }, - "license": "Apache-2.0", - "repository": { - "type": "git", - "url": "git+https://github.com/kaelio/ktx.git", - "directory": "packages/connector-sqlite" - }, - "bugs": { - "url": "https://github.com/kaelio/ktx/issues" - }, - "homepage": "https://github.com/kaelio/ktx#readme" -} diff --git a/packages/connector-sqlite/src/index.ts b/packages/connector-sqlite/src/index.ts deleted file mode 100644 index 653a9e81..00000000 --- a/packages/connector-sqlite/src/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -export { KtxSqliteDialect } from './dialect.js'; -export { - isKtxSqliteConnectionConfig, - KtxSqliteScanConnector, - sqliteDatabasePathFromConfig, - type KtxSqliteColumnDistinctValuesOptions, - type KtxSqliteColumnDistinctValuesResult, - type KtxSqliteConnectionConfig, - type KtxSqliteReadOnlyQueryInput, - type KtxSqliteScanConnectorOptions, - type SqliteDatabasePathInput, -} from './connector.js'; -export { - createSqliteLiveDatabaseIntrospection, - type CreateSqliteLiveDatabaseIntrospectionOptions, -} from './live-database-introspection.js'; diff --git a/packages/connector-sqlite/src/package-exports.test.ts b/packages/connector-sqlite/src/package-exports.test.ts deleted file mode 100644 index bc6ff6fb..00000000 --- a/packages/connector-sqlite/src/package-exports.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -describe('@ktx/connector-sqlite package exports', () => { - it('exports the native SQLite scan connector surface', async () => { - const connector = await import('./index.js'); - - expect(connector.KtxSqliteDialect).toBeTypeOf('function'); - expect(connector.KtxSqliteScanConnector).toBeTypeOf('function'); - expect(connector.createSqliteLiveDatabaseIntrospection).toBeTypeOf('function'); - expect(connector.isKtxSqliteConnectionConfig).toBeTypeOf('function'); - expect(connector.sqliteDatabasePathFromConfig).toBeTypeOf('function'); - }); -}); diff --git a/packages/connector-sqlite/tsconfig.json b/packages/connector-sqlite/tsconfig.json deleted file mode 100644 index 965e6978..00000000 --- a/packages/connector-sqlite/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*.ts"], - "exclude": ["dist", "node_modules"] -} diff --git a/packages/connector-sqlserver/package.json b/packages/connector-sqlserver/package.json deleted file mode 100644 index cf8dac83..00000000 --- a/packages/connector-sqlserver/package.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "name": "@ktx/connector-sqlserver", - "version": "0.0.0-private", - "description": "SQL Server connector package for KTX scan interfaces", - "private": true, - "type": "module", - "engines": { - "node": ">=22.0.0" - }, - "main": "dist/index.js", - "types": "dist/index.d.ts", - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - }, - "./package.json": "./package.json" - }, - "files": [ - "dist" - ], - "scripts": { - "build": "tsc -p tsconfig.json", - "test": "vitest run", - "type-check": "tsc -p tsconfig.json --noEmit" - }, - "dependencies": { - "@ktx/context": "workspace:*", - "mssql": "^12.5.2" - }, - "devDependencies": { - "@types/mssql": "^12.3.0", - "@types/node": "^25.7.0", - "@vitest/coverage-v8": "^4.1.6", - "typescript": "^6.0.3", - "vitest": "^4.1.6" - }, - "license": "Apache-2.0", - "repository": { - "type": "git", - "url": "git+https://github.com/kaelio/ktx.git", - "directory": "packages/connector-sqlserver" - }, - "bugs": { - "url": "https://github.com/kaelio/ktx/issues" - }, - "homepage": "https://github.com/kaelio/ktx#readme" -} diff --git a/packages/connector-sqlserver/src/index.ts b/packages/connector-sqlserver/src/index.ts deleted file mode 100644 index cbea4d95..00000000 --- a/packages/connector-sqlserver/src/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -export { KtxSqlServerDialect } from './dialect.js'; -export { - isKtxSqlServerConnectionConfig, - KtxSqlServerScanConnector, - sqlServerConnectionPoolConfigFromConfig, - type KtxSqlServerColumnDistinctValuesOptions, - type KtxSqlServerColumnDistinctValuesResult, - type KtxSqlServerConnectionConfig, - type KtxSqlServerEndpointResolver, - type KtxSqlServerPool, - type KtxSqlServerPoolConfig, - type KtxSqlServerPoolFactory, - type KtxSqlServerQueryResult, - type KtxSqlServerReadOnlyQueryInput, - type KtxSqlServerScanConnectorOptions, -} from './connector.js'; -export { createSqlServerLiveDatabaseIntrospection } from './live-database-introspection.js'; diff --git a/packages/connector-sqlserver/src/package-exports.test.ts b/packages/connector-sqlserver/src/package-exports.test.ts deleted file mode 100644 index 0755738c..00000000 --- a/packages/connector-sqlserver/src/package-exports.test.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -describe('@ktx/connector-sqlserver package exports', () => { - it('exports public connector APIs during package bootstrap', async () => { - const connector = await import('./index.js'); - - expect(connector.KtxSqlServerDialect).toBeTypeOf('function'); - expect(connector.KtxSqlServerScanConnector).toBeTypeOf('function'); - expect(connector.createSqlServerLiveDatabaseIntrospection).toBeTypeOf('function'); - expect(connector.sqlServerConnectionPoolConfigFromConfig).toBeTypeOf('function'); - }); -}); diff --git a/packages/connector-sqlserver/tsconfig.json b/packages/connector-sqlserver/tsconfig.json deleted file mode 100644 index 965e6978..00000000 --- a/packages/connector-sqlserver/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*.ts"], - "exclude": ["dist", "node_modules"] -} diff --git a/packages/context/package.json b/packages/context/package.json deleted file mode 100644 index 11cfc28f..00000000 --- a/packages/context/package.json +++ /dev/null @@ -1,176 +0,0 @@ -{ - "name": "@ktx/context", - "version": "0.0.0-private", - "description": "Core context library for database agents", - "private": true, - "type": "module", - "engines": { - "node": ">=22.0.0" - }, - "main": "dist/index.js", - "types": "dist/index.d.ts", - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - }, - "./agent": { - "types": "./dist/agent/index.d.ts", - "import": "./dist/agent/index.js", - "default": "./dist/agent/index.js" - }, - "./core": { - "types": "./dist/core/index.d.ts", - "import": "./dist/core/index.js", - "default": "./dist/core/index.js" - }, - "./connections": { - "types": "./dist/connections/index.d.ts", - "import": "./dist/connections/index.js", - "default": "./dist/connections/index.js" - }, - "./daemon": { - "types": "./dist/daemon/index.d.ts", - "import": "./dist/daemon/index.js", - "default": "./dist/daemon/index.js" - }, - "./ingest": { - "types": "./dist/ingest/index.d.ts", - "import": "./dist/ingest/index.js", - "default": "./dist/ingest/index.js" - }, - "./ingest/memory-flow": { - "types": "./dist/ingest/memory-flow/index.d.ts", - "import": "./dist/ingest/memory-flow/index.js", - "default": "./dist/ingest/memory-flow/index.js" - }, - "./ingest/metabase-mapping": { - "types": "./dist/ingest/metabase-mapping.d.ts", - "import": "./dist/ingest/metabase-mapping.js", - "default": "./dist/ingest/metabase-mapping.js" - }, - "./index-sync": { - "types": "./dist/index-sync/index.d.ts", - "import": "./dist/index-sync/index.js", - "default": "./dist/index-sync/index.js" - }, - "./scan": { - "types": "./dist/scan/index.d.ts", - "import": "./dist/scan/index.js", - "default": "./dist/scan/index.js" - }, - "./search": { - "types": "./dist/search/index.d.ts", - "import": "./dist/search/index.js", - "default": "./dist/search/index.js" - }, - "./sql-analysis": { - "types": "./dist/sql-analysis/index.d.ts", - "import": "./dist/sql-analysis/index.js", - "default": "./dist/sql-analysis/index.js" - }, - "./memory": { - "types": "./dist/memory/index.d.ts", - "import": "./dist/memory/index.js", - "default": "./dist/memory/index.js" - }, - "./mcp": { - "types": "./dist/mcp/index.d.ts", - "import": "./dist/mcp/index.js", - "default": "./dist/mcp/index.js" - }, - "./project": { - "types": "./dist/project/index.d.ts", - "import": "./dist/project/index.js", - "default": "./dist/project/index.js" - }, - "./prompts": { - "types": "./dist/prompts/index.d.ts", - "import": "./dist/prompts/index.js", - "default": "./dist/prompts/index.js" - }, - "./skills": { - "types": "./dist/skills/index.d.ts", - "import": "./dist/skills/index.js", - "default": "./dist/skills/index.js" - }, - "./sl": { - "types": "./dist/sl/index.d.ts", - "import": "./dist/sl/index.js", - "default": "./dist/sl/index.js" - }, - "./sl/descriptions": { - "types": "./dist/sl/descriptions.d.ts", - "import": "./dist/sl/descriptions.js", - "default": "./dist/sl/descriptions.js" - }, - "./tools": { - "types": "./dist/tools/index.d.ts", - "import": "./dist/tools/index.js", - "default": "./dist/tools/index.js" - }, - "./wiki": { - "types": "./dist/wiki/index.d.ts", - "import": "./dist/wiki/index.js", - "default": "./dist/wiki/index.js" - }, - "./package.json": "./package.json" - }, - "files": [ - "dist", - "prompts", - "skills" - ], - "scripts": { - "build": "tsc -p tsconfig.json", - "relationships:benchmarks": "pnpm --silent run build && node scripts/relationship-benchmark-report.mjs", - "relationships:benchmarks:test": "KTX_RUN_RELATIONSHIP_BENCHMARKS=1 vitest run src/scan/relationship-benchmarks.test.ts", - "search:pglite-spike": "node scripts/pglite-hybrid-search-spike.mjs", - "search:pglite-owner-prototype": "node scripts/pglite-owner-process-prototype.mjs", - "search:pglite-sl-prototype": "node scripts/pglite-sl-search-prototype.mjs", - "test": "vitest run --exclude src/scan/relationship-benchmarks.test.ts --exclude src/scan/local-scan.test.ts --exclude src/mcp/local-project-ports.test.ts --exclude src/ingest/local-stage-ingest.test.ts --exclude src/sl/pglite-sl-search-prototype.test.ts --exclude src/core/git.service.test.ts --exclude src/ingest/local-adapters.test.ts --exclude src/ingest/local-bundle-ingest.test.ts --exclude src/ingest/local-metabase-ingest.test.ts --exclude src/sl/local-sl.test.ts --exclude src/search/pglite-owner-process.test.ts --exclude src/scan/local-enrichment-artifacts.test.ts --exclude src/search/pglite-spike.test.ts --exclude src/wiki/local-knowledge.test.ts --exclude src/sl/local-query.test.ts --exclude src/scan/relationship-review-decisions.test.ts --exclude src/scan/relationship-profiling.test.ts", - "test:slow": "vitest run src/scan/local-scan.test.ts src/mcp/local-project-ports.test.ts src/ingest/local-stage-ingest.test.ts src/sl/pglite-sl-search-prototype.test.ts src/core/git.service.test.ts src/ingest/local-adapters.test.ts src/ingest/local-bundle-ingest.test.ts src/ingest/local-metabase-ingest.test.ts src/sl/local-sl.test.ts src/search/pglite-owner-process.test.ts src/scan/local-enrichment-artifacts.test.ts src/search/pglite-spike.test.ts src/wiki/local-knowledge.test.ts src/sl/local-query.test.ts src/scan/relationship-review-decisions.test.ts src/scan/relationship-profiling.test.ts --testTimeout 30000", - "type-check": "tsc -p tsconfig.json --noEmit" - }, - "dependencies": { - "@anthropic-ai/claude-agent-sdk": "0.3.142", - "@ktx/llm": "workspace:*", - "@looker/sdk": "^26.8.0", - "@looker/sdk-node": "^26.8.0", - "@looker/sdk-rtl": "^21.6.5", - "@modelcontextprotocol/sdk": "^1.29.0", - "@notionhq/client": "^5.21.0", - "ai": "^6.0.180", - "better-sqlite3": "^12.10.0", - "handlebars": "^4.7.9", - "lookml-parser": "7.1.0", - "minimatch": "^10.2.5", - "p-limit": "^7.3.0", - "pg": "^8.20.0", - "simple-git": "3.36.0", - "yaml": "^2.9.0", - "zod": "^4.4.3" - }, - "devDependencies": { - "@electric-sql/pglite": "^0.4.5", - "@electric-sql/pglite-socket": "^0.1.5", - "@types/better-sqlite3": "^7.6.13", - "@types/node": "^25.7.0", - "@types/pg": "^8.20.0", - "@vitest/coverage-v8": "^4.1.6", - "ajv": "8.20.0", - "typescript": "^6.0.3", - "vitest": "^4.1.6" - }, - "license": "Apache-2.0", - "repository": { - "type": "git", - "url": "git+https://github.com/kaelio/ktx.git", - "directory": "packages/context" - }, - "bugs": { - "url": "https://github.com/kaelio/ktx/issues" - }, - "homepage": "https://github.com/kaelio/ktx#readme" -} diff --git a/packages/context/src/agent/index.ts b/packages/context/src/agent/index.ts deleted file mode 100644 index 8cc0a17c..00000000 --- a/packages/context/src/agent/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export type { - AgentRunnerPort, - RunLoopParams, - RunLoopResult, - RunLoopStepInfo, - RunLoopStopReason, -} from '../llm/runtime-port.js'; -export { RuntimeAgentRunner } from '../llm/runtime-port.js'; -export type { AgentTelemetryPort } from '../llm/ai-sdk-runtime.js'; diff --git a/packages/context/src/connections/index.ts b/packages/context/src/connections/index.ts deleted file mode 100644 index 0917a7ca..00000000 --- a/packages/context/src/connections/index.ts +++ /dev/null @@ -1,30 +0,0 @@ -export type { - KtxSqlQueryExecutionInput, - KtxSqlQueryExecutionResult, - KtxSqlQueryExecutorPort, -} from './query-executor.js'; -export type { KtxDialect, SupportedDriver } from './dialects.js'; -export { createDefaultLocalQueryExecutor, type DefaultLocalQueryExecutorOptions } from './local-query-executor.js'; -export { getDialectForDriver } from './dialects.js'; -export { normalizeQueryRows } from './query-executor.js'; -export { createPostgresQueryExecutor } from './postgres-query-executor.js'; -export { assertReadOnlySql, limitSqlForExecution } from './read-only-sql.js'; -export { createSqliteQueryExecutor, sqliteDatabasePathFromConnection } from './sqlite-query-executor.js'; -export { connectionTypeSchema, type ConnectionType } from './connection-type.js'; -export { - localConnectionInfoFromConfig, - localConnectionToWarehouseDescriptor, - localConnectionTypeForConfig, - type LocalConnectionInfo, - type LocalWarehouseDescriptor, -} from './local-warehouse-descriptor.js'; -export { - KTX_NOTION_ORG_KNOWLEDGE_WARNING, - notionConnectionToPullConfig, - parseNotionConnectionConfig, - redactNotionConnectionConfig, - resolveNotionConnectionAuthToken, - resolveNotionAuthToken, - type KtxNotionConnectionConfig, - type RedactedKtxNotionConnectionConfig, -} from './notion-config.js'; diff --git a/packages/context/src/core/index.ts b/packages/context/src/core/index.ts deleted file mode 100644 index 892ef98e..00000000 --- a/packages/context/src/core/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -export type { KtxCoreConfig, KtxGitConfig, KtxLogger, KtxStorageConfig } from './config.js'; -export { noopLogger, resolveConfigDir, resolveWorktreesDir } from './config.js'; -export { resolveKtxConfigReference, resolveKtxHomePath } from './config-reference.js'; -export type { KtxEmbeddingPort } from './embedding.js'; -export { - REDACTED_KTX_CREDENTIAL_VALUE, - redactKtxSensitiveMetadata, - redactKtxSensitiveText, - redactKtxSensitiveValue, -} from './redaction.js'; -export type { - KtxFileHistoryEntry, - KtxFileListResult, - KtxFileReadResult, - KtxFileStorePort, - KtxFileWriteResult, -} from './file-store.js'; -export type { GitCommitInfo, SquashMergeResult, WorktreeEntry } from './git.service.js'; -export { GitService } from './git.service.js'; -export type { - SentinelPayload, - SessionOutcome, - SessionWorktree, - SessionWorktreeServiceDeps, - WorktreeConfigPort, -} from './session-worktree.service.js'; -export { SessionWorktreeService } from './session-worktree.service.js'; diff --git a/packages/context/src/daemon/index.ts b/packages/context/src/daemon/index.ts deleted file mode 100644 index 0a218181..00000000 --- a/packages/context/src/daemon/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './semantic-layer-compute.js'; diff --git a/packages/context/src/index-sync/index.ts b/packages/context/src/index-sync/index.ts deleted file mode 100644 index 5863688e..00000000 --- a/packages/context/src/index-sync/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export type { ReindexOptions, ReindexScopeResult, ReindexSummary, ReindexWorkResult } from './types.js'; -export { discoverReindexScopes, reindexLocalIndexes } from './reindex.js'; diff --git a/packages/context/src/index.test.ts b/packages/context/src/index.test.ts deleted file mode 100644 index 255ad3a6..00000000 --- a/packages/context/src/index.test.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { ktxContextPackageInfo } from './index.js'; - -describe('ktxContextPackageInfo', () => { - it('identifies the context package', () => { - expect(ktxContextPackageInfo).toEqual({ - name: '@ktx/context', - version: '0.0.0-private', - }); - }); -}); diff --git a/packages/context/src/index.ts b/packages/context/src/index.ts deleted file mode 100644 index b8623c7c..00000000 --- a/packages/context/src/index.ts +++ /dev/null @@ -1,138 +0,0 @@ -export interface KtxContextPackageInfo { - name: '@ktx/context'; - version: '0.0.0-private'; -} - -export const ktxContextPackageInfo: KtxContextPackageInfo = { - name: '@ktx/context', - version: '0.0.0-private', -}; - -export * from './agent/index.js'; -export * from './core/index.js'; -export * from './daemon/index.js'; -export * from './ingest/index.js'; -export * from './index-sync/index.js'; -export * from './llm/index.js'; -export type { - CaptureSession, - CaptureSignals, - MemoryAgentInput, - MemoryAgentResult, - MemoryAgentServiceDeps, - MemoryAgentSettings, - MemoryAgentSourceType, - MemoryCommitMessagePort, - MemoryConnectionPort, - MemoryFileStorePort, - MemoryKnowledgeSlRefsPort, - MemoryLockPort, - MemorySlSourceReconcilerPort, - MemoryTelemetryPort, - MemoryToolSetLike, - MemoryToolsetFactoryPort, -} from './memory/index.js'; -export * from './project/index.js'; -export * from './prompts/index.js'; -export * from './search/index.js'; -export * from './sql-analysis/index.js'; -export type { - KtxColumnAnalysisResult, - KtxColumnDescriptionPromptInput, - KtxColumnEmbeddingForeignKeys, - KtxColumnEmbeddingTextInput, - KtxColumnSampleInput, - KtxColumnSampleResult, - KtxColumnSampleUpdate, - KtxColumnStatsInput, - KtxColumnStatsResult, - KtxConnectionDriver, - KtxConnectorCapabilities, - KtxCredentialEnvelope, - KtxCredentialEnvReference, - KtxCredentialFileReference, - KtxDataDictionaryColumnState, - KtxDataDictionarySampleDecision, - KtxDataDictionarySettings, - KtxDataDictionarySkipReason, - KtxDataSourceDescriptionPromptInput, - KtxDescriptionCachePort, - KtxDescriptionColumn, - KtxDescriptionColumnTable, - KtxDescriptionGenerationSettings, - KtxDescriptionGeneratorOptions, - KtxDescriptionSource, - KtxDescriptionTableInput, - KtxDescriptionUpdate, - KtxEmbeddingPort as KtxScanEmbeddingPort, - KtxEmbeddingUpdate, - KtxEnrichedColumn, - KtxEnrichedRelationship, - KtxEnrichedSchema, - KtxEnrichedTable, - KtxGenerateColumnDescriptionsInput, - KtxGenerateDataSourceDescriptionInput, - KtxGenerateTableDescriptionInput, - KtxOptionalConnectorCapabilities, - KtxProgressPort, - KtxQueryResult as KtxScanQueryResult, - KtxReadOnlyQueryInput, - KtxRelationshipEndpoint, - KtxRelationshipSource, - KtxRelationshipType, - KtxRelationshipUpdate, - KtxResolvedCredentialEnvelope, - KtxScanArtifactPaths, - KtxScanConnector, - KtxScanContext, - KtxScanDiffSummary, - KtxScanEnrichmentSummary, - KtxScanInput, - KtxScanLoggerPort, - KtxScanMetadataStore, - KtxScanMode, - KtxScanRelationshipSummary, - KtxScanReport, - KtxScanTrigger, - KtxScanWarning, - KtxScanWarningCode, - KtxSchemaColumn, - KtxSchemaDimensionType, - KtxSchemaForeignKey, - KtxSchemaScope, - KtxSchemaSnapshot, - KtxSchemaTable, - KtxSchemaTableKind, - KtxSkippedRelationship, - KtxStructuralSyncPlan, - KtxStructuralSyncStats, - KtxTableDescriptionPromptInput, - KtxTableRef, - KtxTableSampleInput, - KtxTableSampleResult, - KtxColumnTypeMapping, -} from './scan/index.js'; -export { - buildKtxColumnDescriptionPrompt, - buildKtxColumnEmbeddingText, - buildKtxDataSourceDescriptionPrompt, - buildKtxTableDescriptionPrompt, - createKtxConnectorCapabilities, - defaultKtxDataDictionarySettings, - inferKtxDimensionType, - isKtxDataDictionaryCandidate, - ktxColumnTypeMappingFromNative, - KtxDescriptionGenerator, - normalizeKtxNativeType, - REDACTED_KTX_CREDENTIAL_VALUE, - redactKtxCredentialEnvelope, - redactKtxCredentialValue, - redactKtxScanMetadata, - redactKtxScanReport, - redactKtxScanWarning, - shouldKtxSampleColumnForDictionary, -} from './scan/index.js'; -export * from './skills/index.js'; -export * from './sl/index.js'; -export * from './tools/index.js'; -export * from './wiki/index.js'; diff --git a/packages/context/src/ingest/adapters/dbt-descriptions/match-tables.test.ts b/packages/context/src/ingest/adapters/dbt-descriptions/match-tables.test.ts deleted file mode 100644 index ce83e974..00000000 --- a/packages/context/src/ingest/adapters/dbt-descriptions/match-tables.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { DbtParsedTable } from './parse-schema.js'; -import { findMatchingKtxTable, matchDbtTables, type DbtHostTableLite } from './match-tables.js'; - -const hostTables: DbtHostTableLite[] = [ - { id: '1', name: 'orders', catalog: 'warehouse', db: 'analytics', columns: [{ id: 'c1', name: 'id' }] }, - { id: '2', name: 'orders', catalog: 'warehouse', db: 'staging', columns: [{ id: 'c2', name: 'id' }] }, - { id: '3', name: 'customers', catalog: null, db: null, columns: [{ id: 'c3', name: 'id' }] }, -]; - -function table(input: Partial): DbtParsedTable { - return { - name: 'orders', - description: null, - database: null, - schema: null, - columns: [], - resourceType: 'model', - ...input, - }; -} - -describe('dbt descriptions table matching', () => { - it('uses schema plus name first and checks catalog when dbt database is present', () => { - expect( - findMatchingKtxTable(table({ database: 'warehouse', schema: 'analytics' }), hostTables, null)?.id, - ).toBe('1'); - }); - - it('does not fall back to name-only for source tables', () => { - expect(findMatchingKtxTable(table({ resourceType: 'source' }), hostTables, null)).toBeUndefined(); - }); - - it('uses targetSchema for models and name-only only when unique', () => { - expect(findMatchingKtxTable(table({ resourceType: 'model' }), hostTables, 'staging')?.id).toBe('2'); - expect(findMatchingKtxTable(table({ name: 'customers', resourceType: 'model' }), hostTables, null)?.id).toBe( - '3', - ); - expect(findMatchingKtxTable(table({ resourceType: 'model' }), hostTables, null)).toBeUndefined(); - }); - - it('summarizes matched columns and descriptions', () => { - const matches = matchDbtTables( - [ - table({ - name: 'customers', - description: 'Customers', - columns: [ - { name: 'id', description: 'Primary key', dataType: null }, - { name: 'missing', description: 'Missing', dataType: null }, - ], - }), - ], - hostTables, - null, - ); - - expect(matches).toEqual([ - { - dbtTable: 'customers', - dbtSchema: null, - dbtDatabase: null, - hostTableId: '3', - hostTableName: 'customers', - matched: true, - tableDescriptionAction: 'import', - tableDescriptionFound: true, - columnsToImport: 1, - columnsMatched: 1, - columnsTotal: 2, - columnDescriptionsFound: 1, - }, - ]); - }); -}); diff --git a/packages/context/src/ingest/adapters/dbt-descriptions/match-tables.ts b/packages/context/src/ingest/adapters/dbt-descriptions/match-tables.ts deleted file mode 100644 index ca8414ae..00000000 --- a/packages/context/src/ingest/adapters/dbt-descriptions/match-tables.ts +++ /dev/null @@ -1,127 +0,0 @@ -import type { DbtParsedTable } from './parse-schema.js'; - -export interface DbtHostTableLite { - id: string; - name: string; - catalog: string | null; - db: string | null; - columns: Array<{ id: string; name: string }>; -} - -export interface DbtTableMatch { - dbtTable: string; - dbtSchema: string | null; - dbtDatabase: string | null; - hostTableId: string | null; - hostTableName: string | null; - matched: boolean; - tableDescriptionAction: 'skip' | 'import'; - tableDescriptionFound: boolean; - columnsToImport: number; - columnsMatched: number; - columnsTotal: number; - columnDescriptionsFound: number; -} - -export function matchDbtTables( - dbtTables: DbtParsedTable[], - hostTables: DbtHostTableLite[], - targetSchema?: string | null, -): DbtTableMatch[] { - return dbtTables.map((dbtTable) => { - const hostTable = findMatchingKtxTable(dbtTable, hostTables, targetSchema); - - if (!hostTable) { - return { - dbtTable: dbtTable.name, - dbtSchema: dbtTable.schema, - dbtDatabase: dbtTable.database, - hostTableId: null, - hostTableName: null, - matched: false, - tableDescriptionAction: 'skip', - tableDescriptionFound: Boolean(dbtTable.description), - columnsToImport: 0, - columnsMatched: 0, - columnsTotal: dbtTable.columns.length, - columnDescriptionsFound: dbtTable.columns.filter((column) => Boolean(column.description)).length, - }; - } - - const analysis = analyzeColumns(dbtTable, hostTable); - return { - dbtTable: dbtTable.name, - dbtSchema: dbtTable.schema, - dbtDatabase: dbtTable.database, - hostTableId: hostTable.id, - hostTableName: hostTable.name, - matched: true, - tableDescriptionAction: dbtTable.description ? 'import' : 'skip', - tableDescriptionFound: Boolean(dbtTable.description), - ...analysis, - }; - }); -} - -export function findMatchingKtxTable( - dbtTable: DbtParsedTable, - hostTables: DbtHostTableLite[], - targetSchema?: string | null, -): DbtHostTableLite | undefined { - const dbtName = dbtTable.name.toLowerCase(); - const effectiveSchema = dbtTable.schema ?? targetSchema ?? null; - - if (effectiveSchema) { - const strictMatch = hostTables.find((table) => { - const nameMatches = table.name.toLowerCase() === dbtName; - const schemaMatches = table.db?.toLowerCase() === effectiveSchema.toLowerCase(); - if (!nameMatches || !schemaMatches) { - return false; - } - if (dbtTable.database && table.catalog) { - return table.catalog.toLowerCase() === dbtTable.database.toLowerCase(); - } - return true; - }); - if (strictMatch) { - return strictMatch; - } - } - - if (dbtTable.resourceType === 'source') { - return undefined; - } - - const nameMatches = hostTables.filter((table) => table.name.toLowerCase() === dbtName); - return nameMatches.length === 1 ? nameMatches[0] : undefined; -} - -function analyzeColumns( - dbtTable: DbtParsedTable, - hostTable: DbtHostTableLite, -): Pick { - let columnsToImport = 0; - let columnsMatched = 0; - let columnDescriptionsFound = 0; - - for (const dbtColumn of dbtTable.columns) { - const hostColumn = hostTable.columns.find( - (column) => column.name.toLowerCase() === dbtColumn.name.toLowerCase(), - ); - if (!hostColumn) { - continue; - } - columnsMatched++; - if (dbtColumn.description) { - columnDescriptionsFound++; - columnsToImport++; - } - } - - return { - columnsToImport, - columnsMatched, - columnsTotal: dbtTable.columns.length, - columnDescriptionsFound, - }; -} diff --git a/packages/context/src/ingest/adapters/dbt-descriptions/merge-semantic-model-tables.test.ts b/packages/context/src/ingest/adapters/dbt-descriptions/merge-semantic-model-tables.test.ts deleted file mode 100644 index 9d1ec735..00000000 --- a/packages/context/src/ingest/adapters/dbt-descriptions/merge-semantic-model-tables.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { ParsedSemanticModel } from '../metricflow/deep-parse.js'; -import { mergeSemanticModelTables } from './merge-semantic-model-tables.js'; -import type { DbtSchemaParseResult } from './parse-schema.js'; - -const semanticModel: ParsedSemanticModel = { - name: 'orders_semantic', - description: 'Order facts', - modelRef: 'fct_orders', - dimensions: [ - { name: 'status', column: 'status', type: 'categorical', description: 'Order status' }, - { name: 'ordered_at', column: 'ordered_at', type: 'time' }, - ], - measures: [], - entities: [], - defaultTimeDimension: null, -}; - -describe('mergeSemanticModelTables', () => { - it('adds missing MetricFlow model refs as dbt model tables', () => { - const input: DbtSchemaParseResult = { projectName: null, dbtVersion: null, tables: [], relationships: [] }; - - expect(mergeSemanticModelTables(input, [semanticModel])).toEqual({ - projectName: null, - dbtVersion: null, - relationships: [], - tables: [ - { - name: 'fct_orders', - description: 'Order facts', - database: null, - schema: null, - resourceType: 'model', - columns: [ - { name: 'status', description: 'Order status', dataType: null }, - { name: 'ordered_at', description: null, dataType: 'TIMESTAMP' }, - ], - }, - ], - }); - }); - - it('does not add a duplicate table when schema parsing already found the model ref', () => { - const input: DbtSchemaParseResult = { - projectName: null, - dbtVersion: null, - relationships: [], - tables: [ - { - name: 'FCT_ORDERS', - description: 'Existing', - database: null, - schema: null, - resourceType: 'model', - columns: [], - }, - ], - }; - - expect(mergeSemanticModelTables(input, [semanticModel]).tables).toHaveLength(1); - }); -}); diff --git a/packages/context/src/ingest/adapters/dbt-descriptions/merge-semantic-model-tables.ts b/packages/context/src/ingest/adapters/dbt-descriptions/merge-semantic-model-tables.ts deleted file mode 100644 index 2991153f..00000000 --- a/packages/context/src/ingest/adapters/dbt-descriptions/merge-semantic-model-tables.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { ParsedSemanticModel } from '../metricflow/deep-parse.js'; -import type { DbtSchemaParseResult } from './parse-schema.js'; - -export function mergeSemanticModelTables( - parseResult: DbtSchemaParseResult, - semanticModels: ParsedSemanticModel[], -): DbtSchemaParseResult { - const merged: DbtSchemaParseResult = { - ...parseResult, - tables: [...parseResult.tables], - relationships: [...parseResult.relationships], - }; - const existingTableNames = new Set(merged.tables.map((table) => table.name.toLowerCase())); - - for (const model of semanticModels) { - const tableName = model.modelRef; - if (existingTableNames.has(tableName.toLowerCase())) { - continue; - } - - merged.tables.push({ - name: tableName, - description: model.description, - database: null, - schema: null, - columns: model.dimensions.map((dimension) => ({ - name: dimension.column, - description: dimension.description ?? null, - dataType: dimension.type === 'time' ? 'TIMESTAMP' : null, - })), - resourceType: 'model', - }); - existingTableNames.add(tableName.toLowerCase()); - } - - return merged; -} diff --git a/packages/context/src/ingest/adapters/dbt-descriptions/to-description-updates.test.ts b/packages/context/src/ingest/adapters/dbt-descriptions/to-description-updates.test.ts deleted file mode 100644 index 0b083213..00000000 --- a/packages/context/src/ingest/adapters/dbt-descriptions/to-description-updates.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { DbtSchemaParseResult } from './parse-schema.js'; -import { toDescriptionUpdates } from './to-description-updates.js'; -import type { DbtHostTableLite } from './match-tables.js'; - -const hostTables: DbtHostTableLite[] = [ - { - id: '1', - name: 'orders', - catalog: 'warehouse', - db: 'analytics', - columns: [ - { id: 'c1', name: 'id' }, - { id: 'c2', name: 'amount' }, - ], - }, -]; - -function parseResult(description: string | null, columnDescription: string | null): DbtSchemaParseResult { - return { - projectName: null, - dbtVersion: null, - relationships: [], - tables: [ - { - name: 'orders', - description, - database: 'warehouse', - schema: 'analytics', - resourceType: 'model', - columns: [ - { name: 'id', description: columnDescription, dataType: null }, - { name: 'missing', description: 'not imported', dataType: null }, - ], - }, - ], - }; -} - -describe('dbt descriptions update payloads', () => { - it('emits dbt writes and matching ai invalidations when descriptions exist', () => { - expect( - toDescriptionUpdates({ - connectionId: 'conn-1', - parseResult: parseResult('Orders table', 'Primary key'), - hostTables, - targetSchema: null, - }), - ).toEqual({ - dbt: [ - { - connectionId: 'conn-1', - table: { catalog: 'warehouse', db: 'analytics', name: 'orders' }, - source: 'dbt', - tableDescription: 'Orders table', - columnDescriptions: { id: 'Primary key' }, - }, - ], - aiInvalidations: [ - { - connectionId: 'conn-1', - table: { catalog: 'warehouse', db: 'analytics', name: 'orders' }, - source: 'ai', - }, - ], - }); - }); - - it('does not emit spurious dbt writes or ai invalidations when no descriptions exist', () => { - expect( - toDescriptionUpdates({ - connectionId: 'conn-1', - parseResult: parseResult(null, null), - hostTables, - targetSchema: null, - }), - ).toEqual({ dbt: [], aiInvalidations: [] }); - }); - - it('emits ai invalidation without a dbt description write when only structural metadata exists', () => { - const result = parseResult(null, null); - result.tables[0]!.tagsDbt = ['finance']; - - expect( - toDescriptionUpdates({ - connectionId: 'conn-1', - parseResult: result, - hostTables, - targetSchema: null, - }), - ).toEqual({ - dbt: [], - aiInvalidations: [ - { - connectionId: 'conn-1', - table: { catalog: 'warehouse', db: 'analytics', name: 'orders' }, - source: 'ai', - }, - ], - }); - }); -}); diff --git a/packages/context/src/ingest/adapters/dbt-descriptions/to-description-updates.ts b/packages/context/src/ingest/adapters/dbt-descriptions/to-description-updates.ts deleted file mode 100644 index 70c895be..00000000 --- a/packages/context/src/ingest/adapters/dbt-descriptions/to-description-updates.ts +++ /dev/null @@ -1,70 +0,0 @@ -import type { KtxDescriptionUpdate } from '../../../scan/enrichment-types.js'; -import { findMatchingKtxTable, type DbtHostTableLite } from './match-tables.js'; -import type { DbtSchemaParseResult } from './parse-schema.js'; - -export interface DbtDescriptionUpdates { - dbt: KtxDescriptionUpdate[]; - aiInvalidations: KtxDescriptionUpdate[]; -} - -export function toDescriptionUpdates(input: { - connectionId: string; - parseResult: DbtSchemaParseResult; - hostTables: DbtHostTableLite[]; - targetSchema: string | null; -}): DbtDescriptionUpdates { - const dbt: KtxDescriptionUpdate[] = []; - const aiInvalidations: KtxDescriptionUpdate[] = []; - - for (const dbtTable of input.parseResult.tables) { - const hostTable = findMatchingKtxTable(dbtTable, input.hostTables, input.targetSchema); - if (!hostTable) { - continue; - } - - const tableDescription = dbtTable.description ?? undefined; - const columnDescriptions: Record = {}; - - for (const dbtColumn of dbtTable.columns) { - if (!dbtColumn.description) { - continue; - } - const hostColumn = hostTable.columns.find( - (column) => column.name.toLowerCase() === dbtColumn.name.toLowerCase(), - ); - if (hostColumn) { - columnDescriptions[hostColumn.name] = dbtColumn.description; - } - } - - const hasColumnDescriptions = Object.keys(columnDescriptions).length > 0; - const hasDescriptionChange = tableDescription !== undefined || hasColumnDescriptions; - const hasMetadataChange = - !!dbtTable.tagsDbt?.length || - dbtTable.freshnessDbt !== undefined || - dbtTable.columns.some( - (column) => column.constraints !== undefined || !!column.enumValuesDbt?.length || !!column.dataTests?.length, - ); - if (!hasDescriptionChange && !hasMetadataChange) { - continue; - } - - const tableRef = { catalog: hostTable.catalog, db: hostTable.db, name: hostTable.name }; - if (hasDescriptionChange) { - dbt.push({ - connectionId: input.connectionId, - table: tableRef, - source: 'dbt', - ...(tableDescription !== undefined ? { tableDescription } : {}), - ...(hasColumnDescriptions ? { columnDescriptions } : {}), - }); - } - aiInvalidations.push({ - connectionId: input.connectionId, - table: tableRef, - source: 'ai', - }); - } - - return { dbt, aiInvalidations }; -} diff --git a/packages/context/src/ingest/adapters/dbt-descriptions/to-metadata-updates.test.ts b/packages/context/src/ingest/adapters/dbt-descriptions/to-metadata-updates.test.ts deleted file mode 100644 index ee33b369..00000000 --- a/packages/context/src/ingest/adapters/dbt-descriptions/to-metadata-updates.test.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { toMetadataUpdates } from './to-metadata-updates.js'; - -describe('toMetadataUpdates', () => { - it('emits source-keyed dbt metadata updates for matched tables and columns', () => { - const updates = toMetadataUpdates({ - connectionId: 'conn_1', - targetSchema: 'analytics', - hostTables: [ - { - id: 'orders-id', - name: 'orders', - catalog: 'warehouse', - db: 'analytics', - columns: [ - { id: 'status-id', name: 'status' }, - { id: 'created-id', name: 'created_at' }, - ], - }, - ], - parseResult: { - projectName: null, - dbtVersion: null, - relationships: [], - tables: [ - { - name: 'orders', - description: null, - database: 'warehouse', - schema: 'analytics', - resourceType: 'model', - tagsDbt: ['finance'], - freshnessDbt: { loadedAtField: 'created_at' }, - columns: [ - { - name: 'status', - description: null, - dataType: null, - enumValuesDbt: ['placed', 'shipped'], - constraints: { dbt: { not_null: true } }, - dataTests: [{ name: 'accepted_values', package: 'dbt', kwargs: { values: ['placed', 'shipped'] } }], - }, - ], - }, - ], - }, - }); - - expect(updates).toEqual([ - { - connectionId: 'conn_1', - table: { catalog: 'warehouse', db: 'analytics', name: 'orders' }, - source: 'dbt', - tableFields: { - tags: ['finance'], - freshness: { loaded_at_field: 'created_at' }, - }, - columnFields: { - status: { - constraints: { not_null: true }, - enum_values: ['placed', 'shipped'], - tests: [ - { name: 'accepted_values', package: 'dbt', kwargs: { values: ['placed', 'shipped'] } }, - ], - }, - }, - }, - ]); - }); -}); diff --git a/packages/context/src/ingest/adapters/dbt-descriptions/to-metadata-updates.ts b/packages/context/src/ingest/adapters/dbt-descriptions/to-metadata-updates.ts deleted file mode 100644 index 6ebe4a8f..00000000 --- a/packages/context/src/ingest/adapters/dbt-descriptions/to-metadata-updates.ts +++ /dev/null @@ -1,74 +0,0 @@ -import type { KtxMetadataUpdate } from '../../../scan/enrichment-types.js'; -import { findMatchingKtxTable, type DbtHostTableLite } from './match-tables.js'; -import type { DbtSchemaParseResult } from './parse-schema.js'; - -export function toMetadataUpdates(input: { - connectionId: string; - parseResult: DbtSchemaParseResult; - hostTables: DbtHostTableLite[]; - targetSchema: string | null; -}): KtxMetadataUpdate[] { - const updates: KtxMetadataUpdate[] = []; - - for (const dbtTable of input.parseResult.tables) { - const hostTable = findMatchingKtxTable(dbtTable, input.hostTables, input.targetSchema); - if (!hostTable) { - continue; - } - - const tableFields: Record = {}; - if (dbtTable.tagsDbt?.length) { - tableFields.tags = dbtTable.tagsDbt; - } - if (dbtTable.freshnessDbt) { - tableFields.freshness = { - ...(dbtTable.freshnessDbt.raw !== undefined ? { raw: dbtTable.freshnessDbt.raw } : {}), - ...(dbtTable.freshnessDbt.loadedAtField !== undefined - ? { loaded_at_field: dbtTable.freshnessDbt.loadedAtField } - : {}), - }; - } - - const columnFields: Record> = {}; - for (const dbtColumn of dbtTable.columns) { - const hostColumn = hostTable.columns.find( - (column) => column.name.toLowerCase() === dbtColumn.name.toLowerCase(), - ); - if (!hostColumn) { - continue; - } - - const fields: Record = {}; - if (dbtColumn.constraints) { - fields.constraints = dbtColumn.constraints.dbt; - } - if (dbtColumn.enumValuesDbt?.length) { - fields.enum_values = dbtColumn.enumValuesDbt; - } - if (dbtColumn.dataTests?.length) { - fields.tests = dbtColumn.dataTests.map((test) => ({ - name: test.name, - package: test.package, - ...(test.kwargs ? { kwargs: test.kwargs } : {}), - })); - } - if (Object.keys(fields).length > 0) { - columnFields[hostColumn.name] = fields; - } - } - - if (Object.keys(tableFields).length === 0 && Object.keys(columnFields).length === 0) { - continue; - } - - updates.push({ - connectionId: input.connectionId, - table: { catalog: hostTable.catalog, db: hostTable.db, name: hostTable.name }, - source: 'dbt', - ...(Object.keys(tableFields).length > 0 ? { tableFields } : {}), - ...(Object.keys(columnFields).length > 0 ? { columnFields } : {}), - }); - } - - return updates; -} diff --git a/packages/context/src/ingest/adapters/dbt-descriptions/to-relationship-updates.test.ts b/packages/context/src/ingest/adapters/dbt-descriptions/to-relationship-updates.test.ts deleted file mode 100644 index 02fe5f63..00000000 --- a/packages/context/src/ingest/adapters/dbt-descriptions/to-relationship-updates.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { DbtHostTableLite } from './match-tables.js'; -import type { DbtSchemaParseResult } from './parse-schema.js'; -import { toRelationshipUpdates } from './to-relationship-updates.js'; - -const DBT_SYSTEM_EMAIL = ['system@kae', 'lio.dev'].join(''); - -const hostTables: DbtHostTableLite[] = [ - { - id: '1', - name: 'orders', - catalog: 'warehouse', - db: 'analytics', - columns: [{ id: 'c1', name: 'customer_id' }], - }, - { - id: '2', - name: 'customers', - catalog: 'warehouse', - db: 'staging', - columns: [{ id: 'c2', name: 'id' }], - }, -]; - -const parseResult: DbtSchemaParseResult = { - projectName: null, - dbtVersion: null, - tables: [], - relationships: [ - { - fromTable: 'orders', - fromColumn: 'customer_id', - toTable: 'customers', - toColumn: 'id', - fromSchema: 'analytics', - toSchema: 'analytics', - description: 'schema intentionally differs from the host customers table', - }, - { fromTable: 'orders', fromColumn: 'missing', toTable: 'customers', toColumn: 'id' }, - { fromTable: 'orders', fromColumn: 'customer_id', toTable: 'missing_table', toColumn: 'id' }, - ], -}; - -describe('dbt relationship update payloads', () => { - it('validates relationships using the current name-only matching behavior and dbt provenance', () => { - expect(toRelationshipUpdates({ connectionId: 'conn-1', parseResult, hostTables })).toEqual({ - joins: [ - { - connectionId: 'conn-1', - fromTable: 'orders', - fromColumns: ['customer_id'], - toTable: 'customers', - toColumns: ['id'], - relationship: 'many_to_one', - author: 'dbt', - authorEmail: DBT_SYSTEM_EMAIL, - }, - ], - skippedNoMatch: 2, - }); - }); -}); diff --git a/packages/context/src/ingest/adapters/dbt-descriptions/to-relationship-updates.ts b/packages/context/src/ingest/adapters/dbt-descriptions/to-relationship-updates.ts deleted file mode 100644 index 9f649434..00000000 --- a/packages/context/src/ingest/adapters/dbt-descriptions/to-relationship-updates.ts +++ /dev/null @@ -1,57 +0,0 @@ -import type { KtxJoinUpdate } from '../../../scan/enrichment-types.js'; -import type { DbtHostTableLite } from './match-tables.js'; -import type { DbtSchemaParseResult } from './parse-schema.js'; - -export interface DbtRelationshipUpdates { - joins: KtxJoinUpdate[]; - skippedNoMatch: number; -} - -const DBT_SYSTEM_EMAIL = ['system@kae', 'lio.dev'].join(''); - -export function toRelationshipUpdates(input: { - connectionId: string; - parseResult: DbtSchemaParseResult; - hostTables: DbtHostTableLite[]; -}): DbtRelationshipUpdates { - const tablesByName = new Map(); - for (const table of input.hostTables) { - tablesByName.set(table.name.toLowerCase(), table); - } - - const joins: KtxJoinUpdate[] = []; - let skippedNoMatch = 0; - - for (const relationship of input.parseResult.relationships) { - const fromTable = tablesByName.get(relationship.fromTable.toLowerCase()); - const toTable = tablesByName.get(relationship.toTable.toLowerCase()); - if (!fromTable || !toTable) { - skippedNoMatch++; - continue; - } - - const fromColumn = fromTable.columns.find( - (column) => column.name.toLowerCase() === relationship.fromColumn.toLowerCase(), - ); - const toColumn = toTable.columns.find( - (column) => column.name.toLowerCase() === relationship.toColumn.toLowerCase(), - ); - if (!fromColumn || !toColumn) { - skippedNoMatch++; - continue; - } - - joins.push({ - connectionId: input.connectionId, - fromTable: fromTable.name, - fromColumns: [fromColumn.name], - toTable: toTable.name, - toColumns: [toColumn.name], - relationship: 'many_to_one', - author: 'dbt', - authorEmail: DBT_SYSTEM_EMAIL, - }); - } - - return { joins, skippedNoMatch }; -} diff --git a/packages/context/src/ingest/adapters/dbt-extraction-golden-parity.test.ts b/packages/context/src/ingest/adapters/dbt-extraction-golden-parity.test.ts deleted file mode 100644 index ee96377c..00000000 --- a/packages/context/src/ingest/adapters/dbt-extraction-golden-parity.test.ts +++ /dev/null @@ -1,410 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { type DbtHostTableLite, matchDbtTables } from './dbt-descriptions/match-tables.js'; -import { mergeSemanticModelTables } from './dbt-descriptions/merge-semantic-model-tables.js'; -import { parseDbtSchemaFiles } from './dbt-descriptions/parse-schema.js'; -import { toDescriptionUpdates } from './dbt-descriptions/to-description-updates.js'; -import { toRelationshipUpdates } from './dbt-descriptions/to-relationship-updates.js'; -import { parseMetricflowFiles } from './metricflow/deep-parse.js'; -import { mapCrossModelMetricToSource, mapSemanticModelToSource } from './metricflow/semantic-models.js'; - -const DBT_SYSTEM_EMAIL = ['system@kae', 'lio.dev'].join(''); - -const metricflowYaml = ` -semantic_models: - - name: orders_semantic - description: MetricFlow order facts - model: ref('fct_orders') - defaults: - agg_time_dimension: ordered_at - entities: - - name: customer - type: foreign - expr: customer_id - description: Customer relationship - dimensions: - - name: status - type: categorical - expr: status - description: Order status - - name: ordered_at - type: time - expr: ordered_at - measures: - - name: total_revenue - agg: sum - expr: amount - description: Revenue - - name: customers_semantic - description: Customer dimension - model: ref('dim_customers') - entities: - - name: customer - type: primary - expr: id - dimensions: - - name: country - type: categorical - expr: country - description: Customer country - measures: - - name: customer_count - agg: count - expr: id - description: Customer count -metrics: - - name: total_revenue - type: simple - type_params: - measure: total_revenue - - name: customer_count - type: simple - type_params: - measure: customer_count - - name: revenue_per_customer - description: Revenue per customer - type: derived - type_params: - expr: total_revenue / NULLIF(customer_count, 0) - metrics: - - name: total_revenue - alias: total_revenue - - name: customer_count - alias: customer_count -`; - -const schemaYaml = ` -version: 2 -sources: - - name: raw - database: warehouse - schema: landing - tables: - - name: customers - identifier: dim_customers - description: Raw customer dimension - columns: - - name: id - description: Customer primary key - - name: country - description: Country name -models: - - name: "{{ var('orders_model', 'fct_orders') }}" - schema: "{{ var('mart_schema', 'analytics') }}" - description: Modeled orders - columns: - - name: customer_id - description: Linked customer id - tests: - - relationships: - to: ref('dim_customers') - field: id - - name: status - description: Order status - - name: amount - description: Gross amount -`; - -const hostTables: DbtHostTableLite[] = [ - { - id: 'orders-table', - name: 'fct_orders', - catalog: 'warehouse', - db: 'analytics', - columns: [ - { id: 'orders-customer-id', name: 'customer_id' }, - { id: 'orders-status', name: 'status' }, - { id: 'orders-amount', name: 'amount' }, - { id: 'orders-ordered-at', name: 'ordered_at' }, - ], - }, - { - id: 'customers-table', - name: 'dim_customers', - catalog: 'warehouse', - db: 'landing', - columns: [ - { id: 'customers-id', name: 'id' }, - { id: 'customers-country', name: 'country' }, - ], - }, -]; - -describe('dbt extraction golden parity fixture', () => { - it('freezes the relocated MetricFlow and dbt-description contract together', () => { - const metricflow = parseMetricflowFiles([{ path: 'semantic_models/orders.yml', content: metricflowYaml }]); - - expect(metricflow).toEqual({ - semanticModels: [ - { - name: 'orders_semantic', - description: 'MetricFlow order facts', - modelRef: 'fct_orders', - dimensions: [ - { - name: 'status', - column: 'status', - type: 'string', - label: 'Status', - description: 'Order status', - }, - { - name: 'ordered_at', - column: 'ordered_at', - type: 'time', - label: 'Ordered At', - description: undefined, - }, - ], - measures: [ - { - type: 'simple', - name: 'total_revenue', - column: 'amount', - aggregation: 'sum', - label: 'Total Revenue', - description: 'Revenue', - }, - ], - entities: [{ name: 'customer', type: 'foreign', expr: 'customer_id', description: 'Customer relationship' }], - defaultTimeDimension: 'ordered_at', - }, - { - name: 'customers_semantic', - description: 'Customer dimension', - modelRef: 'dim_customers', - dimensions: [ - { - name: 'country', - column: 'country', - type: 'string', - label: 'Country', - description: 'Customer country', - }, - ], - measures: [ - { - type: 'simple', - name: 'customer_count', - column: 'id', - aggregation: 'count', - label: 'Customer Count', - description: 'Customer count', - }, - ], - entities: [{ name: 'customer', type: 'primary', expr: 'id' }], - defaultTimeDimension: null, - }, - ], - crossModelMetrics: [ - { - name: 'revenue_per_customer', - label: null, - description: 'Revenue per customer', - type: 'derived', - expr: 'total_revenue / NULLIF(customer_count, 0)', - dependsOn: [ - { metricName: 'orders_semantic', alias: 'total_revenue' }, - { metricName: 'customers_semantic', alias: 'customer_count' }, - ], - filter: null, - }, - ], - relationships: [ - { - fromTable: 'fct_orders', - fromColumn: 'customer_id', - toTable: 'dim_customers', - toColumn: 'id', - description: 'Customer relationship', - }, - ], - warnings: [], - }); - - expect(mapSemanticModelToSource(metricflow.semanticModels[0], 'analytics.fct_orders')).toEqual({ - name: 'fct-orders', - table: 'analytics.fct_orders', - grain: ['status', 'ordered_at'], - columns: [ - { name: 'status', type: 'string', description: 'Order status' }, - { name: 'ordered_at', type: 'time' }, - ], - measures: [ - { - name: 'total_revenue', - expr: 'sum(amount)', - description: 'Revenue', - }, - ], - joins: [], - descriptions: { dbt: 'MetricFlow order facts' }, - }); - - expect(mapCrossModelMetricToSource(metricflow.crossModelMetrics[0])).toEqual({ - name: 'revenue-per-customer', - sql: 'total_revenue / NULLIF(customer_count, 0)', - descriptions: { dbt: 'Revenue per customer' }, - grain: [], - columns: [], - measures: [ - { - name: 'revenue_per_customer', - expr: 'total_revenue / NULLIF(customer_count, 0)', - description: 'Revenue per customer', - }, - ], - joins: [], - }); - - const schema = parseDbtSchemaFiles( - [{ path: 'models/schema.yml', content: schemaYaml }], - new Map([ - ['orders_model', 'fct_orders'], - ['mart_schema', 'analytics'], - ]), - ); - const merged = mergeSemanticModelTables(schema, metricflow.semanticModels); - - expect(merged).toEqual({ - projectName: null, - dbtVersion: null, - tables: [ - { - name: 'dim_customers', - description: 'Raw customer dimension', - database: 'warehouse', - schema: 'landing', - columns: [ - { name: 'id', description: 'Customer primary key', dataType: null }, - { name: 'country', description: 'Country name', dataType: null }, - ], - resourceType: 'source', - }, - { - name: 'fct_orders', - description: 'Modeled orders', - database: null, - schema: 'analytics', - columns: [ - { - name: 'customer_id', - description: 'Linked customer id', - dataType: null, - dataTests: [ - { - name: 'relationships', - package: 'dbt', - kwargs: { to: "ref('dim_customers')", field: 'id' }, - }, - ], - }, - { name: 'status', description: 'Order status', dataType: null }, - { name: 'amount', description: 'Gross amount', dataType: null }, - ], - resourceType: 'model', - }, - ], - relationships: [ - { - fromTable: 'fct_orders', - fromColumn: 'customer_id', - toTable: 'dim_customers', - toColumn: 'id', - fromSchema: 'analytics', - }, - ], - }); - - expect(matchDbtTables(merged.tables, hostTables, 'analytics')).toEqual([ - { - dbtTable: 'dim_customers', - dbtSchema: 'landing', - dbtDatabase: 'warehouse', - hostTableId: 'customers-table', - hostTableName: 'dim_customers', - matched: true, - tableDescriptionAction: 'import', - tableDescriptionFound: true, - columnsToImport: 2, - columnsMatched: 2, - columnsTotal: 2, - columnDescriptionsFound: 2, - }, - { - dbtTable: 'fct_orders', - dbtSchema: 'analytics', - dbtDatabase: null, - hostTableId: 'orders-table', - hostTableName: 'fct_orders', - matched: true, - tableDescriptionAction: 'import', - tableDescriptionFound: true, - columnsToImport: 3, - columnsMatched: 3, - columnsTotal: 3, - columnDescriptionsFound: 3, - }, - ]); - - expect( - toDescriptionUpdates({ - connectionId: 'warehouse-1', - parseResult: merged, - hostTables, - targetSchema: 'analytics', - }), - ).toEqual({ - dbt: [ - { - connectionId: 'warehouse-1', - table: { catalog: 'warehouse', db: 'landing', name: 'dim_customers' }, - source: 'dbt', - tableDescription: 'Raw customer dimension', - columnDescriptions: { - id: 'Customer primary key', - country: 'Country name', - }, - }, - { - connectionId: 'warehouse-1', - table: { catalog: 'warehouse', db: 'analytics', name: 'fct_orders' }, - source: 'dbt', - tableDescription: 'Modeled orders', - columnDescriptions: { - customer_id: 'Linked customer id', - status: 'Order status', - amount: 'Gross amount', - }, - }, - ], - aiInvalidations: [ - { - connectionId: 'warehouse-1', - table: { catalog: 'warehouse', db: 'landing', name: 'dim_customers' }, - source: 'ai', - }, - { - connectionId: 'warehouse-1', - table: { catalog: 'warehouse', db: 'analytics', name: 'fct_orders' }, - source: 'ai', - }, - ], - }); - - expect(toRelationshipUpdates({ connectionId: 'warehouse-1', parseResult: merged, hostTables })).toEqual({ - joins: [ - { - connectionId: 'warehouse-1', - fromTable: 'fct_orders', - fromColumns: ['customer_id'], - toTable: 'dim_customers', - toColumns: ['id'], - relationship: 'many_to_one', - author: 'dbt', - authorEmail: DBT_SYSTEM_EMAIL, - }, - ], - skippedNoMatch: 0, - }); - }); -}); diff --git a/packages/context/src/ingest/adapters/live-database/extracted-schema.test.ts b/packages/context/src/ingest/adapters/live-database/extracted-schema.test.ts deleted file mode 100644 index 69bd062d..00000000 --- a/packages/context/src/ingest/adapters/live-database/extracted-schema.test.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { KtxSchemaSnapshot } from '../../../scan/types.js'; -import { buildLiveDatabaseTableNaturalKey, ktxSchemaSnapshotToExtractedSchema } from './extracted-schema.js'; - -function snapshot(): KtxSchemaSnapshot { - return { - connectionId: 'conn-1', - driver: 'postgres', - extractedAt: '2026-04-27T00:00:00.000Z', - scope: { schemas: ['public'] }, - metadata: { driver: 'postgres' }, - tables: [ - { - name: 'orders', - catalog: null, - db: 'public', - kind: 'table', - comment: 'Orders placed by customers', - estimatedRows: null, - columns: [ - { - name: 'id', - nativeType: 'integer', - normalizedType: 'integer', - dimensionType: 'number', - nullable: false, - primaryKey: true, - comment: 'Primary key', - }, - { - name: 'customer_id', - nativeType: 'integer', - normalizedType: 'integer', - dimensionType: 'number', - nullable: false, - primaryKey: false, - comment: null, - }, - ], - foreignKeys: [ - { - fromColumn: 'customer_id', - toCatalog: null, - toDb: 'public', - toTable: 'customers', - toColumn: 'id', - constraintName: 'orders_customer_id_fkey', - }, - ], - }, - { - name: 'customers', - catalog: null, - db: 'public', - kind: 'table', - comment: null, - estimatedRows: null, - columns: [ - { - name: 'id', - nativeType: 'integer', - normalizedType: 'integer', - dimensionType: 'number', - nullable: false, - primaryKey: true, - comment: null, - }, - ], - foreignKeys: [], - }, - ], - }; -} - -describe('ktxSchemaSnapshotToExtractedSchema', () => { - it('preserves structural table, column, comment, and key metadata', () => { - const extracted = ktxSchemaSnapshotToExtractedSchema(snapshot()); - - expect(extracted.tables).toEqual([ - { - name: 'orders', - catalog: null, - db: 'public', - dbComment: 'Orders placed by customers', - columns: [ - { - name: 'id', - type: 'integer', - nullable: false, - primaryKey: true, - dbComment: 'Primary key', - }, - { - name: 'customer_id', - type: 'integer', - nullable: false, - primaryKey: false, - dbComment: null, - }, - ], - foreignKeys: [ - { - fromTable: 'orders', - fromColumn: 'customer_id', - toTable: 'customers', - toColumn: 'id', - constraintName: 'orders_customer_id_fkey', - }, - ], - }, - { - name: 'customers', - catalog: null, - db: 'public', - dbComment: null, - columns: [ - { - name: 'id', - type: 'integer', - nullable: false, - primaryKey: true, - dbComment: null, - }, - ], - foreignKeys: [], - }, - ]); - }); - - it('builds the same natural key shape used by schema sync', () => { - expect(buildLiveDatabaseTableNaturalKey({ catalog: null, db: 'public', name: 'orders' })).toBe('|public|orders'); - expect(buildLiveDatabaseTableNaturalKey({ catalog: 'warehouse', db: 'analytics', name: 'events' })).toBe( - 'warehouse|analytics|events', - ); - }); -}); diff --git a/packages/context/src/ingest/adapters/live-database/extracted-schema.ts b/packages/context/src/ingest/adapters/live-database/extracted-schema.ts deleted file mode 100644 index 39950c69..00000000 --- a/packages/context/src/ingest/adapters/live-database/extracted-schema.ts +++ /dev/null @@ -1,61 +0,0 @@ -import type { KtxSchemaSnapshot, KtxSchemaTable } from '../../../scan/types.js'; - -export interface LiveDatabaseExtractedForeignKey { - fromTable: string; - fromColumn: string; - toTable: string; - toColumn: string; - constraintName?: string; -} - -export interface LiveDatabaseExtractedColumn { - name: string; - type: string; - nullable: boolean; - primaryKey: boolean; - dbComment: string | null; -} - -export interface LiveDatabaseExtractedTable { - name: string; - catalog: string | null; - db: string | null; - dbComment: string | null; - columns: LiveDatabaseExtractedColumn[]; - foreignKeys: LiveDatabaseExtractedForeignKey[]; -} - -export interface LiveDatabaseExtractedSchema { - connectionId?: string; - tables: LiveDatabaseExtractedTable[]; -} - -export function buildLiveDatabaseTableNaturalKey(table: Pick): string { - return `${table.catalog ?? ''}|${table.db ?? ''}|${table.name}`; -} - -export function ktxSchemaSnapshotToExtractedSchema(snapshot: KtxSchemaSnapshot): LiveDatabaseExtractedSchema { - return { - connectionId: snapshot.connectionId, - tables: snapshot.tables.map((table) => ({ - name: table.name, - catalog: table.catalog ?? null, - db: table.db ?? null, - dbComment: table.comment ?? null, - columns: table.columns.map((column) => ({ - name: column.name, - type: column.nativeType, - nullable: column.nullable, - primaryKey: column.primaryKey, - dbComment: column.comment ?? null, - })), - foreignKeys: table.foreignKeys.map((foreignKey) => ({ - fromTable: table.name, - fromColumn: foreignKey.fromColumn, - toTable: foreignKey.toTable, - toColumn: foreignKey.toColumn, - ...(foreignKey.constraintName ? { constraintName: foreignKey.constraintName } : {}), - })), - })), - }; -} diff --git a/packages/context/src/ingest/adapters/live-database/structural-sync.test.ts b/packages/context/src/ingest/adapters/live-database/structural-sync.test.ts deleted file mode 100644 index 1df5faf8..00000000 --- a/packages/context/src/ingest/adapters/live-database/structural-sync.test.ts +++ /dev/null @@ -1,428 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { type LiveDatabaseSyncedSchema, planLiveDatabaseStructuralSync } from './structural-sync.js'; - -function idFactory(): () => string { - let next = 1; - return () => `id-${next++}`; -} - -describe('planLiveDatabaseStructuralSync', () => { - it('plans table and column creates, updates, deletes, and metadata invalidation', () => { - const current: LiveDatabaseSyncedSchema = { - connectionId: 'conn-1', - tables: [ - { - id: 'tbl-orders', - name: 'orders', - catalog: null, - db: 'public', - enabled: true, - descriptions: { ai: 'Old AI order text', db: 'Old DB order text' }, - columns: [ - { - id: 'col-order-id', - name: 'id', - type: 'number', - nullable: false, - primaryKey: true, - parentColumnId: null, - descriptions: { db: 'Order id' }, - embedding: [1, 2, 3], - sampleValues: null, - cardinality: null, - }, - { - id: 'col-order-total', - name: 'total', - type: 'number', - nullable: true, - primaryKey: false, - parentColumnId: null, - descriptions: { ai: 'Old AI total text', db: 'Old total text' }, - embedding: [4, 5, 6], - sampleValues: ['10'], - cardinality: 12, - }, - { - id: 'col-order-removed', - name: 'removed', - type: 'string', - nullable: true, - primaryKey: false, - parentColumnId: null, - descriptions: {}, - embedding: null, - sampleValues: null, - cardinality: null, - }, - ], - }, - { - id: 'tbl-removed', - name: 'removed_table', - catalog: null, - db: 'public', - enabled: true, - descriptions: {}, - columns: [ - { - id: 'col-removed-id', - name: 'id', - type: 'number', - nullable: false, - primaryKey: true, - parentColumnId: null, - descriptions: {}, - embedding: null, - sampleValues: null, - cardinality: null, - }, - ], - }, - ], - links: [ - { - id: 'inferred-total-link', - fromTableId: 'tbl-orders', - fromColumnId: 'col-order-total', - toTableId: 'tbl-orders', - toColumnId: 'col-order-id', - source: 'inferred', - confidence: 0.7, - relationshipType: 'MANY_TO_ONE', - isPrimaryKeyReference: true, - }, - ], - }; - - const plan = planLiveDatabaseStructuralSync({ - connectionId: 'conn-1', - current, - extracted: { - connectionId: 'conn-1', - tables: [ - { - name: 'orders', - catalog: null, - db: 'public', - dbComment: 'Fresh DB order text', - columns: [ - { - name: 'id', - type: 'number', - nullable: false, - primaryKey: true, - dbComment: 'Order id', - }, - { - name: 'total', - type: 'string', - nullable: false, - primaryKey: false, - dbComment: 'Fresh total text', - }, - { - name: 'created_at', - type: 'time', - nullable: false, - primaryKey: false, - dbComment: 'Creation timestamp', - }, - ], - foreignKeys: [], - }, - { - name: 'customers', - catalog: null, - db: 'public', - dbComment: 'Customer table', - columns: [ - { - name: 'id', - type: 'number', - nullable: false, - primaryKey: true, - dbComment: null, - }, - ], - foreignKeys: [], - }, - ], - }, - idFactory: idFactory(), - }); - - expect(plan.stats).toEqual({ - tablesCreated: 1, - tablesDeleted: 1, - columnsCreated: 2, - columnsDeleted: 2, - columnsModified: 1, - formalLinksCreated: 0, - formalLinksDeleted: 0, - }); - expect(plan.operations.deleteTableIds).toEqual(['tbl-removed']); - expect(plan.operations.deleteColumnIds).toEqual(['col-order-removed']); - expect(plan.operations.insertTables).toEqual([ - { - id: 'id-2', - connectionId: 'conn-1', - name: 'customers', - catalog: null, - db: 'public', - enabled: true, - }, - ]); - expect(plan.operations.insertColumns).toEqual([ - { - id: 'id-1', - tableId: 'tbl-orders', - name: 'created_at', - parentColumnId: null, - }, - { - id: 'id-3', - tableId: 'id-2', - name: 'id', - parentColumnId: null, - }, - ]); - expect(plan.operations.touchColumnIds).toEqual(['col-order-total']); - expect(plan.operations.invalidateColumnEmbeddingIds).toEqual(['col-order-total']); - expect(plan.inferredLinksToValidate).toEqual(['inferred-total-link']); - expect(plan.changes).toEqual({ - newTableIds: ['id-2'], - newColumnIds: ['id-1', 'id-3'], - tablesWithStructuralChanges: ['tbl-orders', 'id-2'], - columnsWithTypeChange: ['col-order-total'], - columnsWithDescriptionChange: ['col-order-total'], - tablesWithDescriptionChange: ['tbl-orders'], - }); - - const orders = plan.schema.tables.find((table) => table.name === 'orders'); - expect(orders?.descriptions).toEqual({ db: 'Fresh DB order text' }); - expect(orders?.columns.map((column) => column.name)).toEqual(['id', 'total', 'created_at']); - expect(orders?.columns.find((column) => column.name === 'total')).toMatchObject({ - id: 'col-order-total', - type: 'string', - nullable: false, - primaryKey: false, - descriptions: { db: 'Fresh total text' }, - embedding: null, - sampleValues: ['10'], - cardinality: 12, - }); - }); - - it('builds formal links from extracted foreign keys and preserves valid inferred links', () => { - const current: LiveDatabaseSyncedSchema = { - connectionId: 'conn-1', - tables: [ - { - id: 'tbl-orders', - name: 'orders', - catalog: null, - db: 'public', - enabled: true, - descriptions: {}, - columns: [ - { - id: 'col-orders-id', - name: 'id', - type: 'number', - nullable: false, - primaryKey: true, - parentColumnId: null, - descriptions: {}, - embedding: null, - sampleValues: null, - cardinality: null, - }, - { - id: 'col-orders-customer', - name: 'customer_id', - type: 'number', - nullable: false, - primaryKey: false, - parentColumnId: null, - descriptions: {}, - embedding: null, - sampleValues: null, - cardinality: null, - }, - ], - }, - { - id: 'tbl-customers', - name: 'customers', - catalog: null, - db: 'public', - enabled: true, - descriptions: {}, - columns: [ - { - id: 'col-customers-id', - name: 'id', - type: 'number', - nullable: false, - primaryKey: true, - parentColumnId: null, - descriptions: {}, - embedding: null, - sampleValues: null, - cardinality: null, - }, - ], - }, - ], - links: [ - { - id: 'formal-existing', - fromTableId: 'tbl-orders', - fromColumnId: 'col-orders-customer', - toTableId: 'tbl-customers', - toColumnId: 'col-customers-id', - source: 'formal', - confidence: 1, - relationshipType: 'MANY_TO_ONE', - isPrimaryKeyReference: true, - }, - { - id: 'inferred-existing', - fromTableId: 'tbl-orders', - fromColumnId: 'col-orders-id', - toTableId: 'tbl-customers', - toColumnId: 'col-customers-id', - source: 'inferred', - confidence: 0.6, - relationshipType: 'MANY_TO_ONE', - isPrimaryKeyReference: true, - }, - ], - }; - - const plan = planLiveDatabaseStructuralSync({ - connectionId: 'conn-1', - current, - extracted: { - connectionId: 'conn-1', - tables: [ - { - name: 'orders', - catalog: null, - db: 'public', - dbComment: null, - columns: [ - { name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }, - { name: 'customer_id', type: 'number', nullable: false, primaryKey: false, dbComment: null }, - ], - foreignKeys: [ - { - fromTable: 'orders', - fromColumn: 'customer_id', - toTable: 'customers', - toColumn: 'id', - }, - ], - }, - { - name: 'customers', - catalog: null, - db: 'public', - dbComment: null, - columns: [{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }], - foreignKeys: [], - }, - ], - }, - idFactory: idFactory(), - }); - - expect(plan.stats.formalLinksCreated).toBe(0); - expect(plan.stats.formalLinksDeleted).toBe(0); - expect(plan.schema.links.map((link) => link.id)).toEqual(['formal-existing', 'inferred-existing']); - - const planAfterForeignKeyRemoval = planLiveDatabaseStructuralSync({ - connectionId: 'conn-1', - current, - extracted: { - connectionId: 'conn-1', - tables: [ - { - name: 'orders', - catalog: null, - db: 'public', - dbComment: null, - columns: [ - { name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }, - { name: 'customer_id', type: 'number', nullable: false, primaryKey: false, dbComment: null }, - ], - foreignKeys: [], - }, - { - name: 'customers', - catalog: null, - db: 'public', - dbComment: null, - columns: [{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }], - foreignKeys: [], - }, - ], - }, - idFactory: idFactory(), - }); - - expect(planAfterForeignKeyRemoval.stats.formalLinksDeleted).toBe(1); - expect(planAfterForeignKeyRemoval.schema.links.map((link) => link.id)).toEqual(['inferred-existing']); - - const planAfterForeignKeyCreation = planLiveDatabaseStructuralSync({ - connectionId: 'conn-1', - current: { ...current, links: [current.links[1]] }, - extracted: { - connectionId: 'conn-1', - tables: [ - { - name: 'orders', - catalog: null, - db: 'public', - dbComment: null, - columns: [ - { name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }, - { name: 'customer_id', type: 'number', nullable: false, primaryKey: false, dbComment: null }, - ], - foreignKeys: [ - { - fromTable: 'orders', - fromColumn: 'customer_id', - toTable: 'customers', - toColumn: 'id', - }, - ], - }, - { - name: 'customers', - catalog: null, - db: 'public', - dbComment: null, - columns: [{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }], - foreignKeys: [], - }, - ], - }, - idFactory: idFactory(), - }); - - expect(planAfterForeignKeyCreation.stats.formalLinksCreated).toBe(1); - expect(planAfterForeignKeyCreation.schema.links[0]).toMatchObject({ - id: 'id-1', - fromTableId: 'tbl-orders', - fromColumnId: 'col-orders-customer', - toTableId: 'tbl-customers', - toColumnId: 'col-customers-id', - source: 'formal', - confidence: 1, - relationshipType: 'MANY_TO_ONE', - isPrimaryKeyReference: true, - }); - }); -}); diff --git a/packages/context/src/ingest/adapters/live-database/structural-sync.ts b/packages/context/src/ingest/adapters/live-database/structural-sync.ts deleted file mode 100644 index d98a692b..00000000 --- a/packages/context/src/ingest/adapters/live-database/structural-sync.ts +++ /dev/null @@ -1,525 +0,0 @@ -import type { LiveDatabaseExtractedSchema, LiveDatabaseExtractedTable } from './extracted-schema.js'; -import { buildLiveDatabaseTableNaturalKey } from './extracted-schema.js'; - -export interface LiveDatabaseSyncedColumn { - id: string; - name: string; - type: string; - nullable: boolean; - primaryKey: boolean; - parentColumnId: string | null; - descriptions: Record; - embedding: number[] | null; - sampleValues: string[] | null; - cardinality: number | null; -} - -export interface LiveDatabaseSyncedTable { - id: string; - name: string; - catalog: string | null; - db: string | null; - enabled: boolean; - descriptions: Record; - columns: LiveDatabaseSyncedColumn[]; -} - -export interface LiveDatabaseSyncedLink { - id: string; - fromTableId: string; - fromColumnId: string; - toTableId: string; - toColumnId: string; - source: 'formal' | 'inferred' | 'manual'; - confidence: number; - relationshipType: string; - isPrimaryKeyReference: boolean; -} - -export interface LiveDatabaseSyncedSchema { - connectionId: string; - tables: LiveDatabaseSyncedTable[]; - links: LiveDatabaseSyncedLink[]; -} - -export interface LiveDatabaseStructuralChanges { - newTableIds: string[]; - newColumnIds: string[]; - tablesWithStructuralChanges: string[]; - columnsWithTypeChange: string[]; - columnsWithDescriptionChange: string[]; - tablesWithDescriptionChange: string[]; -} - -export interface LiveDatabaseStructuralSyncStats { - tablesCreated: number; - tablesDeleted: number; - columnsCreated: number; - columnsDeleted: number; - columnsModified: number; - formalLinksCreated: number; - formalLinksDeleted: number; -} - -export interface LiveDatabaseStructuralSyncOperations { - deleteTableIds: string[]; - deleteColumnIds: string[]; - insertTables: Array<{ - id: string; - connectionId: string; - name: string; - catalog: string | null; - db: string | null; - enabled: boolean; - }>; - insertColumns: Array<{ - id: string; - tableId: string; - name: string; - parentColumnId: string | null; - }>; - touchColumnIds: string[]; - invalidateColumnEmbeddingIds: string[]; -} - -export interface LiveDatabaseStructuralSyncPlan { - schema: LiveDatabaseSyncedSchema; - inferredLinksToValidate: string[]; - stats: LiveDatabaseStructuralSyncStats; - changes: LiveDatabaseStructuralChanges; - operations: LiveDatabaseStructuralSyncOperations; -} - -export interface PlanLiveDatabaseStructuralSyncInput { - connectionId: string; - current: LiveDatabaseSyncedSchema | null; - extracted: LiveDatabaseExtractedSchema; - idFactory: () => string; -} - -interface UpdatedTableResult { - table: LiveDatabaseSyncedTable; - columnsCreated: number; - columnsDeleted: number; - columnsModified: number; - newColumnIds: string[]; - columnsWithTypeChange: string[]; - columnsWithDescriptionChange: string[]; - tableDescriptionChanged: boolean; -} - -function updateDescription( - descriptions: Record, - dbComment: string | null | undefined, - changed: boolean, -): Record { - const updated = { ...descriptions }; - if (dbComment) { - updated.db = dbComment; - } else { - delete updated.db; - } - if (changed) { - delete updated.ai; - } - return updated; -} - -function descriptionFromDbComment(dbComment: string | null | undefined): Record { - return dbComment ? { db: dbComment } : {}; -} - -function planUpdatedTable(args: { - currentTable: LiveDatabaseSyncedTable; - extractedTable: LiveDatabaseExtractedTable; - currentLinks: LiveDatabaseSyncedLink[]; - inferredLinksToValidate: string[]; - operations: LiveDatabaseStructuralSyncOperations; - idFactory: () => string; -}): UpdatedTableResult { - const { currentTable, extractedTable, currentLinks, inferredLinksToValidate, operations, idFactory } = args; - - let columnsCreated = 0; - let columnsDeleted = 0; - let columnsModified = 0; - const newColumnIds: string[] = []; - const columnsWithTypeChange: string[] = []; - const columnsWithDescriptionChange: string[] = []; - const updatedColumns: LiveDatabaseSyncedColumn[] = []; - - const tableDescriptionChanged = (currentTable.descriptions.db ?? null) !== (extractedTable.dbComment ?? null); - const currentColumnsByName = new Map(currentTable.columns.map((column) => [column.name, column])); - const extractedColumnsByName = new Map(extractedTable.columns.map((column) => [column.name, column])); - - for (const [name, currentColumn] of currentColumnsByName) { - if (!extractedColumnsByName.has(name)) { - operations.deleteColumnIds.push(currentColumn.id); - columnsDeleted++; - } - } - - for (const [name, extractedColumn] of extractedColumnsByName) { - const currentColumn = currentColumnsByName.get(name); - if (!currentColumn) { - const columnId = idFactory(); - operations.insertColumns.push({ - id: columnId, - tableId: currentTable.id, - name: extractedColumn.name, - parentColumnId: null, - }); - columnsCreated++; - newColumnIds.push(columnId); - updatedColumns.push({ - id: columnId, - name: extractedColumn.name, - type: extractedColumn.type, - nullable: extractedColumn.nullable, - primaryKey: extractedColumn.primaryKey, - descriptions: descriptionFromDbComment(extractedColumn.dbComment), - parentColumnId: null, - embedding: null, - sampleValues: null, - cardinality: null, - }); - continue; - } - - const typeChanged = currentColumn.type !== extractedColumn.type; - const nullableChanged = currentColumn.nullable !== extractedColumn.nullable; - const primaryKeyChanged = currentColumn.primaryKey !== extractedColumn.primaryKey; - const dbDescriptionChanged = (currentColumn.descriptions.db ?? null) !== (extractedColumn.dbComment ?? null); - - if (typeChanged || nullableChanged || primaryKeyChanged || dbDescriptionChanged) { - operations.touchColumnIds.push(currentColumn.id); - columnsModified++; - - if (typeChanged || dbDescriptionChanged) { - operations.invalidateColumnEmbeddingIds.push(currentColumn.id); - } - - if (typeChanged) { - columnsWithTypeChange.push(currentColumn.id); - const affectedLinks = currentLinks.filter( - (link) => - link.source === 'inferred' && - (link.fromColumnId === currentColumn.id || link.toColumnId === currentColumn.id), - ); - for (const link of affectedLinks) { - if (!inferredLinksToValidate.includes(link.id)) { - inferredLinksToValidate.push(link.id); - } - } - } - - if (dbDescriptionChanged) { - columnsWithDescriptionChange.push(currentColumn.id); - } - } - - updatedColumns.push({ - ...currentColumn, - type: extractedColumn.type, - nullable: extractedColumn.nullable, - primaryKey: extractedColumn.primaryKey, - descriptions: updateDescription(currentColumn.descriptions, extractedColumn.dbComment, dbDescriptionChanged), - embedding: typeChanged ? null : currentColumn.embedding, - }); - } - - return { - table: { - ...currentTable, - descriptions: updateDescription(currentTable.descriptions, extractedTable.dbComment, tableDescriptionChanged), - columns: updatedColumns, - }, - columnsCreated, - columnsDeleted, - columnsModified, - newColumnIds, - columnsWithTypeChange, - columnsWithDescriptionChange, - tableDescriptionChanged, - }; -} - -function planCreatedTable(args: { - connectionId: string; - extractedTable: LiveDatabaseExtractedTable; - operations: LiveDatabaseStructuralSyncOperations; - idFactory: () => string; -}): LiveDatabaseSyncedTable { - const { connectionId, extractedTable, operations, idFactory } = args; - const tableId = idFactory(); - operations.insertTables.push({ - id: tableId, - connectionId, - name: extractedTable.name, - catalog: extractedTable.catalog, - db: extractedTable.db, - enabled: true, - }); - - const columns: LiveDatabaseSyncedColumn[] = extractedTable.columns.map((extractedColumn) => { - const columnId = idFactory(); - operations.insertColumns.push({ - id: columnId, - tableId, - name: extractedColumn.name, - parentColumnId: null, - }); - return { - id: columnId, - name: extractedColumn.name, - type: extractedColumn.type, - nullable: extractedColumn.nullable, - primaryKey: extractedColumn.primaryKey, - descriptions: descriptionFromDbComment(extractedColumn.dbComment), - parentColumnId: null, - embedding: null, - sampleValues: null, - cardinality: null, - }; - }); - - return { - id: tableId, - name: extractedTable.name, - catalog: extractedTable.catalog, - db: extractedTable.db, - enabled: true, - descriptions: descriptionFromDbComment(extractedTable.dbComment), - columns, - }; -} - -function syncFormalLinks(args: { - extracted: LiveDatabaseExtractedSchema; - tables: LiveDatabaseSyncedTable[]; - tableNaturalKeyToId: Map; - currentLinks: LiveDatabaseSyncedLink[]; - idFactory: () => string; -}): { links: LiveDatabaseSyncedLink[]; created: number; deleted: number } { - const { extracted, tables, tableNaturalKeyToId, currentLinks, idFactory } = args; - const columnKeyToId = new Map(); - - for (const table of tables) { - const tableKey = buildLiveDatabaseTableNaturalKey(table); - for (const column of table.columns) { - columnKeyToId.set(`${tableKey}.${column.name}`, column.id); - } - } - - const extractedFormalLinks: Array<{ - fromTableId: string; - fromColumnId: string; - toTableId: string; - toColumnId: string; - }> = []; - - for (const table of extracted.tables) { - const fromTableKey = buildLiveDatabaseTableNaturalKey(table); - const fromTableId = tableNaturalKeyToId.get(fromTableKey); - if (!fromTableId) { - continue; - } - - for (const foreignKey of table.foreignKeys) { - const toTableKey = buildLiveDatabaseTableNaturalKey({ - catalog: table.catalog, - db: table.db, - name: foreignKey.toTable, - }); - const toTableId = tableNaturalKeyToId.get(toTableKey); - if (!toTableId) { - continue; - } - - const fromColumnId = columnKeyToId.get(`${fromTableKey}.${foreignKey.fromColumn}`); - const toColumnId = columnKeyToId.get(`${toTableKey}.${foreignKey.toColumn}`); - if (!fromColumnId || !toColumnId) { - continue; - } - - extractedFormalLinks.push({ fromTableId, fromColumnId, toTableId, toColumnId }); - } - } - - const currentFormalLinks = currentLinks.filter((link) => link.source === 'formal'); - const extractedLinkKeys = new Set(extractedFormalLinks.map((link) => `${link.fromColumnId}->${link.toColumnId}`)); - const linksToDelete = currentFormalLinks.filter( - (link) => !extractedLinkKeys.has(`${link.fromColumnId}->${link.toColumnId}`), - ); - - const currentLinkKeys = new Set(currentFormalLinks.map((link) => `${link.fromColumnId}->${link.toColumnId}`)); - const linksToCreate = extractedFormalLinks.filter( - (link) => !currentLinkKeys.has(`${link.fromColumnId}->${link.toColumnId}`), - ); - - const newLinks = linksToCreate.map((linkData) => ({ - id: idFactory(), - fromTableId: linkData.fromTableId, - fromColumnId: linkData.fromColumnId, - toTableId: linkData.toTableId, - toColumnId: linkData.toColumnId, - source: 'formal' as const, - confidence: 1, - relationshipType: 'MANY_TO_ONE', - isPrimaryKeyReference: true, - })); - - const deletedLinkIds = new Set(linksToDelete.map((link) => link.id)); - const preservedFormalLinks = currentFormalLinks.filter((link) => !deletedLinkIds.has(link.id)); - - return { - links: [...preservedFormalLinks, ...newLinks], - created: linksToCreate.length, - deleted: linksToDelete.length, - }; -} - -export function planLiveDatabaseStructuralSync( - input: PlanLiveDatabaseStructuralSyncInput, -): LiveDatabaseStructuralSyncPlan { - const operations: LiveDatabaseStructuralSyncOperations = { - deleteTableIds: [], - deleteColumnIds: [], - insertTables: [], - insertColumns: [], - touchColumnIds: [], - invalidateColumnEmbeddingIds: [], - }; - const stats: LiveDatabaseStructuralSyncStats = { - tablesCreated: 0, - tablesDeleted: 0, - columnsCreated: 0, - columnsDeleted: 0, - columnsModified: 0, - formalLinksCreated: 0, - formalLinksDeleted: 0, - }; - const changes: LiveDatabaseStructuralChanges = { - newTableIds: [], - newColumnIds: [], - tablesWithStructuralChanges: [], - columnsWithTypeChange: [], - columnsWithDescriptionChange: [], - tablesWithDescriptionChange: [], - }; - const inferredLinksToValidate: string[] = []; - - const currentTablesByKey = new Map(); - const extractedTablesByKey = new Map(); - - if (input.current) { - for (const table of input.current.tables) { - currentTablesByKey.set(buildLiveDatabaseTableNaturalKey(table), table); - } - } - for (const table of input.extracted.tables) { - extractedTablesByKey.set(buildLiveDatabaseTableNaturalKey(table), table); - } - - const tablesToDelete: LiveDatabaseSyncedTable[] = []; - const tablesToUpdate: Array<{ - current: LiveDatabaseSyncedTable; - extracted: LiveDatabaseExtractedTable; - }> = []; - const tablesToCreate: LiveDatabaseExtractedTable[] = []; - - for (const [key, table] of currentTablesByKey) { - const extractedTable = extractedTablesByKey.get(key); - if (!extractedTable) { - tablesToDelete.push(table); - } else { - tablesToUpdate.push({ current: table, extracted: extractedTable }); - } - } - - for (const [key, table] of extractedTablesByKey) { - if (!currentTablesByKey.has(key)) { - tablesToCreate.push(table); - } - } - - for (const table of tablesToDelete) { - operations.deleteTableIds.push(table.id); - stats.tablesDeleted++; - stats.columnsDeleted += table.columns.length; - } - - const updatedTables: LiveDatabaseSyncedTable[] = []; - for (const { current, extracted } of tablesToUpdate) { - const result = planUpdatedTable({ - currentTable: current, - extractedTable: extracted, - currentLinks: input.current?.links ?? [], - inferredLinksToValidate, - operations, - idFactory: input.idFactory, - }); - updatedTables.push(result.table); - stats.columnsCreated += result.columnsCreated; - stats.columnsDeleted += result.columnsDeleted; - stats.columnsModified += result.columnsModified; - changes.newColumnIds.push(...result.newColumnIds); - changes.columnsWithTypeChange.push(...result.columnsWithTypeChange); - changes.columnsWithDescriptionChange.push(...result.columnsWithDescriptionChange); - if (result.tableDescriptionChanged) { - changes.tablesWithDescriptionChange.push(current.id); - } - if (result.columnsCreated > 0 || result.columnsDeleted > 0 || result.columnsWithTypeChange.length > 0) { - changes.tablesWithStructuralChanges.push(current.id); - } - } - - const createdTables: LiveDatabaseSyncedTable[] = []; - for (const extractedTable of tablesToCreate) { - const table = planCreatedTable({ - connectionId: input.connectionId, - extractedTable, - operations, - idFactory: input.idFactory, - }); - createdTables.push(table); - stats.tablesCreated++; - stats.columnsCreated += table.columns.length; - changes.newTableIds.push(table.id); - changes.newColumnIds.push(...table.columns.map((column) => column.id)); - changes.tablesWithStructuralChanges.push(table.id); - } - - const allTables = [...updatedTables, ...createdTables]; - const tableNaturalKeyToId = new Map(); - for (const table of allTables) { - tableNaturalKeyToId.set(buildLiveDatabaseTableNaturalKey(table), table.id); - } - - const formalLinkResult = syncFormalLinks({ - extracted: input.extracted, - tables: allTables, - tableNaturalKeyToId, - currentLinks: input.current?.links ?? [], - idFactory: input.idFactory, - }); - stats.formalLinksCreated = formalLinkResult.created; - stats.formalLinksDeleted = formalLinkResult.deleted; - - const deletedTableIds = new Set(tablesToDelete.map((table) => table.id)); - const preservedInferredLinks = (input.current?.links ?? []).filter( - (link) => - link.source === 'inferred' && !deletedTableIds.has(link.fromTableId) && !deletedTableIds.has(link.toTableId), - ); - - return { - schema: { - connectionId: input.connectionId, - tables: allTables, - links: [...formalLinkResult.links, ...preservedInferredLinks], - }, - inferredLinksToValidate, - stats, - changes, - operations, - }; -} diff --git a/packages/context/src/ingest/adapters/looker/reconcile.test.ts b/packages/context/src/ingest/adapters/looker/reconcile.test.ts deleted file mode 100644 index 68e2cda8..00000000 --- a/packages/context/src/ingest/adapters/looker/reconcile.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { buildLookerReconcileNotes, lookerRuntimeSourceToFileAdapterSource } from './reconcile.js'; - -describe('lookerRuntimeSourceToFileAdapterSource', () => { - it('maps API-derived Looker source names to file-adapter source names', () => { - expect(lookerRuntimeSourceToFileAdapterSource('looker__b2b__sales_pipeline')).toBe('b2b__sales_pipeline'); - expect(lookerRuntimeSourceToFileAdapterSource('looker__finance__orders')).toBe('finance__orders'); - }); - - it('ignores non-Looker and malformed source names', () => { - expect(lookerRuntimeSourceToFileAdapterSource('b2b__sales_pipeline')).toBeNull(); - expect(lookerRuntimeSourceToFileAdapterSource('looker__missing_explore')).toBeNull(); - }); -}); - -describe('buildLookerReconcileNotes', () => { - it('instructs reconciliation to record subsumed provenance', () => { - expect(buildLookerReconcileNotes()).toEqual([ - [ - 'Looker runtime API-derived SL sources use looker____.', - 'If the unprefixed file-adapter source __ exists, prefer it in wiki sl_refs, delete or avoid the API-derived source, and call emit_artifact_resolution with actionType="subsumed" for the API raw explore path.', - ].join(' '), - ]); - }); -}); diff --git a/packages/context/src/ingest/adapters/looker/reconcile.ts b/packages/context/src/ingest/adapters/looker/reconcile.ts deleted file mode 100644 index fe5e74a5..00000000 --- a/packages/context/src/ingest/adapters/looker/reconcile.ts +++ /dev/null @@ -1,21 +0,0 @@ -export function lookerRuntimeSourceToFileAdapterSource(sourceName: string): string | null { - if (!sourceName.startsWith('looker__')) { - return null; - } - const stripped = sourceName.slice('looker__'.length); - const parts = stripped.split('__'); - if (parts.length < 2 || parts.some((part) => part.length === 0)) { - return null; - } - const [model, ...exploreParts] = parts; - return `${model}__${exploreParts.join('__')}`; -} - -export function buildLookerReconcileNotes(): string[] { - return [ - [ - 'Looker runtime API-derived SL sources use looker____.', - 'If the unprefixed file-adapter source __ exists, prefer it in wiki sl_refs, delete or avoid the API-derived source, and call emit_artifact_resolution with actionType="subsumed" for the API raw explore path.', - ].join(' '), - ]; -} diff --git a/packages/context/src/ingest/context-candidates/index.ts b/packages/context/src/ingest/context-candidates/index.ts deleted file mode 100644 index 7c8e0f78..00000000 --- a/packages/context/src/ingest/context-candidates/index.ts +++ /dev/null @@ -1,29 +0,0 @@ -export type { CandidateDedupServiceDeps } from './candidate-dedup.service.js'; -export { CandidateDedupService } from './candidate-dedup.service.js'; -export type { - ContextCandidateCarryforwardArgs, - ContextCandidateCarryforwardResult, - ContextCandidateCarryforwardServiceDeps, -} from './context-candidate-carryforward.service.js'; -export { ContextCandidateCarryforwardService } from './context-candidate-carryforward.service.js'; -export type { CuratorPaginationInput, CuratorPaginationServiceDeps } from './curator-pagination.service.js'; -export { CuratorPaginationService } from './curator-pagination.service.js'; -export { buildContextCandidateEmbeddingText } from './embedding-text.js'; -export type { ContextCandidateStorePort } from './store.js'; -export type { - BudgetExhaustedCandidateForCarryForward, - CandidateDedupSettings, - ContextCandidateActionHint, - ContextCandidateCarryforwardSettings, - ContextCandidateEmbeddingPort, - ContextCandidateForPrompt, - ContextCandidateLane, - ContextCandidateRejectionReason, - ContextCandidateScoreAggregation, - ContextCandidateStatus, - ContextCandidateVerdictSummary, - CuratorPaginationSettings, - CurrentRunEvidenceChunkForCarryForward, - InsertContextCandidateInput, - MarkContextCandidateClusterInput, -} from './types.js'; diff --git a/packages/context/src/ingest/context-evidence/index.ts b/packages/context/src/ingest/context-evidence/index.ts deleted file mode 100644 index 99be0177..00000000 --- a/packages/context/src/ingest/context-evidence/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -export { ContextEvidenceIndexService } from './context-evidence-index.service.js'; -export { SqliteContextEvidenceStore } from './sqlite-context-evidence-store.js'; -export type { - ContextEvidenceDocumentRef, - ContextEvidenceEmbeddingPort, - ContextEvidenceIndexSummary, - EvidencePublishState, - ReplaceContextEvidenceChunk, - UpsertContextEvidenceDocument, -} from './types.js'; -export type { ContextEvidenceIndexStorePort } from './store.js'; -export type { SqliteContextEvidenceStoreOptions } from './sqlite-context-evidence-store.js'; diff --git a/packages/context/src/ingest/index.ts b/packages/context/src/ingest/index.ts deleted file mode 100644 index d8e9c856..00000000 --- a/packages/context/src/ingest/index.ts +++ /dev/null @@ -1,670 +0,0 @@ -export { DbtSourceAdapter } from './adapters/dbt/dbt.adapter.js'; -export { FakeSourceAdapter } from './adapters/fake/fake.adapter.js'; -export type { - DaemonLiveDatabaseIntrospectionOptions, - KtxDaemonDatabaseHttpJsonRunner, - KtxDaemonDatabaseIntrospectionCommand, - KtxDaemonDatabaseJsonRunner, -} from './adapters/live-database/daemon-introspection.js'; -export { createDaemonLiveDatabaseIntrospection } from './adapters/live-database/daemon-introspection.js'; -export type { - LiveDatabaseExtractedColumn, - LiveDatabaseExtractedForeignKey, - LiveDatabaseExtractedSchema, - LiveDatabaseExtractedTable, -} from './adapters/live-database/extracted-schema.js'; -export { - buildLiveDatabaseTableNaturalKey, - ktxSchemaSnapshotToExtractedSchema, -} from './adapters/live-database/extracted-schema.js'; -export { - assertSemanticLayerTargetPathsAllowed, - findDisallowedSemanticLayerTargetPaths, - semanticLayerConnectionIdFromPath, -} from './semantic-layer-target-policy.js'; -export { LiveDatabaseSourceAdapter } from './adapters/live-database/live-database.adapter.js'; -export type { - BuildLiveDatabaseManifestShardsInput, - BuildLiveDatabaseManifestShardsResult, - LiveDatabaseManifestColumn, - LiveDatabaseManifestExistingDescriptions, - LiveDatabaseManifestJoinData, - LiveDatabaseManifestJoinEntry, - LiveDatabaseManifestShard, - LiveDatabaseManifestTableData, - LiveDatabaseManifestTableEntry, -} from './adapters/live-database/manifest.js'; -export { buildLiveDatabaseManifestShards } from './adapters/live-database/manifest.js'; -export type { - LiveDatabaseStructuralChanges, - LiveDatabaseStructuralSyncOperations, - LiveDatabaseStructuralSyncPlan, - LiveDatabaseStructuralSyncStats, - LiveDatabaseSyncedColumn, - LiveDatabaseSyncedLink, - LiveDatabaseSyncedSchema, - LiveDatabaseSyncedTable, - PlanLiveDatabaseStructuralSyncInput, -} from './adapters/live-database/structural-sync.js'; -export { planLiveDatabaseStructuralSync } from './adapters/live-database/structural-sync.js'; -export type { - LiveDatabaseIntrospectionPort, - LiveDatabaseSourceAdapterDeps, -} from './adapters/live-database/types.js'; -export { getLookerTriageSignals, writeLookerEvidenceDocuments } from './adapters/looker/evidence-documents.js'; -export { LookerClient } from './adapters/looker/client.js'; -export type { - LookerClientDeps, - LookerClientLogger, - LookerConnectionParams, - LookerSdkPort, - LookerWarehouseConnectionInfo, - TestConnectionResult as LookerTestConnectionResult, -} from './adapters/looker/client.js'; -export type { - LookerClientFactory, - LookerEntityRef, - LookerRuntimeClient, -} from './adapters/looker/fetch.js'; -export { - DefaultLookerClientFactory, - DefaultLookerConnectionClientFactory, -} from './adapters/looker/factory.js'; -export { - createDaemonLookerTableIdentifierParser, - type DaemonLookerTableIdentifierParserOptions, - type KtxDaemonTableIdentifierHttpJsonRunner, -} from './adapters/looker/daemon-table-identifier-parser.js'; -export type { - LookerConnectionClientFactory, - LookerCredentialResolver, -} from './adapters/looker/factory.js'; -export { - createLocalLookerCredentialResolver, - createLocalLookerSourceAdapter, - lookerCredentialsFromLocalConnection, -} from './adapters/looker/local-looker.adapter.js'; -export { - LocalLookerRuntimeStore, - type ClearLocalLookerMappingsInput, - type LocalLookerConnectionMappingListRow, - type LocalLookerMappingSource, - type LookerSourceStateReader, - type RefreshLocalLookerDiscoveredConnectionsInput, - type UpsertLocalLookerConnectionMappingInput, -} from './adapters/looker/local-runtime-store.js'; -export { - LOOKER_DIALECT_TO_CONNECTION_TYPE, - buildLookerPullConfigFromInputs, - collectExploreParseItems, - computeLookerMappingDrift, - discoverLookerConnections, - extractWarehouseDatabase, - extractWarehouseHost, - lookerDialectToConnectionType, - normalizeHost, - normalizeName, - projectParsedIdentifier, - refreshLookerMappingPlaceholders, - sqlglotDialectForConnectionType, - suggestKtxConnectionForLookerConnection, - validateLookerMappings, - validateLookerWarehouseTarget, -} from './adapters/looker/mapping.js'; -export type { - LookerConnectionMapping as KtxLookerConnectionMapping, - LookerMappingCandidateConnection, - LookerMappingClient, - LookerMappingDrift, - LookerMappingValidationResult, - LookerParsedIdentifier, - LookerTableIdentifierParseItem, - LookerTableIdentifierParser, - LookerTargetConnection, - LookerWarehouseTargetConnectionType, -} from './adapters/looker/mapping.js'; -export { - readLookerFetchReport, - writeLookerFetchReport, -} from './adapters/looker/fetch-report.js'; -export { LookerSourceAdapter, type LookerSourceAdapterDeps } from './adapters/looker/looker.adapter.js'; -export { - describeLookerScope, - hashLookerScope, - isPathInLookerScope, - readLookerScope, -} from './adapters/looker/scope.js'; -export type { - LookerQueryToSlInput, - LookerSlFieldProposal, - LookerSlMeasureProposal, - LookerSlProposal, - LookerSlSegmentProposal, -} from './adapters/looker/tools/looker-query-to-sl.tool.js'; -export { - buildLookerSlProposal, - createLookerQueryToSlTool, - formatLookerSlProposal, - lookerQueryToSlInputSchema, -} from './adapters/looker/tools/looker-query-to-sl.tool.js'; -export type { - LookerPullConfig, - LookerRuntimeCursors, - StagedDashboardFile, - StagedExploreFile, - StagedFoldersTreeFile, - StagedGroupFile, - StagedLookerFetchIssue, - StagedLookerFetchReport, - StagedLookerQuery, - StagedLookerScopeFile, - StagedLookerSignalsFile, - StagedLookFile, - StagedLookmlModelsFile, - StagedUserFile, -} from './adapters/looker/types.js'; -export { - lookerConnectionIdSchema, - lookerRuntimeCursorsSchema, - stagedLookerFetchIssueSchema, - stagedLookerFetchReportSchema, - stagedLookerScopeFileSchema, - stagedSyncConfigSchema, -} from './adapters/looker/types.js'; -export { LookmlSourceAdapter } from './adapters/lookml/lookml.adapter.js'; -export { parseLookmlStagedDir } from './adapters/lookml/parse.js'; -export type { ParsedLookmlProject } from './adapters/lookml/parse.js'; -export { - DEFAULT_METABASE_CLIENT_CONFIG, - DefaultMetabaseConnectionClientFactory, - MetabaseClient, - getDummyValueForWidgetType, - stripOptionalClauses, -} from './adapters/metabase/client.js'; -export { CardReferenceCycleError, expandCardReferences } from './adapters/metabase/card-references.js'; -export { IngestMetabaseClientFactory } from './adapters/metabase/client-port.js'; -export type { MetabaseClientLogger } from './adapters/metabase/client.js'; -export type { - MetabaseCard, - MetabaseCardSummary, - MetabaseClientConfig, - MetabaseClientFactory, - MetabaseClientRuntimeConfig, - MetabaseCollection, - MetabaseCollectionItem, - MetabaseConnectionClientFactory, - MetabaseDatabase, - MetabaseDatasetQuery, - MetabaseNativeQueryResult, - MetabaseParameter, - MetabaseResolvedTemplateTag, - MetabaseResultMetadataColumn, - MetabaseRuntimeClient, - MetabaseTemplateTag, - MetabaseUser, - ResolvedSqlResult, - TestConnectionResult, -} from './adapters/metabase/client-port.js'; -export type { - MetabaseSourceState, - MetabaseSourceStateMapping, - MetabaseSourceStateReader, - MetabaseSourceStateSelection, -} from './adapters/metabase/source-state-port.js'; -export { - METABASE_ENGINE_TO_CONNECTION_TYPE, - computeMetabaseMappingDrift, - computeMetabaseMappingPhysicalMismatches, - discoverMetabaseDatabases, - findBestMatch, - refreshMetabaseMapping, - validateMappingPhysicalMatch, - validateMetabaseMappings, -} from './adapters/metabase/mapping.js'; -export type { - AutoMatchCandidate, - AutoMatchResult as MetabaseAutoMatchResult, - DiscoveredMetabaseDatabase, - KtxConnectionPhysicalInfo, - MappingPhysicalInfo, - MappingRefreshReport, - MetabaseMappedConnectionType, - MetabaseMappingDrift, - MetabaseMappingValidationResult, - PhysicalMismatch, - PhysicalMismatchInput, -} from './adapters/metabase/mapping.js'; -export { planMetabaseFanoutChildren } from './adapters/metabase/fanout-planner.js'; -export type { - MetabaseFanoutChildPlan, - MetabaseFanoutMappingInput, - PlanMetabaseFanoutChildrenInput, -} from './adapters/metabase/fanout-planner.js'; -export { MetabaseSourceAdapter } from './adapters/metabase/metabase.adapter.js'; -export { - createLocalMetabaseSourceAdapter, - metabaseRuntimeConfigFromLocalConnection, -} from './adapters/metabase/local-metabase.adapter.js'; -export { - KtxYamlMetabaseSourceStateReader, - LocalMetabaseDiscoveryCache, -} from './adapters/metabase/local-source-state-store.js'; -export type { - LocalMetabaseDiscoveredDatabaseRow, - LocalMetabaseMappingListRow, - LocalMetabaseMappingSource, - RefreshLocalMetabaseDiscoveredDatabasesInput, -} from './adapters/metabase/local-source-state-store.js'; -export { metabaseLocalConnectionIdSchema, metabasePullConfigSchema, parseMetabasePullConfig } from './adapters/metabase/types.js'; -export type { MetabasePullConfig, MetabaseSyncMode } from './adapters/metabase/types.js'; -export { - fetchMetricflowRepo, -} from './adapters/metricflow/fetch.js'; -export type { FetchMetricflowRepoParams, FetchMetricflowRepoResult } from './adapters/metricflow/fetch.js'; -export { - parseMetricflowFiles, - translateMetricflowJinjaFilter, -} from './adapters/metricflow/deep-parse.js'; -export type { - DimensionDefinition, - MeasureDefinition, - MetricFlowParseResult, - MetricflowParseOptions, - ParsedCrossModelMetric, - ParsedMetricflowRelationship, - ParsedSemanticModel, - SimpleMeasureDefinition, -} from './adapters/metricflow/deep-parse.js'; -export { - buildMetricflowColumns, - buildMetricflowJoinsForModel, - buildMetricflowMeasures, - buildMetricflowSemanticModelSource, - countImportableMetricflowRelationships, - filterValidMetricflowRelationships, - findMatchingMetricflowTable, - getMetricflowAvailableColumnNames, - mapCrossModelMetricToSource, - mapSemanticModelToSource, - normalizeMetricflowJoinOn, - parseMetricflowJoinReference, - resolveMetricflowSemanticModelSourceName, - rewriteMetricflowJoinOn, - rewriteMetricflowManifestJoins, - toKebabCaseMetricflowName, -} from './adapters/metricflow/semantic-models.js'; -export { importMetricflowSemanticModels } from './adapters/metricflow/import-semantic-models.js'; -export type { - ImportMetricflowSemanticModelsDeps, - ImportMetricflowSemanticModelsInput, - MetricFlowImportResult, - MetricflowSemanticLayerService, - MetricflowSemanticLayerWriter, -} from './adapters/metricflow/import-semantic-models.js'; -export type { - MetricflowHostTable, - MetricflowSemanticModelImportContext, - MetricflowSemanticModelJoin, - MetricflowWritableSemanticLayerSource, -} from './adapters/metricflow/semantic-models.js'; -export { MetricflowSourceAdapter, type MetricflowSourceAdapterDeps } from './adapters/metricflow/metricflow.adapter.js'; -export { - metricflowPullConfigSchema, - parseMetricflowPullConfig, - pullConfigFromMetricflowIntegration, -} from './adapters/metricflow/pull-config.js'; -export type { - MetricflowIntegrationLike, - MetricflowPullConfig, -} from './adapters/metricflow/pull-config.js'; -export { NOTION_ORG_KNOWLEDGE_WARNING } from './adapters/notion/chunk.js'; -export { NOTION_DEFAULT_MAX_KNOWLEDGE_CREATES_PER_RUN } from './adapters/notion/types.js'; -export { LocalNotionRuntimeStore } from './adapters/notion/local-state-store.js'; -export { NotionSourceAdapter, type NotionSourceAdapterDeps } from './adapters/notion/notion.adapter.js'; -export { NotionClient, type NotionApi, type NotionBotInfo } from './adapters/notion/notion-client.js'; -export { bucketDistinctUsers, bucketErrorRate, bucketExecutions, bucketP95Runtime, bucketRecency } from './adapters/historic-sql/buckets.js'; -export { chunkHistoricSqlUnifiedStagedDir, describeHistoricSqlUnifiedScope } from './adapters/historic-sql/chunk-unified.js'; -export { detectHistoricSqlStagedDir } from './adapters/historic-sql/detect.js'; -export { - HistoricSqlExtensionMissingError, - HistoricSqlGrantsMissingError, - HistoricSqlVersionUnsupportedError, -} from './adapters/historic-sql/errors.js'; -export { HistoricSqlSourceAdapter } from './adapters/historic-sql/historic-sql.adapter.js'; -export { BigQueryHistoricSqlQueryHistoryReader } from './adapters/historic-sql/bigquery-query-history-reader.js'; -export type { BigQueryHistoricSqlQueryHistoryReaderOptions } from './adapters/historic-sql/bigquery-query-history-reader.js'; -export { PostgresPgssReader } from './adapters/historic-sql/postgres-pgss-reader.js'; -export { SnowflakeHistoricSqlQueryHistoryReader } from './adapters/historic-sql/snowflake-query-history-reader.js'; -export { stageHistoricSqlAggregatedSnapshot } from './adapters/historic-sql/stage-unified.js'; -export { - historicSqlEvidenceEnvelopeSchema, - historicSqlEvidencePath, - historicSqlPatternEvidenceSchema, - historicSqlTableUsageEvidenceSchema, - serializeHistoricSqlEvidence, -} from './adapters/historic-sql/evidence.js'; -export type { - HistoricSqlEvidenceEnvelope, - HistoricSqlPatternEvidence, - HistoricSqlTableUsageEvidence, -} from './adapters/historic-sql/evidence.js'; -export { createEmitHistoricSqlEvidenceTool } from './adapters/historic-sql/evidence-tool.js'; -export { projectHistoricSqlEvidence } from './adapters/historic-sql/projection.js'; -export type { HistoricSqlProjectionInput, HistoricSqlProjectionResult } from './adapters/historic-sql/projection.js'; -export { - patternOutputSchema, - patternsArraySchema, - tableUsageOutputSchema, -} from './adapters/historic-sql/skill-schemas.js'; -export type { - PatternOutput, - TableUsageOutput, -} from './adapters/historic-sql/skill-schemas.js'; -export type { - AggregatedTemplate, - HistoricSqlDialect, - HistoricSqlProbeResult, - HistoricSqlReader, - HistoricSqlSourceAdapterDeps, - HistoricSqlTimeWindow, - HistoricSqlUnifiedPullConfig, - KtxPostgresQueryClient, - PostgresPgssProbeResult, - StagedManifest, - StagedPatternsInput, - StagedTableInput, -} from './adapters/historic-sql/types.js'; -export { - HISTORIC_SQL_SOURCE_KEY, - aggregatedTemplateSchema, - historicSqlUnifiedPullConfigSchema, - stagedManifestSchema, - stagedPatternsInputSchema, - stagedTableInputSchema, -} from './adapters/historic-sql/types.js'; -export type { CanonicalPin } from './canonical-pins.js'; -export { buildCanonicalPinsPromptBlock, selectRelevantCanonicalPins } from './canonical-pins.js'; -export type { - BudgetExhaustedCandidateForCarryForward, - CandidateDedupServiceDeps, - CandidateDedupSettings, - ContextCandidateActionHint, - ContextCandidateCarryforwardArgs, - ContextCandidateCarryforwardResult, - ContextCandidateCarryforwardServiceDeps, - ContextCandidateCarryforwardSettings, - ContextCandidateEmbeddingPort, - ContextCandidateForPrompt, - ContextCandidateLane, - ContextCandidateRejectionReason, - ContextCandidateScoreAggregation, - ContextCandidateStatus, - ContextCandidateStorePort, - ContextCandidateVerdictSummary, - CuratorPaginationInput, - CuratorPaginationServiceDeps, - CuratorPaginationSettings, - CurrentRunEvidenceChunkForCarryForward, - InsertContextCandidateInput, - MarkContextCandidateClusterInput, -} from './context-candidates/index.js'; -export { - buildContextCandidateEmbeddingText, - CandidateDedupService, - ContextCandidateCarryforwardService, - CuratorPaginationService, -} from './context-candidates/index.js'; -export type { - ContextEvidenceDocumentRef, - ContextEvidenceEmbeddingPort, - ContextEvidenceIndexStorePort, - ContextEvidenceIndexSummary as PackageContextEvidenceIndexSummary, - EvidencePublishState, - ReplaceContextEvidenceChunk, - SqliteContextEvidenceStoreOptions, - UpsertContextEvidenceDocument, -} from './context-evidence/index.js'; -export { - ContextEvidenceIndexService, - SqliteContextEvidenceStore, -} from './context-evidence/index.js'; -export { DiffSetService } from './diff-set.service.js'; -export { IngestBundleRunner } from './ingest-bundle.runner.js'; -export type { DefaultLocalIngestAdaptersOptions } from './local-adapters.js'; -export { createDefaultLocalIngestAdapters, localPullConfigForAdapter } from './local-adapters.js'; -export type { - LocalIngestMcpOptions, - LocalIngestResult, - LocalMetabaseFanoutChild, - LocalMetabaseFanoutProgress, - LocalMetabaseFanoutProgressChild, - LocalMetabaseFanoutResult, - RunLocalIngestOptions, - RunLocalMetabaseIngestOptions, -} from './local-ingest.js'; -export { getLatestLocalIngestStatus, getLocalIngestStatus, runLocalIngest, runLocalMetabaseIngest } from './local-ingest.js'; -export { seedLocalMappingStateFromKtxYaml } from './local-mapping-reconcile.js'; -export type { - CreateLocalBundleIngestRuntimeOptions, - LocalBundleIngestRuntime, -} from './local-bundle-runtime.js'; -export { createLocalBundleIngestRuntime } from './local-bundle-runtime.js'; -export type { - LocalIngestDiffPaths, - LocalIngestRunRecord, - LocalIngestStatus, - RunLocalStageOnlyIngestOptions, -} from './local-stage-ingest.js'; -export { getLocalStageOnlyIngestStatus, runLocalStageOnlyIngest } from './local-stage-ingest.js'; -export { - ingestReportToMemoryFlowReplay, - localIngestRunToMemoryFlowReplay, -} from './memory-flow/events.js'; -export { - buildAuthenticatedUrl, - cleanupRepoDir, - cloneOrPull, - RepoConfigError, - RepoFetchError, - repoDirExists, - sanitizeRepoError, - testRepoConnection, - validateRepoConfig, -} from './repo-fetch.js'; -export type { RepoFetchConfig } from './repo-fetch.js'; -export { - loadProjectInfo, - parseProjectName, - parseProjectVars, - resolveJinjaVariables, -} from './dbt-shared/project-vars.js'; -export type { DbtProjectInfo, ResolveJinjaVariablesResult } from './dbt-shared/project-vars.js'; -export { findDbtSchemaFiles, loadDbtSchemaFiles } from './dbt-shared/schema-files.js'; -export { - computeDbtSchemaHash, - parseDbtSchemaFile, - parseDbtSchemaFiles, -} from './adapters/dbt-descriptions/parse-schema.js'; -export type { - DbtParsedColumn, - DbtColumnConstraints, - DbtDataTestRef, - DbtParsedRelationship, - DbtParsedTable, - DbtSchemaFile, - DbtSchemaParseResult, -} from './adapters/dbt-descriptions/parse-schema.js'; -export { findMatchingKtxTable, matchDbtTables } from './adapters/dbt-descriptions/match-tables.js'; -export type { DbtHostTableLite, DbtTableMatch } from './adapters/dbt-descriptions/match-tables.js'; -export { toDescriptionUpdates } from './adapters/dbt-descriptions/to-description-updates.js'; -export type { DbtDescriptionUpdates } from './adapters/dbt-descriptions/to-description-updates.js'; -export { toRelationshipUpdates } from './adapters/dbt-descriptions/to-relationship-updates.js'; -export type { DbtRelationshipUpdates } from './adapters/dbt-descriptions/to-relationship-updates.js'; -export { toMetadataUpdates } from './adapters/dbt-descriptions/to-metadata-updates.js'; -export { mergeSemanticModelTables } from './adapters/dbt-descriptions/merge-semantic-model-tables.js'; -export type { KtxJoinUpdate, KtxMetadataUpdate } from '../scan/enrichment-types.js'; -export { - createInitialMemoryFlowInteractionState, - findMemoryFlowSearchMatches, - reduceMemoryFlowInteractionState, - selectedMemoryFlowColumn, - selectedMemoryFlowDetails, - selectMemoryFlowChip, - selectMemoryFlowColumn, - visibleMemoryFlowChips, -} from './memory-flow/interaction.js'; -export { renderMemoryFlowInteractive } from './memory-flow/interactive-render.js'; -export { createMemoryFlowLiveBuffer, sanitizeMemoryFlowError } from './memory-flow/live-buffer.js'; -export { renderMemoryFlowReplay } from './memory-flow/render.js'; -export { formatMemoryFlowFinalSummary } from './memory-flow/summary.js'; -export type { MemoryFlowStreamEvent } from './memory-flow/schema.js'; -export { - memoryFlowActionDetailSchema, - memoryFlowDetailSectionsSchema, - memoryFlowEventSchema, - memoryFlowPlannedWorkUnitSchema, - memoryFlowReplayInputSchema, - memoryFlowRunStatusSchema, - memoryFlowStreamEventSchema, - parseMemoryFlowReplayInput, -} from './memory-flow/schema.js'; -export type { - MemoryFlowChip, - MemoryFlowColumnId, - MemoryFlowColumnView, - MemoryFlowDisplayStatus, - MemoryFlowEvent, - MemoryFlowEventSink, - MemoryFlowFilterMode, - MemoryFlowInteractionCommand, - MemoryFlowInteractionState, - MemoryFlowLiveBufferOptions, - MemoryFlowPaneId, - MemoryFlowPlannedWorkUnit, - MemoryFlowRenderOptions, - MemoryFlowReplayInput, - MemoryFlowReplayPatch, - MemoryFlowRunStatus, - MemoryFlowViewModel, -} from './memory-flow/types.js'; -export { buildMemoryFlowViewModel } from './memory-flow/view-model.js'; -export type { - MemoryFlowStatusBadge, - MemoryFlowVisualColumn, - MemoryFlowVisualModel, -} from './memory-flow/visuals.js'; -export { - buildMemoryFlowVisualModel, - memoryFlowStatusBadge, - renderMemoryFlowConnectorLine, -} from './memory-flow/visuals.js'; -export type { - PageTriageEvidenceChunk, - PageTriageReport, - PageTriageRunArgs, - PageTriageServiceDeps, - PageTriageSettings, - PageTriageStorePort, -} from './page-triage/index.js'; -export { PageTriageService } from './page-triage/index.js'; -export type { - CandidateDedupPort, - CandidateDedupResult, - ContextCandidateCarryforwardPort, - ContextCandidateForDedup, - ContextCandidateSummary, - ContextEvidenceCandidatesPort, - ContextEvidenceIndexPort, - ContextEvidenceIndexSummary, - CreateIngestRunArgs, - CuratorPaginationPort, - CuratorPaginationReport, - DiffSetComputerPort, - IngestBundleRunnerDeps, - IngestCanonicalPinsPort, - IngestCommitMessagePort, - IngestFileStorePort, - IngestGitAuthor, - IngestKnowledgeIndexPort, - IngestLockPort, - IngestProvenanceInsert, - IngestProvenancePort, - IngestProvenanceRow, - IngestReportsPort, - IngestRunnerJob, - IngestRunRecord, - IngestRunsPort, - IngestSessionWorktree, - IngestSessionWorktreePort, - IngestSettingsPort, - IngestStoragePort, - IngestToolsetFactoryPort, - IngestToolsetLike, - PageTriagePort, - PageTriageRunResult, - ProvenanceActionType, - SourceAdapterRegistryPort, -} from './ports.js'; -export { - buildSyncId, - provenanceMarker, - rawSourcesDirForSync, - rawSourcesRoot, -} from './raw-sources-paths.js'; -export { ingestReportSnapshotSchema, parseIngestReportSnapshot } from './report-snapshot.js'; -export type { IngestReportBody, IngestReportSnapshot } from './reports.js'; -export * from './artifact-gates.js'; -export * from './ingest-trace.js'; -export * from './isolated-diff/git-patch.js'; -export * from './isolated-diff/patch-integrator.js'; -export * from './isolated-diff/work-unit-executor.js'; -export * from './reports.js'; -export { SourceAdapterRegistry } from './source-adapter-registry.js'; -export type { SqliteBundleIngestStoreOptions } from './sqlite-bundle-ingest-store.js'; -export { SqliteBundleIngestStore } from './sqlite-bundle-ingest-store.js'; -export type { - SaveCompletedLocalIngestRunInput, - SqliteLocalIngestStoreOptions, -} from './sqlite-local-ingest-store.js'; -export { SqliteLocalIngestStore } from './sqlite-local-ingest-store.js'; -export type { - ReconcileCandidateForPrompt, - ReconcileCandidateSummary, - ReconcilePromptRunState, - WikiPageRef, -} from './stages/build-reconcile-context.js'; -export { - buildReconcileSystemPrompt, - buildReconcileToolSet, - buildReconcileUserPrompt, -} from './stages/build-reconcile-context.js'; -export type { ReconciliationOutcome } from './stages/stage-4-reconciliation.js'; -export { runReconciliationStage4 } from './stages/stage-4-reconciliation.js'; -export type { StageIndex } from './stages/stage-index.types.js'; -export type { - ChunkResult, - DiffSet, - EvictionUnit, - FetchContext, - IngestBundleJob, - IngestBundleRef, - IngestBundleResult, - IngestDiffSummary, - IngestJobContext, - IngestJobPhase, - IngestTrigger, - ScopeDescriptor, - SourceAdapter, - SourceFetchIssue, - SourceFetchReport, - TriageLane, - TriageSignals, - UnresolvedCardInfo, - WorkUnit, - DeterministicProjectionContext, - ProjectionResult, - DeterministicFinalizationContext, - FinalizationOverrideReplay, - FinalizationResult, -} from './types.js'; -export * from './wiki-body-refs.js'; diff --git a/packages/context/src/ingest/memory-flow/index.ts b/packages/context/src/ingest/memory-flow/index.ts deleted file mode 100644 index fa4f8fc5..00000000 --- a/packages/context/src/ingest/memory-flow/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -export { - memoryFlowReplayInputSchema, - memoryFlowStreamEventSchema, - parseMemoryFlowReplayInput, -} from './schema.js'; -export type { MemoryFlowStreamEvent } from './schema.js'; -export { buildMemoryFlowViewModel } from './view-model.js'; -export { renderMemoryFlowReplay } from './render.js'; -export { formatMemoryFlowFinalSummary } from './summary.js'; -export type { - MemoryFlowDetailSections, - MemoryFlowEvent, - MemoryFlowPlannedWorkUnit, - MemoryFlowReplayInput, - MemoryFlowRunStatus, - MemoryFlowViewModel, -} from './types.js'; diff --git a/packages/context/src/ingest/memory-flow/package-export.test.ts b/packages/context/src/ingest/memory-flow/package-export.test.ts deleted file mode 100644 index 5ddd9b91..00000000 --- a/packages/context/src/ingest/memory-flow/package-export.test.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -describe('@ktx/context/ingest/memory-flow lightweight export', () => { - it('exports replay parsing and text rendering without the full ingest entry point', async () => { - const memoryFlow = await import('./index.js'); - - expect(memoryFlow.parseMemoryFlowReplayInput).toBeTypeOf('function'); - expect(memoryFlow.buildMemoryFlowViewModel).toBeTypeOf('function'); - expect(memoryFlow.renderMemoryFlowReplay).toBeTypeOf('function'); - }); -}); diff --git a/packages/context/src/ingest/metabase-mapping.ts b/packages/context/src/ingest/metabase-mapping.ts deleted file mode 100644 index 5ce66a12..00000000 --- a/packages/context/src/ingest/metabase-mapping.ts +++ /dev/null @@ -1,23 +0,0 @@ -export { - computeMetabaseMappingDrift, - computeMetabaseMappingPhysicalMismatches, - discoverMetabaseDatabases, - findBestMatch, - METABASE_ENGINE_TO_CONNECTION_TYPE, - refreshMetabaseMapping, - validateMappingPhysicalMatch, - validateMetabaseMappings, -} from './adapters/metabase/mapping.js'; -export type { - AutoMatchCandidate, - AutoMatchResult as MetabaseAutoMatchResult, - DiscoveredMetabaseDatabase, - KtxConnectionPhysicalInfo, - MappingPhysicalInfo, - MappingRefreshReport, - MetabaseMappedConnectionType, - MetabaseMappingDrift, - MetabaseMappingValidationResult, - PhysicalMismatch, - PhysicalMismatchInput, -} from './adapters/metabase/mapping.js'; diff --git a/packages/context/src/ingest/page-triage/index.ts b/packages/context/src/ingest/page-triage/index.ts deleted file mode 100644 index e589165e..00000000 --- a/packages/context/src/ingest/page-triage/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export type { - PageTriageEvidenceChunk, - PageTriageReport, - PageTriageRunArgs, - PageTriageServiceDeps, - PageTriageSettings, - PageTriageStorePort, -} from './page-triage.service.js'; -export { PageTriageService } from './page-triage.service.js'; diff --git a/packages/context/src/llm/index.ts b/packages/context/src/llm/index.ts deleted file mode 100644 index 96da8dcf..00000000 --- a/packages/context/src/llm/index.ts +++ /dev/null @@ -1,44 +0,0 @@ -export { KtxIngestEmbeddingPortAdapter, KtxScanEmbeddingPortAdapter } from './embedding-port.js'; -export { AiSdkKtxLlmRuntime } from './ai-sdk-runtime.js'; -export type { AgentTelemetryPort, AiSdkKtxLlmRuntimeDeps } from './ai-sdk-runtime.js'; -export { createKtxClaudeCodeEnv, CLAUDE_CODE_PROVIDER_ENV_DENYLIST } from './claude-code-env.js'; -export { resolveClaudeCodeModel } from './claude-code-models.js'; -export { ClaudeCodeKtxLlmRuntime, mapClaudeCodeStopReason, runClaudeCodeAuthProbe } from './claude-code-runtime.js'; -export type { - AgentRunnerPort, - KtxGenerateObjectInput, - KtxGenerateTextInput, - KtxLlmRuntimePort, - KtxRuntimeToolDescriptor, - KtxRuntimeToolOutput, - KtxRuntimeToolSet, - RunLoopParams, - RunLoopResult, - RunLoopStepInfo, - RunLoopStopReason, -} from './runtime-port.js'; -export { RuntimeAgentRunner } from './runtime-port.js'; -export { - createAiSdkToolSet, - createClaudeSdkTools, - createRuntimeToolDescriptorFromAiTool, - createRuntimeToolSetFromAiSdkTools, - normalizeKtxRuntimeToolOutput, -} from './runtime-tools.js'; -export type { - KtxLlmDebugProviderOptionsEntry, - KtxLlmDebugRequest, - KtxLlmDebugRequestRecorder, - SummarizeKtxLlmDebugRequestInput, -} from './debug-request-recorder.js'; -export { - createJsonlKtxLlmDebugRequestRecorder, - summarizeKtxLlmDebugRequest, -} from './debug-request-recorder.js'; -export { - createLocalKtxEmbeddingProviderFromConfig, - createLocalKtxLlmProviderFromConfig, - createLocalKtxLlmRuntimeFromConfig, - resolveLocalKtxEmbeddingConfig, - resolveLocalKtxLlmConfig, -} from './local-config.js'; diff --git a/packages/context/src/mcp/index.ts b/packages/context/src/mcp/index.ts deleted file mode 100644 index f241c68e..00000000 --- a/packages/context/src/mcp/index.ts +++ /dev/null @@ -1,25 +0,0 @@ -export type { RegisterKtxContextToolsDeps } from './context-tools.js'; -export { jsonErrorToolResult, jsonToolResult, registerKtxContextTools } from './context-tools.js'; -export { createLocalProjectMcpContextPorts } from './local-project-ports.js'; -export { createDefaultKtxMcpServer, createKtxMcpServer } from './server.js'; -export type { - KtxConnectionSummary, - KtxConnectionsMcpPort, - KtxDiscoverDataMcpPort, - KtxDictionarySearchMcpPort, - KtxEntityDetailsMcpPort, - KtxKnowledgeMcpPort, - KtxKnowledgePage, - KtxKnowledgeSearchResponse, - KtxKnowledgeSearchResult, - KtxMcpContextPorts, - KtxMcpServerDeps, - KtxMcpServerLike, - KtxMcpTextContent, - KtxMcpToolResult, - KtxMcpUserContext, - KtxSemanticLayerMcpPort, - KtxSemanticLayerQueryResponse, - KtxSemanticLayerReadResponse, - MemoryIngestPort, -} from './types.js'; diff --git a/packages/context/src/memory/index.ts b/packages/context/src/memory/index.ts deleted file mode 100644 index 28efcd2e..00000000 --- a/packages/context/src/memory/index.ts +++ /dev/null @@ -1,41 +0,0 @@ -export { - buildRequiredSkillsBlock, - DEFAULT_SKILL_NAMES, - detectCaptureSignals, - isWorthAnalyzing, - prefilterSkipReason, - promptNameFor, - stepBudgetFor, -} from './capture-signals.js'; -export { MemoryAgentService } from './memory-agent.service.js'; -export { createLocalProjectMemoryIngest, type CreateLocalProjectMemoryIngestOptions } from './local-memory.js'; -export { LocalMemoryRunStore, type LocalMemoryRunStoreOptions } from './local-memory-runs.js'; -export { - MemoryIngestService, - type MemoryIngestServiceDeps, - type MemoryIngestStartResult, - type MemoryIngestStatus, - type MemoryRunRecord, - type MemoryRunStatus, - type MemoryRunStorePort, -} from './memory-runs.js'; - -export type { - CaptureSession, - CaptureSignals, - MemoryAction, - MemoryAgentInput, - MemoryAgentResult, - MemoryAgentServiceDeps, - MemoryAgentSettings, - MemoryAgentSourceType, - MemoryCommitMessagePort, - MemoryConnectionPort, - MemoryFileStorePort, - MemoryKnowledgeSlRefsPort, - MemoryLockPort, - MemorySlSourceReconcilerPort, - MemoryTelemetryPort, - MemoryToolSetLike, - MemoryToolsetFactoryPort, -} from './types.js'; diff --git a/packages/context/src/package-exports.test.ts b/packages/context/src/package-exports.test.ts deleted file mode 100644 index bbd660e9..00000000 --- a/packages/context/src/package-exports.test.ts +++ /dev/null @@ -1,267 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { - DeterministicFinalizationContext, - FinalizationOverrideReplay, - FinalizationResult, -} from './ingest/index.js'; -import type { - ApplyLocalScanRelationshipReviewDecisionsInput, - ApplyLocalScanRelationshipReviewDecisionsResult, -} from './scan/index.js'; - -const scanTypeExportCoverage: Partial<{ - input: ApplyLocalScanRelationshipReviewDecisionsInput; - result: ApplyLocalScanRelationshipReviewDecisionsResult; -}> = {}; - -const ingestFinalizationTypeExportCoverage: Partial<{ - context: DeterministicFinalizationContext; - overrideReplay: FinalizationOverrideReplay; - result: FinalizationResult; -}> = {}; - -describe('@ktx/context package exports', () => { - it('exports package entry points used by host adapters', async () => { - const core = await import('./core/index.js'); - const connections = await import('./connections/index.js'); - const scan = await import('./scan/index.js'); - const search = await import('./search/index.js'); - const agent = await import('./agent/index.js'); - const prompts = await import('./prompts/index.js'); - const skills = await import('./skills/index.js'); - const sl = await import('./sl/index.js'); - const wiki = await import('./wiki/index.js'); - const tools = await import('./tools/index.js'); - const memory = await import('./memory/index.js'); - const ingest = await import('./ingest/index.js'); - const memoryFlow = await import('./ingest/memory-flow/index.js'); - const metabaseMapping = await import('./ingest/metabase-mapping.js'); - const mcp = await import('./mcp/index.js'); - const project = await import('./project/index.js'); - const daemon = await import('./daemon/index.js'); - const sqlAnalysis = await import('./sql-analysis/index.js'); - const root = await import('./index.js'); - - expect(core).toBeDefined(); - expect(connections.createPostgresQueryExecutor).toBeTypeOf('function'); - expect(connections.createSqliteQueryExecutor).toBeTypeOf('function'); - expect(connections.createDefaultLocalQueryExecutor).toBeTypeOf('function'); - expect(connections.sqliteDatabasePathFromConnection).toBeTypeOf('function'); - expect(connections.parseNotionConnectionConfig).toBeTypeOf('function'); - expect(connections.redactNotionConnectionConfig).toBeTypeOf('function'); - expect(connections.notionConnectionToPullConfig).toBeTypeOf('function'); - expect(scan).toBeDefined(); - expect(scanTypeExportCoverage).toEqual({}); - expect(ingestFinalizationTypeExportCoverage).toEqual({}); - expect(scan.createKtxConnectorCapabilities).toBeTypeOf('function'); - expect(`liveDatabaseSnapshotToKtx${'SchemaSnapshot'}` in scan).toBe(false); - expect(scan.normalizeKtxNativeType).toBeTypeOf('function'); - expect(scan.inferKtxDimensionType).toBeTypeOf('function'); - expect(scan.redactKtxCredentialEnvelope).toBeTypeOf('function'); - expect(scan.redactKtxScanReport).toBeTypeOf('function'); - expect(scan.redactKtxScanWarning).toBeTypeOf('function'); - expect(core.redactKtxSensitiveMetadata).toBeTypeOf('function'); - expect(core.redactKtxSensitiveText).toBeTypeOf('function'); - expect(scan.isKtxDataDictionaryCandidate).toBeTypeOf('function'); - expect(scan.buildKtxColumnEmbeddingText).toBeTypeOf('function'); - expect(scan.KtxDescriptionGenerator).toBeTypeOf('function'); - expect(scan.runLocalScan).toBeTypeOf('function'); - expect(scan.writeLocalScanEnrichmentArtifacts).toBeTypeOf('function'); - expect(scan.readLocalScanStructuralSnapshot).toBeTypeOf('function'); - expect(scan.writeLocalScanManifestShards).toBeTypeOf('function'); - expect(scan.buildKtxColumnDescriptionPrompt).toBeTypeOf('function'); - expect(scan.buildKtxTableDescriptionPrompt).toBeTypeOf('function'); - expect(scan.buildKtxDataSourceDescriptionPrompt).toBeTypeOf('function'); - expect(scan.currentKtxRelationshipBenchmarkDetector).toBeTypeOf('function'); - expect(scan.generateKtxRelationshipDiscoveryCandidates).toBeTypeOf('function'); - expect(scan.inferKtxRelationshipTargetPks).toBeTypeOf('function'); - expect(scan.mergeKtxRelationshipDiscoveryCandidates).toBeTypeOf('function'); - expect(scan.normalizeKtxRelationshipName).toBeTypeOf('function'); - expect(scan.tokenizeKtxRelationshipName).toBeTypeOf('function'); - expect(scan.tokenSimilarity).toBeTypeOf('function'); - expect(scan.localCandidateTables).toBeTypeOf('function'); - expect(scan.scoreKtxRelationshipCandidate).toBeTypeOf('function'); - expect(scan.defaultKtxRelationshipScoreWeights).toBeTypeOf('function'); - expect(scan.normalizeKtxRelationshipScoreWeights).toBeTypeOf('function'); - expect(scan.calibrateWeightsFromSyntheticFixtures).toBeTypeOf('function'); - expect(scan.singularizeKtxRelationshipToken).toBeTypeOf('function'); - expect(scan.pluralizeKtxRelationshipToken).toBeTypeOf('function'); - expect(scan.collectKtxFormalMetadataRelationships).toBeTypeOf('function'); - expect(scan.discoverKtxCompositeRelationships).toBeTypeOf('function'); - expect(scan.proposeKtxRelationshipCandidatesWithLlm).toBeTypeOf('function'); - expect(scan.profileKtxRelationshipSchema).toBeTypeOf('function'); - expect(scan.quoteKtxRelationshipIdentifier).toBeTypeOf('function'); - expect(scan.formatKtxRelationshipTableRef).toBeTypeOf('function'); - expect(scan.validateKtxRelationshipDiscoveryCandidates).toBeTypeOf('function'); - expect(scan.applyKtxRelationshipValidationBudget).toBeTypeOf('function'); - expect(scan.defaultKtxRelationshipValidationBudget).toBeTypeOf('function'); - expect(scan.resolveKtxRelationshipGraph).toBeTypeOf('function'); - expect(scan.discoverKtxRelationships).toBeTypeOf('function'); - expect('KtxRelationshipDetector' in scan).toBe(false); - expect('defaultKtxRelationshipDetectionSettings' in scan).toBe(false); - expect('KTX_RELATIONSHIP_DETECTION_CONFIDENCE' in scan).toBe(false); - expect(scan.buildKtxRelationshipArtifacts).toBeTypeOf('function'); - expect(scan.buildKtxRelationshipDiagnostics).toBeTypeOf('function'); - expect(scan.readLocalScanRelationshipArtifacts).toBeTypeOf('function'); - expect(scan.writeLocalScanRelationshipReviewDecision).toBeTypeOf('function'); - expect(scan.applyLocalScanRelationshipReviewDecisions).toBeTypeOf('function'); - expect(scan.exportLocalRelationshipFeedbackLabels).toBeTypeOf('function'); - expect(scan.formatKtxRelationshipFeedbackLabelsJsonl).toBeTypeOf('function'); - expect(scan.buildKtxRelationshipFeedbackCalibrationReport).toBeTypeOf('function'); - expect(scan.calibrateLocalRelationshipFeedbackLabels).toBeTypeOf('function'); - expect(scan.formatKtxRelationshipFeedbackCalibrationMarkdown).toBeTypeOf('function'); - expect(scan.buildKtxRelationshipThresholdAdviceReport).toBeTypeOf('function'); - expect(scan.adviseLocalRelationshipFeedbackThresholds).toBeTypeOf('function'); - expect(scan.formatKtxRelationshipThresholdAdviceMarkdown).toBeTypeOf('function'); - expect(scan.emptyKtxRelationshipProfileArtifact).toBeTypeOf('function'); - expect(scan.loadKtxRelationshipBenchmarkFixture).toBeTypeOf('function'); - expect(scan.loadKtxRelationshipBenchmarkFixtures).toBeTypeOf('function'); - expect(scan.maskKtxRelationshipBenchmarkSnapshot).toBeTypeOf('function'); - expect(scan.runKtxRelationshipBenchmarkCase).toBeTypeOf('function'); - expect(scan.runKtxRelationshipBenchmarkSuite).toBeTypeOf('function'); - expect(scan.KTX_RELATIONSHIP_BENCHMARK_MODES).toEqual([ - 'metadata_present', - 'declared_fks_removed', - 'declared_pks_removed', - 'declared_pks_and_declared_fks_removed', - 'llm_disabled', - 'profiling_disabled', - 'validation_disabled', - 'embeddings_disabled', - ]); - expect(scan.buildKtxRelationshipBenchmarkReport).toBeTypeOf('function'); - expect(scan.formatKtxRelationshipBenchmarkReportMarkdown).toBeTypeOf('function'); - expect(search).toBeDefined(); - expect(search.HybridSearchCore).toBeTypeOf('function'); - expect(search.normalizeSearchQuery).toBeTypeOf('function'); - expect(search.rrfContribution).toBeTypeOf('function'); - expect(search.assertSearchBackendConformanceCase).toBeTypeOf('function'); - expect(search.assertSearchBackendCapabilities).toBeTypeOf('function'); - expect(core.resolveKtxConfigReference).toBeTypeOf('function'); - expect(root.HybridSearchCore).toBeTypeOf('function'); - expect(root.assertSearchBackendConformanceCase).toBeTypeOf('function'); - expect(root.assertSearchBackendCapabilities).toBeTypeOf('function'); - expect(root.createLocalKtxEmbeddingProviderFromConfig).toBeTypeOf('function'); - expect(agent).toBeDefined(); - expect(agent.RuntimeAgentRunner).toBeTypeOf('function'); - expect(root.RuntimeAgentRunner).toBeTypeOf('function'); - expect(root.createLocalKtxLlmProviderFromConfig).toBeTypeOf('function'); - expect(prompts).toBeDefined(); - expect(skills).toBeDefined(); - expect(sl).toBeDefined(); - expect(sl.writeLocalSlSource).toBeTypeOf('function'); - expect(sl.readLocalSlSource).toBeTypeOf('function'); - expect(sl.validateLocalSlSource).toBeTypeOf('function'); - expect(sl.searchLocalSlSources).toBeTypeOf('function'); - expect(sl.SqliteSlSourcesIndex).toBeTypeOf('function'); - expect('searchLocalSlSourcesWithPglitePrototype' in sl).toBe(false); - expect(sl.compileLocalSlQuery).toBeTypeOf('function'); - expect(wiki).toBeDefined(); - expect(wiki.writeLocalKnowledgePage).toBeTypeOf('function'); - expect(wiki.readLocalKnowledgePage).toBeTypeOf('function'); - expect(wiki.searchLocalKnowledgePages).toBeTypeOf('function'); - expect(wiki.SqliteKnowledgeIndex).toBeTypeOf('function'); - expect('WikiSearchMatchReason' in wiki).toBe(false); - expect(tools).toBeDefined(); - expect(memory).toBeDefined(); - expect(ingest).toBeDefined(); - expect(memoryFlow.parseMemoryFlowReplayInput).toBeTypeOf('function'); - expect(memoryFlow.renderMemoryFlowReplay).toBeTypeOf('function'); - expect(ingest.LiveDatabaseSourceAdapter).toBeTypeOf('function'); - expect(ingest.createDaemonLiveDatabaseIntrospection).toBeTypeOf('function'); - expect(ingest.buildLiveDatabaseManifestShards).toBeTypeOf('function'); - expect(ingest.planLiveDatabaseStructuralSync).toBeTypeOf('function'); - expect(ingest.runLocalIngest).toBeTypeOf('function'); - expect(ingest.runLocalMetabaseIngest).toBeTypeOf('function'); - expect(ingest.getLocalIngestStatus).toBeTypeOf('function'); - expect(ingest.createLocalBundleIngestRuntime).toBeTypeOf('function'); - expect(ingest.runLocalStageOnlyIngest).toBeTypeOf('function'); - expect(ingest.getLocalStageOnlyIngestStatus).toBeTypeOf('function'); - expect(ingest.createDefaultLocalIngestAdapters).toBeTypeOf('function'); - expect(ingest.createLookerQueryToSlTool).toBeTypeOf('function'); - expect(ingest.buildLookerSlProposal).toBeTypeOf('function'); - expect(ingest.describeLookerScope).toBeTypeOf('function'); - expect(ingest.hashLookerScope).toBeTypeOf('function'); - expect(ingest.readLookerFetchReport).toBeTypeOf('function'); - expect(ingest.writeLookerFetchReport).toBeTypeOf('function'); - expect(ingest.writeLookerEvidenceDocuments).toBeTypeOf('function'); - expect(ingest.getLookerTriageSignals).toBeTypeOf('function'); - expect(ingest.LookerClient).toBeTypeOf('function'); - expect(ingest.DefaultLookerConnectionClientFactory).toBeTypeOf('function'); - expect(ingest.DefaultLookerClientFactory).toBeTypeOf('function'); - expect(ingest.LocalLookerRuntimeStore).toBeTypeOf('function'); - expect(ingest.createDaemonLookerTableIdentifierParser).toBeTypeOf('function'); - expect(ingest.createLocalLookerCredentialResolver).toBeTypeOf('function'); - expect(ingest.discoverLookerConnections).toBeTypeOf('function'); - expect(ingest.computeLookerMappingDrift).toBeTypeOf('function'); - expect(ingest.validateLookerMappings).toBeTypeOf('function'); - expect(ingest.refreshLookerMappingPlaceholders).toBeTypeOf('function'); - expect(ingest.suggestKtxConnectionForLookerConnection).toBeTypeOf('function'); - expect(ingest.buildLookerPullConfigFromInputs).toBeTypeOf('function'); - expect(ingest.validateLookerWarehouseTarget).toBeTypeOf('function'); - expect(ingest.sqlglotDialectForConnectionType).toBeTypeOf('function'); - expect(ingest.lookerConnectionIdSchema).toBeDefined(); - expect(ingest.lookerRuntimeCursorsSchema).toBeDefined(); - expect(ingest.stagedSyncConfigSchema).toBeDefined(); - expect(ingest.stagedLookerScopeFileSchema).toBeDefined(); - expect(ingest.stagedLookerFetchReportSchema).toBeDefined(); - expect('LocalMetabaseSourceStateReader' in ingest).toBe(false); - expect(ingest.KtxYamlMetabaseSourceStateReader).toBeTypeOf('function'); - expect(ingest.LocalMetabaseDiscoveryCache).toBeTypeOf('function'); - expect(ingest.createLocalMetabaseSourceAdapter).toBeTypeOf('function'); - expect(ingest.metabaseRuntimeConfigFromLocalConnection).toBeTypeOf('function'); - expect(ingest.IngestMetabaseClientFactory).toBeTypeOf('function'); - expect(ingest.MetabaseClient).toBeTypeOf('function'); - expect(ingest.DefaultMetabaseConnectionClientFactory).toBeTypeOf('function'); - expect(ingest.DEFAULT_METABASE_CLIENT_CONFIG).toMatchObject({ - maxRetries: 2, - timeoutMs: 60000, - retryableStatuses: [429, 500, 502, 503, 504], - }); - expect(ingest.expandCardReferences).toBeTypeOf('function'); - expect(ingest.CardReferenceCycleError).toBeTypeOf('function'); - expect(ingest.parseMetabasePullConfig).toBeTypeOf('function'); - expect(ingest.METABASE_ENGINE_TO_CONNECTION_TYPE).toMatchObject({ - postgres: 'POSTGRESQL', - bigquery: 'BIGQUERY', - snowflake: 'SNOWFLAKE', - }); - expect(metabaseMapping.METABASE_ENGINE_TO_CONNECTION_TYPE).toBe(ingest.METABASE_ENGINE_TO_CONNECTION_TYPE); - expect(metabaseMapping.validateMappingPhysicalMatch).toBeTypeOf('function'); - expect(ingest.discoverMetabaseDatabases).toBeTypeOf('function'); - expect(ingest.computeMetabaseMappingDrift).toBeTypeOf('function'); - expect(ingest.computeMetabaseMappingPhysicalMismatches).toBeTypeOf('function'); - expect(ingest.refreshMetabaseMapping).toBeTypeOf('function'); - expect(ingest.validateMetabaseMappings).toBeTypeOf('function'); - expect(ingest.validateMappingPhysicalMatch).toBeTypeOf('function'); - expect(ingest.findBestMatch).toBeTypeOf('function'); - expect(ingest.NotionSourceAdapter).toBeTypeOf('function'); - expect(ingest.NotionClient).toBeTypeOf('function'); - expect(ingest.HistoricSqlSourceAdapter).toBeTypeOf('function'); - expect(ingest.SnowflakeHistoricSqlQueryHistoryReader).toBeTypeOf('function'); - expect(ingest.BigQueryHistoricSqlQueryHistoryReader).toBeTypeOf('function'); - expect(ingest.PostgresPgssReader).toBeTypeOf('function'); - expect(ingest.HistoricSqlExtensionMissingError).toBeTypeOf('function'); - expect(ingest.HistoricSqlVersionUnsupportedError).toBeTypeOf('function'); - expect(ingest.HISTORIC_SQL_SOURCE_KEY).toBe('historic-sql'); - expect(ingest.historicSqlUnifiedPullConfigSchema).toBeDefined(); - expect(ingest.aggregatedTemplateSchema).toBeDefined(); - expect(ingest.stagedTableInputSchema).toBeDefined(); - expect(ingest.historicSqlEvidenceEnvelopeSchema).toBeDefined(); - expect(ingest.historicSqlEvidencePath).toBeTypeOf('function'); - expect(ingest.createEmitHistoricSqlEvidenceTool).toBeTypeOf('function'); - expect(ingest.SqliteContextEvidenceStore).toBeTypeOf('function'); - expect(ingest.SqliteBundleIngestStore).toBeTypeOf('function'); - expect(ingest.CuratorPaginationService).toBeTypeOf('function'); - expect(mcp).toBeDefined(); - expect(project).toBeDefined(); - expect(daemon).toBeDefined(); - expect(mcp.registerKtxContextTools).toBeTypeOf('function'); - expect(mcp.createLocalProjectMcpContextPorts).toBeTypeOf('function'); - expect(project.buildDefaultKtxProjectConfig).toBeTypeOf('function'); - expect(daemon.createHttpSemanticLayerComputePort).toBeTypeOf('function'); - expect(daemon.createPythonSemanticLayerComputePort).toBeTypeOf('function'); - expect(sqlAnalysis.createHttpSqlAnalysisPort).toBeTypeOf('function'); - expect(root.createHttpSqlAnalysisPort).toBeTypeOf('function'); - }); -}); diff --git a/packages/context/src/project/index.ts b/packages/context/src/project/index.ts deleted file mode 100644 index a0c08767..00000000 --- a/packages/context/src/project/index.ts +++ /dev/null @@ -1,45 +0,0 @@ -export type { - KtxConfigIssue, - KtxConfigValidation, - KtxProjectConfig, - KtxProjectConnectionConfig, - KtxProjectEmbeddingConfig, - KtxProjectLlmConfig, - KtxSearchBackend, - KtxStorageState, -} from './config.js'; -export { - buildDefaultKtxProjectConfig, - generateKtxProjectConfigJsonSchema, - parseKtxProjectConfig, - serializeKtxProjectConfig, - validateKtxProjectConfig, -} from './config.js'; -export type { KtxConnectionConfig } from './driver-schemas.js'; -export type { LocalGitFileStoreDeps } from './local-git-file-store.js'; -export { LocalGitFileStore } from './local-git-file-store.js'; -export { ktxLocalStateDbPath } from './local-state-db.js'; -export type { - ConnectionMappingBootstrap, - LookerMappingBootstrap, - LookmlMappingBootstrap, - MetabaseMappingBootstrap, -} from './mappings-yaml-schema.js'; -export { - parseConnectionMappingBootstrap, - parseLookerMappingBootstrap, - parseLookmlMappingBootstrap, - parseMetabaseMappingBootstrap, -} from './mappings-yaml-schema.js'; -export type { InitKtxProjectOptions, InitKtxProjectResult, KtxLocalProject, LoadKtxProjectOptions } from './project.js'; -export { initKtxProject, loadKtxProject } from './project.js'; -export type { KtxSetupStep } from './setup-config.js'; -export { - KTX_SETUP_STEPS, - ktxSetupStatePath, - markKtxSetupStateStepComplete, - mergeKtxSetupGitignoreEntries, - readKtxSetupState, - setKtxSetupDatabaseConnectionIds, - writeKtxSetupState, -} from './setup-config.js'; diff --git a/packages/context/src/prompts/index.ts b/packages/context/src/prompts/index.ts deleted file mode 100644 index c70cbfc2..00000000 --- a/packages/context/src/prompts/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export type { PromptContext, PromptServiceOptions } from './prompt.service.js'; -export { PromptService } from './prompt.service.js'; diff --git a/packages/context/src/scan/index.ts b/packages/context/src/scan/index.ts deleted file mode 100644 index 6b4cb950..00000000 --- a/packages/context/src/scan/index.ts +++ /dev/null @@ -1,409 +0,0 @@ -export { - REDACTED_KTX_CREDENTIAL_VALUE, - redactKtxCredentialEnvelope, - redactKtxCredentialValue, - redactKtxScanMetadata, - redactKtxScanReport, - redactKtxScanWarning, -} from './credentials.js'; -export type { - KtxDataDictionaryColumnState, - KtxDataDictionarySampleDecision, - KtxDataDictionarySettings, - KtxDataDictionarySkipReason, -} from './data-dictionary.js'; -export { - defaultKtxDataDictionarySettings, - isKtxDataDictionaryCandidate, - shouldKtxSampleColumnForDictionary, -} from './data-dictionary.js'; -export type { - KtxColumnAnalysisResult, - KtxColumnDescriptionPromptInput, - KtxDataSourceDescriptionPromptInput, - KtxDescriptionCachePort, - KtxDescriptionColumn, - KtxDescriptionColumnTable, - KtxDescriptionGenerationSettings, - KtxDescriptionGeneratorOptions, - KtxDescriptionSamplingPort, - KtxDescriptionTableInput, - KtxGenerateColumnDescriptionsInput, - KtxGenerateDataSourceDescriptionInput, - KtxGenerateTableDescriptionInput, - KtxTableDescriptionPromptInput, -} from './description-generation.js'; -export { - buildKtxColumnDescriptionPrompt, - buildKtxDataSourceDescriptionPrompt, - buildKtxTableDescriptionPrompt, - KtxDescriptionGenerator, -} from './description-generation.js'; -export type { KtxColumnEmbeddingForeignKeys, KtxColumnEmbeddingTextInput } from './embedding-text.js'; -export { buildKtxColumnEmbeddingText } from './embedding-text.js'; -export type { - ComputeKtxScanEnrichmentInputHashInput, - KtxScanEnrichmentCompletedStage, - KtxScanEnrichmentFailedStage, - KtxScanEnrichmentStageLookup, - KtxScanEnrichmentStageRecord, - KtxScanEnrichmentStateStore, -} from './enrichment-state.js'; -export { - completedKtxScanEnrichmentStateSummary, - computeKtxScanEnrichmentInputHash, - KTX_SCAN_ENRICHMENT_STAGES, - summarizeKtxScanEnrichmentState, -} from './enrichment-state.js'; -export { - failedKtxScanEnrichmentSummary, - ktxScanErrorMessage, - skippedKtxScanEnrichmentSummary, -} from './enrichment-summary.js'; -export type { - KtxEntityDetailsColumn, - KtxEntityDetailsErrorCode, - KtxEntityDetailsErrorResult, - KtxEntityDetailsInput, - KtxEntityDetailsRecord, - KtxEntityDetailsResponse, - KtxEntityDetailsSnapshotInfo, - KtxEntityDetailsTableInput, -} from './entity-details.js'; -export { createKtxEntityDetailsService } from './entity-details.js'; -export type { - DisplayTargetResolution, - RawSchemaHit, - TableDetail, - WarehouseCatalogServiceDeps, -} from './warehouse-catalog.js'; -export { WarehouseCatalogService } from './warehouse-catalog.js'; -export type { - KtxColumnSampleUpdate, - KtxDescriptionSource, - KtxDescriptionUpdate, - KtxEmbeddingUpdate, - KtxEnrichedColumn, - KtxEnrichedRelationship, - KtxEnrichedSchema, - KtxEnrichedTable, - KtxRelationshipEndpoint, - KtxRelationshipSource, - KtxRelationshipType, - KtxRelationshipUpdate, - KtxScanMetadataStore, - KtxSkippedRelationship, - KtxStructuralSyncPlan, -} from './enrichment-types.js'; -export type { - KtxLocalScanEnrichmentInput, - KtxLocalScanEnrichmentProviders, - KtxLocalScanEnrichmentResult, -} from './local-enrichment.js'; -export { - createDeterministicLocalScanEnrichmentProviders, - runLocalScanEnrichment, - snapshotToKtxEnrichedSchema, -} from './local-enrichment.js'; -export type { - WriteLocalScanEnrichmentArtifactsInput, - WriteLocalScanEnrichmentArtifactsResult, - WriteLocalScanManifestShardsInput, - WriteLocalScanManifestShardsResult, -} from './local-enrichment-artifacts.js'; -export { - writeLocalScanEnrichmentArtifacts, - writeLocalScanManifestShards, -} from './local-enrichment-artifacts.js'; -export type { - LocalScanMcpOptions, - LocalScanRunResult, - LocalScanStatusResponse, - RunLocalScanOptions, -} from './local-scan.js'; -export { filterSnapshotTables, getLocalScanReport, getLocalScanStatus, resolveEnabledTables, runLocalScan } from './local-scan.js'; -export type { ReadLocalScanStructuralSnapshotInput } from './local-structural-artifacts.js'; -export { readLocalScanStructuralSnapshot } from './local-structural-artifacts.js'; -export type { - KtxRelationshipArtifactStatus, - ReadLocalScanRelationshipArtifactsResult, -} from './relationship-artifacts.js'; -export { readLocalScanRelationshipArtifacts } from './relationship-artifacts.js'; -export type { - KtxRelationshipBenchmarkReport, - KtxRelationshipBenchmarkReportCase, - KtxRelationshipBenchmarkReportCaseStatus, -} from './relationship-benchmark-report.js'; -export { - buildKtxRelationshipBenchmarkReport, - formatKtxRelationshipBenchmarkReportMarkdown, -} from './relationship-benchmark-report.js'; -export type { - KtxRelationshipBenchmarkCaseResult, - KtxRelationshipBenchmarkDetectedLink, - KtxRelationshipBenchmarkDetectedPk, - KtxRelationshipBenchmarkDetector, - KtxRelationshipBenchmarkDetectorInput, - KtxRelationshipBenchmarkDetectorResult, - KtxRelationshipBenchmarkExpectedLink, - KtxRelationshipBenchmarkExpectedLinks, - KtxRelationshipBenchmarkExpectedPk, - KtxRelationshipBenchmarkFixture, - KtxRelationshipBenchmarkMetrics, - KtxRelationshipBenchmarkMode, - KtxRelationshipBenchmarkStatus, - KtxRelationshipBenchmarkSuiteResult, - KtxRelationshipBenchmarkTier, -} from './relationship-benchmarks.js'; -export { - currentKtxRelationshipBenchmarkDetector, - ktxRelationshipBenchmarkDetectorWithLlm, - KTX_RELATIONSHIP_BENCHMARK_MODES, - KTX_RELATIONSHIP_BENCHMARK_TIERS, - loadKtxRelationshipBenchmarkFixture, - loadKtxRelationshipBenchmarkFixtures, - maskKtxRelationshipBenchmarkSnapshot, - runKtxRelationshipBenchmarkCase, - runKtxRelationshipBenchmarkSuite, -} from './relationship-benchmarks.js'; -export type { - ApplyKtxRelationshipValidationBudgetInput, - KtxRelationshipBudgetedCandidate, - KtxRelationshipValidationBudget, - KtxRelationshipValidationBudgetResult, -} from './relationship-budget.js'; -export { - applyKtxRelationshipValidationBudget, - defaultKtxRelationshipValidationBudget, -} from './relationship-budget.js'; -export type { - KtxRelationshipDiscoveryCandidate, - KtxRelationshipDiscoveryCandidateEvidence, - KtxRelationshipDiscoveryCandidateOptions, - KtxRelationshipDiscoveryCandidateSource, - KtxRelationshipDiscoveryCandidateStatus, - KtxRelationshipInferredTargetPk, -} from './relationship-candidates.js'; -export { - generateKtxRelationshipDiscoveryCandidates, - inferKtxRelationshipTargetPks, - mergeKtxRelationshipDiscoveryCandidates, -} from './relationship-candidates.js'; -export type { - DiscoverKtxCompositeRelationshipsInput, - DiscoverKtxCompositeRelationshipsResult, - KtxCompositePrimaryKeyCandidate, - KtxCompositeRelationshipCandidate, - KtxCompositeRelationshipStatus, - KtxCompositeRelationshipTupleEndpoint, - KtxCompositeRelationshipValidationEvidence, -} from './relationship-composite-candidates.js'; -export { discoverKtxCompositeRelationships } from './relationship-composite-candidates.js'; -export type { - BuildKtxRelationshipArtifactsInput, - BuildKtxRelationshipDiagnosticsInput, - EmptyKtxRelationshipProfileArtifactInput, - KtxRelationshipArtifact, - KtxRelationshipArtifactEdge, - KtxRelationshipArtifactEndpoint, - KtxRelationshipDiagnosticsArtifact, - KtxRelationshipDiagnosticsSummary, - KtxRelationshipDiagnosticsThresholds, - KtxRelationshipDiagnosticsValidation, -} from './relationship-diagnostics.js'; -export { - buildKtxRelationshipArtifacts, - buildKtxRelationshipDiagnostics, - emptyKtxRelationshipProfileArtifact, -} from './relationship-diagnostics.js'; -export type { - BuildKtxRelationshipFeedbackCalibrationReportInput, - CalibrateLocalRelationshipFeedbackLabelsInput, - KtxRelationshipFeedbackCalibrationBucket, - KtxRelationshipFeedbackCalibrationLabel, - KtxRelationshipFeedbackCalibrationReport, -} from './relationship-feedback-calibration.js'; -export { - buildKtxRelationshipFeedbackCalibrationReport, - calibrateLocalRelationshipFeedbackLabels, - formatKtxRelationshipFeedbackCalibrationMarkdown, -} from './relationship-feedback-calibration.js'; -export type { - ExportLocalRelationshipFeedbackLabelsInput, - ExportLocalRelationshipFeedbackLabelsResult, - KtxRelationshipFeedbackDecisionFilter, - KtxRelationshipFeedbackExportWarning, - KtxRelationshipFeedbackLabel, -} from './relationship-feedback-export.js'; -export { - exportLocalRelationshipFeedbackLabels, - formatKtxRelationshipFeedbackLabelsJsonl, -} from './relationship-feedback-export.js'; -export { - collectKtxFormalMetadataRelationships, - type KtxFormalMetadataRelationshipCollection, -} from './relationship-formal-metadata.js'; -export type { - KtxRelationshipGraphResolutionResult, - KtxRelationshipGraphResolverSettings, - KtxResolvedRelationshipDiscoveryCandidate, - KtxResolvedRelationshipGraphEvidence, - KtxResolvedRelationshipPk, - KtxResolvedRelationshipPkEvidence, - KtxResolvedRelationshipStatus, - ResolveKtxRelationshipGraphInput, -} from './relationship-graph-resolver.js'; -export { resolveKtxRelationshipGraph } from './relationship-graph-resolver.js'; -export type { - KtxRelationshipLlmProposalResult, - KtxRelationshipLlmProposalSettings, - ProposeKtxRelationshipCandidatesWithLlmInput, -} from './relationship-llm-proposal.js'; -export { proposeKtxRelationshipCandidatesWithLlm } from './relationship-llm-proposal.js'; -export type { - KtxRelationshipLocalityCandidateTable, - LocalKtxRelationshipCandidateTablesInput, -} from './relationship-locality.js'; -export { localCandidateTables } from './relationship-locality.js'; -export type { - KtxRelationshipNormalizedName, - KtxRelationshipTokenInput, -} from './relationship-name-similarity.js'; -export { - normalizeKtxRelationshipName, - pluralizeKtxRelationshipToken, - singularizeKtxRelationshipToken, - tokenizeKtxRelationshipName, - tokenSimilarity, -} from './relationship-name-similarity.js'; -export type { - DiscoverKtxRelationshipsInput, - DiscoverKtxRelationshipsResult, -} from './relationship-discovery.js'; -export { discoverKtxRelationships } from './relationship-discovery.js'; -export type { - KtxRelationshipColumnProfile, - KtxRelationshipProfileArtifact, - KtxRelationshipReadOnlyExecutor, - KtxRelationshipTableProfile, - ProfileKtxRelationshipSchemaInput, -} from './relationship-profiling.js'; -export { - formatKtxRelationshipTableRef, - profileKtxRelationshipSchema, - quoteKtxRelationshipIdentifier, -} from './relationship-profiling.js'; -export type { - AppliedRelationshipReviewDecision, - ApplyLocalScanRelationshipReviewDecisionsInput, - ApplyLocalScanRelationshipReviewDecisionsResult, -} from './relationship-review-apply.js'; -export { applyLocalScanRelationshipReviewDecisions } from './relationship-review-apply.js'; -export type { - KtxRelationshipReviewDecisionArtifact, - KtxRelationshipReviewDecisionEntry, - KtxRelationshipReviewDecisionValue, - WriteLocalScanRelationshipReviewDecisionInput, - WriteLocalScanRelationshipReviewDecisionResult, -} from './relationship-review-decisions.js'; -export { writeLocalScanRelationshipReviewDecision } from './relationship-review-decisions.js'; -export type { - KtxRelationshipFixtureOrigin, - KtxRelationshipScoreBreakdown, - KtxRelationshipScoreSignal, - KtxRelationshipScoreWeights, - KtxRelationshipScoringCalibrationObservation, - KtxRelationshipSignalVector, -} from './relationship-scoring.js'; -export { - calibrateWeightsFromSyntheticFixtures, - defaultKtxRelationshipScoreWeights, - KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS, - normalizeKtxRelationshipScoreWeights, - scoreKtxRelationshipCandidate, -} from './relationship-scoring.js'; -export type { - AdviseLocalRelationshipFeedbackThresholdsInput, - BuildKtxRelationshipThresholdAdviceReportInput, - KtxRelationshipThresholdAdviceCandidate, - KtxRelationshipThresholdAdviceReport, - KtxRelationshipThresholdAdviceStatus, -} from './relationship-threshold-advice.js'; -export { - adviseLocalRelationshipFeedbackThresholds, - buildKtxRelationshipThresholdAdviceReport, - formatKtxRelationshipThresholdAdviceMarkdown, -} from './relationship-threshold-advice.js'; -export type { - KtxRelationshipValidationEvidence, - KtxRelationshipValidationSettings, - KtxValidatedRelationshipDiscoveryCandidate, - KtxValidatedRelationshipStatus, - ValidateKtxRelationshipDiscoveryCandidatesInput, -} from './relationship-validation.js'; -export { validateKtxRelationshipDiscoveryCandidates } from './relationship-validation.js'; -export type { SqliteLocalScanEnrichmentStateStoreOptions } from './sqlite-local-enrichment-state-store.js'; -export { SqliteLocalScanEnrichmentStateStore } from './sqlite-local-enrichment-state-store.js'; -export type { KtxColumnTypeMapping } from './type-normalization.js'; -export { - inferKtxDimensionType, - ktxColumnTypeMappingFromNative, - normalizeKtxNativeType, -} from './type-normalization.js'; -export type { - KtxColumnSampleInput, - KtxColumnSampleResult, - KtxColumnStatsInput, - KtxColumnStatsResult, - KtxConnectionDriver, - KtxConnectorCapabilities, - KtxCredentialEnvelope, - KtxCredentialEnvReference, - KtxCredentialFileReference, - KtxEmbeddingPort, - KtxEventPropertyDiscovery, - KtxEventPropertyDiscoveryInput, - KtxEventPropertyValuesInput, - KtxEventPropertyValuesResult, - KtxEventStreamDiscoveryPort, - KtxEventTypeDiscovery, - KtxEventTypeDiscoveryInput, - KtxNetworkEndpoint, - KtxNetworkTunnelPort, - KtxNetworkTunnelRequest, - KtxOptionalConnectorCapabilities, - KtxProgressPort, - KtxProgressUpdateOptions, - KtxQueryResult, - KtxReadOnlyQueryInput, - KtxResolvedCredentialEnvelope, - KtxConnectorTestResult, - KtxScanArtifactPaths, - KtxScanConnector, - KtxScanContext, - KtxScanDiffSummary, - KtxScanEnrichmentStage, - KtxScanEnrichmentStateSummary, - KtxScanEnrichmentSummary, - KtxScanInput, - KtxScanLoggerPort, - KtxScanMode, - KtxScanRelationshipSummary, - KtxScanReport, - KtxScanTrigger, - KtxScanWarning, - KtxScanWarningCode, - KtxSchemaColumn, - KtxSchemaDimensionType, - KtxSchemaForeignKey, - KtxSchemaScope, - KtxSchemaSnapshot, - KtxSchemaTable, - KtxSchemaTableKind, - KtxStructuralSyncStats, - KtxTableListEntry, - KtxTableRef, - KtxTableSampleInput, - KtxTableSampleResult, -} from './types.js'; -export { createKtxConnectorCapabilities } from './types.js'; diff --git a/packages/context/src/scan/relationship-artifacts.test.ts b/packages/context/src/scan/relationship-artifacts.test.ts deleted file mode 100644 index f66de0c2..00000000 --- a/packages/context/src/scan/relationship-artifacts.test.ts +++ /dev/null @@ -1,308 +0,0 @@ -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; -import { runLocalStageOnlyIngest, type SourceAdapter } from '../ingest/index.js'; -import { initKtxProject, loadKtxProject } from '../project/index.js'; -import { describe, expect, it } from 'vitest'; -import { readLocalScanRelationshipArtifacts } from './relationship-artifacts.js'; -import type { KtxRelationshipArtifact, KtxRelationshipDiagnosticsArtifact } from './relationship-diagnostics.js'; -import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js'; -import type { KtxScanReport } from './types.js'; - -async function writeProjectFile(projectDir: string, relativePath: string, content: string): Promise { - const absolutePath = join(projectDir, relativePath); - await mkdir(dirname(absolutePath), { recursive: true }); - await writeFile(absolutePath, content, 'utf-8'); -} - -async function writeWarehouseConfig(projectDir: string): Promise { - await writeFile( - join(projectDir, 'ktx.yaml'), - [ - 'connections:', - ' warehouse:', - ' driver: sqlite', - ' path: warehouse.db', - 'ingest:', - ' adapters:', - ' - live-database', - '', - ].join('\n'), - 'utf-8', - ); -} - -function liveDatabaseAdapter(): SourceAdapter { - return { - source: 'live-database', - skillNames: ['live_database_ingest'], - async fetch(_pullConfig, stagedDir) { - await mkdir(join(stagedDir, 'tables'), { recursive: true }); - await writeFile(join(stagedDir, 'connection.json'), '{"connectionId":"warehouse"}\n', 'utf-8'); - await writeFile(join(stagedDir, 'foreign-keys.json'), '{"foreignKeys":[]}\n', 'utf-8'); - await writeFile( - join(stagedDir, 'tables', 'orders.json'), - '{"name":"orders","db":"public","columns":[{"name":"id","type":"integer","nullable":false,"primaryKey":true}]}\n', - 'utf-8', - ); - }, - async detect(stagedDir) { - await writeFile(join(stagedDir, 'connection.json'), '{"connectionId":"warehouse"}\n', 'utf-8'); - return true; - }, - async chunk() { - return { - workUnits: [ - { - unitKey: 'live-database-public-orders', - rawFiles: ['tables/orders.json'], - dependencyPaths: ['connection.json', 'foreign-keys.json'], - peerFileIndex: [], - }, - ], - }; - }, - }; -} - -async function createLiveDatabaseRun(projectDir: string, runId: string) { - await initKtxProject({ projectDir }); - await writeWarehouseConfig(projectDir); - const project = await loadKtxProject({ projectDir }); - await runLocalStageOnlyIngest({ - project, - adapters: [liveDatabaseAdapter()], - adapter: 'live-database', - connectionId: 'warehouse', - jobId: runId, - now: () => new Date('2026-05-07T10:00:00.000Z'), - }); - return project; -} - -function scanReport(enrichmentArtifacts: string[], syncId = '2026-05-07-100000-scan-run-review'): KtxScanReport { - return { - connectionId: 'warehouse', - driver: 'sqlite', - syncId, - runId: 'scan-run-review', - trigger: 'cli', - mode: 'relationships', - dryRun: false, - artifactPaths: { - rawSourcesDir: `raw-sources/warehouse/live-database/${syncId}`, - reportPath: `raw-sources/warehouse/live-database/${syncId}/scan-report.json`, - manifestShards: [], - enrichmentArtifacts, - }, - diffSummary: { - tablesAdded: 0, - tablesModified: 0, - tablesDeleted: 0, - tablesUnchanged: 2, - columnsAdded: 0, - columnsModified: 0, - columnsDeleted: 0, - }, - manifestShardsWritten: 0, - structuralSyncStats: { - tablesCreated: 0, - tablesUpdated: 0, - tablesDeleted: 0, - columnsCreated: 0, - columnsUpdated: 0, - columnsDeleted: 0, - }, - enrichment: { - dataDictionary: 'skipped', - tableDescriptions: 'skipped', - columnDescriptions: 'skipped', - embeddings: 'skipped', - deterministicRelationships: 'completed', - llmRelationshipValidation: 'skipped', - statisticalValidation: 'skipped', - }, - capabilityGaps: [], - warnings: [], - relationships: { accepted: 0, review: 1, rejected: 1, skipped: 0 }, - enrichmentState: { - resumedStages: [], - completedStages: ['relationships'], - failedStages: [], - }, - createdAt: '2026-05-07T10:00:00.000Z', - }; -} - -const relationshipArtifact: KtxRelationshipArtifact = { - connectionId: 'warehouse', - accepted: [], - review: [ - { - id: 'orders:orders.customer_id->customers:customers.id', - status: 'review', - source: 'deterministic_name', - from: { - tableId: 'orders', - columnIds: ['orders.customer_id'], - table: { catalog: null, db: 'public', name: 'orders' }, - columns: ['customer_id'], - }, - to: { - tableId: 'customers', - columnIds: ['customers.id'], - table: { catalog: null, db: 'public', name: 'customers' }, - columns: ['id'], - }, - relationshipType: 'many_to_one', - confidence: 0.62, - pkScore: 0.91, - fkScore: 0.62, - score: 0.62, - evidence: { sources: ['table_suffix'] }, - validation: { status: 'unavailable' }, - graph: { reasons: ['validation_unavailable_review_only'] }, - reasons: ['validation_unavailable_review_only', 'fk_score_review'], - }, - ], - rejected: [ - { - id: 'orders:orders.note_id->notes:notes.id', - status: 'rejected', - source: 'deterministic_name', - from: { - tableId: 'orders', - columnIds: ['orders.note_id'], - table: { catalog: null, db: 'public', name: 'orders' }, - columns: ['note_id'], - }, - to: { - tableId: 'notes', - columnIds: ['notes.id'], - table: { catalog: null, db: 'public', name: 'notes' }, - columns: ['id'], - }, - relationshipType: 'many_to_one', - confidence: 0.2, - pkScore: 0.4, - fkScore: 0.2, - score: 0.2, - evidence: { sources: ['exact_column_match'] }, - validation: { status: 'failed' }, - graph: { reasons: ['low_source_coverage'] }, - reasons: ['low_source_coverage'], - }, - ], - skipped: [], -}; - -const diagnosticsArtifact: KtxRelationshipDiagnosticsArtifact = { - connectionId: 'warehouse', - generatedAt: '2026-05-07T10:00:00.000Z', - summary: { accepted: 0, review: 1, rejected: 1, skipped: 0 }, - noAcceptedReason: 'relationship candidates require review before manifest writes', - candidateCountsBySource: { deterministic_name: 2 }, - validation: { available: false, sqlAvailable: false, queryCount: 0 }, - thresholds: { acceptThreshold: 0.85, reviewThreshold: 0.55 }, - policy: { - validationRequiredForManifest: true, - maxCandidatesPerColumn: 25, - profileSampleRows: 10000, - validationConcurrency: 4, - }, - warnings: [], - profileWarnings: ['KTX scan connector cannot run read-only SQL relationship validation'], -}; - -const profileArtifact: KtxRelationshipProfileArtifact = { - connectionId: 'warehouse', - driver: 'sqlite', - sqlAvailable: false, - tables: [], - columns: {}, - queryCount: 0, - warnings: ['KTX scan connector cannot run read-only SQL relationship validation'], -}; - -describe('local scan relationship artifact reader', () => { - it('loads relationship, diagnostics, and profile artifacts for a scan run', async () => { - const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-artifacts-')); - try { - const project = await createLiveDatabaseRun(projectDir, 'scan-run-review'); - const syncId = '2026-05-07-100000-scan-run-review'; - const report = scanReport( - [ - `raw-sources/warehouse/live-database/${syncId}/enrichment/relationships.json`, - `raw-sources/warehouse/live-database/${syncId}/enrichment/relationship-profile.json`, - `raw-sources/warehouse/live-database/${syncId}/enrichment/relationship-diagnostics.json`, - ], - syncId, - ); - await writeProjectFile(projectDir, report.artifactPaths.reportPath ?? '', `${JSON.stringify(report, null, 2)}\n`); - await writeProjectFile( - projectDir, - `raw-sources/warehouse/live-database/${syncId}/enrichment/relationships.json`, - `${JSON.stringify(relationshipArtifact, null, 2)}\n`, - ); - await writeProjectFile( - projectDir, - `raw-sources/warehouse/live-database/${syncId}/enrichment/relationship-diagnostics.json`, - `${JSON.stringify(diagnosticsArtifact, null, 2)}\n`, - ); - await writeProjectFile( - projectDir, - `raw-sources/warehouse/live-database/${syncId}/enrichment/relationship-profile.json`, - `${JSON.stringify(profileArtifact, null, 2)}\n`, - ); - - const result = await readLocalScanRelationshipArtifacts(project, 'scan-run-review'); - - expect(result).toMatchObject({ - runId: 'scan-run-review', - connectionId: 'warehouse', - syncId, - paths: { - relationships: `raw-sources/warehouse/live-database/${syncId}/enrichment/relationships.json`, - diagnostics: `raw-sources/warehouse/live-database/${syncId}/enrichment/relationship-diagnostics.json`, - profile: `raw-sources/warehouse/live-database/${syncId}/enrichment/relationship-profile.json`, - }, - }); - expect(result?.relationships.review[0]).toMatchObject({ - id: 'orders:orders.customer_id->customers:customers.id', - status: 'review', - reasons: ['validation_unavailable_review_only', 'fk_score_review'], - }); - expect(result?.diagnostics?.noAcceptedReason).toBe('relationship candidates require review before manifest writes'); - expect(result?.profile?.sqlAvailable).toBe(false); - } finally { - await rm(projectDir, { recursive: true, force: true }); - } - }); - - it('returns null when the scan run has no report', async () => { - const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-artifacts-missing-run-')); - try { - await initKtxProject({ projectDir }); - const project = await loadKtxProject({ projectDir }); - - await expect(readLocalScanRelationshipArtifacts(project, 'missing-run')).resolves.toBeNull(); - } finally { - await rm(projectDir, { recursive: true, force: true }); - } - }); - - it('throws a focused error when a scan report does not reference relationships.json', async () => { - const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-artifacts-missing-artifact-')); - try { - const project = await createLiveDatabaseRun(projectDir, 'scan-run-review'); - const report = scanReport([]); - await writeProjectFile(projectDir, report.artifactPaths.reportPath ?? '', `${JSON.stringify(report, null, 2)}\n`); - - await expect(readLocalScanRelationshipArtifacts(project, 'scan-run-review')).rejects.toThrow( - 'Scan report "scan-run-review" does not reference relationships.json', - ); - } finally { - await rm(projectDir, { recursive: true, force: true }); - } - }); -}); diff --git a/packages/context/src/scan/relationship-artifacts.ts b/packages/context/src/scan/relationship-artifacts.ts deleted file mode 100644 index 99751b71..00000000 --- a/packages/context/src/scan/relationship-artifacts.ts +++ /dev/null @@ -1,75 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import { getLocalScanReport } from './local-scan.js'; -import type { KtxRelationshipArtifact, KtxRelationshipDiagnosticsArtifact } from './relationship-diagnostics.js'; -import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js'; -import type { KtxScanReport } from './types.js'; - -export type KtxRelationshipArtifactStatus = 'accepted' | 'review' | 'rejected' | 'skipped' | 'all'; - -export interface ReadLocalScanRelationshipArtifactsResult { - runId: string; - connectionId: string; - syncId: string; - report: KtxScanReport; - relationships: KtxRelationshipArtifact; - diagnostics: KtxRelationshipDiagnosticsArtifact | null; - profile: KtxRelationshipProfileArtifact | null; - paths: { - relationships: string; - diagnostics: string | null; - profile: string | null; - }; -} - -function findArtifactPath(report: KtxScanReport, fileName: string): string | null { - return report.artifactPaths.enrichmentArtifacts.find((path) => path.endsWith(`/enrichment/${fileName}`)) ?? null; -} - -async function readJsonArtifact(project: KtxLocalProject, path: string): Promise { - const raw = await project.fileStore.readFile(path); - return JSON.parse(raw.content) as T; -} - -async function readOptionalJsonArtifact(project: KtxLocalProject, path: string | null): Promise { - if (!path) { - return null; - } - try { - return await readJsonArtifact(project, path); - } catch { - return null; - } -} - -export async function readLocalScanRelationshipArtifacts( - project: KtxLocalProject, - runId: string, -): Promise { - const report = await getLocalScanReport(project, runId); - if (!report) { - return null; - } - - const relationshipsPath = findArtifactPath(report, 'relationships.json'); - if (!relationshipsPath) { - throw new Error(`Scan report "${runId}" does not reference relationships.json`); - } - - const diagnosticsPath = findArtifactPath(report, 'relationship-diagnostics.json'); - const profilePath = findArtifactPath(report, 'relationship-profile.json'); - - return { - runId, - connectionId: report.connectionId, - syncId: report.syncId, - report, - relationships: await readJsonArtifact(project, relationshipsPath), - diagnostics: await readOptionalJsonArtifact(project, diagnosticsPath), - profile: await readOptionalJsonArtifact(project, profilePath), - paths: { - relationships: relationshipsPath, - diagnostics: diagnosticsPath, - profile: profilePath, - }, - }; -} diff --git a/packages/context/src/scan/relationship-feedback-calibration.test.ts b/packages/context/src/scan/relationship-feedback-calibration.test.ts deleted file mode 100644 index 43090ad4..00000000 --- a/packages/context/src/scan/relationship-feedback-calibration.test.ts +++ /dev/null @@ -1,211 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import { describe, expect, it, vi } from 'vitest'; -import { - buildKtxRelationshipFeedbackCalibrationReport, - calibrateLocalRelationshipFeedbackLabels, - formatKtxRelationshipFeedbackCalibrationMarkdown, -} from './relationship-feedback-calibration.js'; -import type { - ExportLocalRelationshipFeedbackLabelsResult, - KtxRelationshipFeedbackLabel, -} from './relationship-feedback-export.js'; - -function label( - input: Partial & - Pick, -): KtxRelationshipFeedbackLabel { - return { - schemaVersion: 1, - previousStatus: 'review', - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decidedAt: '2026-05-07T12:00:00.000Z', - reviewer: 'Andrey', - note: null, - relationshipType: 'many_to_one', - source: 'deterministic_name', - confidence: input.score ?? 0, - pkScore: input.pkScore ?? null, - fkScore: input.fkScore ?? input.score, - fromTable: 'public.orders', - fromColumns: ['customer_id'], - toTable: 'public.customers', - toColumns: ['id'], - reasons: [], - artifactPath: 'raw-sources/warehouse/live-database/sync-a/enrichment/relationship-review-decisions.json', - ...input, - }; -} - -function feedback(labels: KtxRelationshipFeedbackLabel[]): ExportLocalRelationshipFeedbackLabelsResult { - return { - generatedAt: '2026-05-07T13:00:00.000Z', - filters: { connectionId: null, decision: 'all' }, - summary: { - total: labels.length, - accepted: labels.filter((item) => item.decision === 'accepted').length, - rejected: labels.filter((item) => item.decision === 'rejected').length, - connections: new Set(labels.map((item) => item.connectionId)).size, - runs: new Set(labels.map((item) => `${item.connectionId}:${item.runId}`)).size, - }, - labels, - warnings: [], - }; -} - -describe('relationship feedback calibration', () => { - it('builds score buckets and threshold-band summary from feedback labels', () => { - const report = buildKtxRelationshipFeedbackCalibrationReport( - feedback([ - label({ - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted', - score: 0.91, - pkScore: 0.97, - fkScore: 0.91, - }), - label({ - candidateId: 'orders:orders.account_id->accounts:accounts.id', - decision: 'accepted', - score: 0.61, - pkScore: 0.88, - fkScore: 0.61, - }), - label({ - candidateId: 'orders:orders.note_id->notes:notes.id', - decision: 'rejected', - score: 0.21, - pkScore: 0.4, - fkScore: 0.21, - }), - label({ - candidateId: 'orders:orders.region_id->regions:regions.id', - decision: 'rejected', - score: 0.88, - pkScore: 0.9, - fkScore: 0.88, - }), - ]), - { - acceptThreshold: 0.85, - reviewThreshold: 0.55, - }, - ); - - expect(report.thresholds).toEqual({ accept: 0.85, review: 0.55 }); - expect(report.summary).toEqual({ - total: 4, - scored: 4, - unscored: 0, - acceptedLabels: 2, - rejectedLabels: 2, - predictedAccepted: 2, - predictedReview: 1, - predictedRejected: 1, - acceptedBandPrecision: 0.5, - rejectedBandPrecision: 1, - reviewBandAcceptedRate: 1, - meanAcceptedScore: 0.76, - meanRejectedScore: 0.545, - }); - expect(report.buckets.map((bucket) => [bucket.label, bucket.total, bucket.accepted, bucket.rejected, bucket.acceptanceRate])).toEqual([ - ['0.00-0.24', 1, 0, 1, 0], - ['0.25-0.49', 0, 0, 0, null], - ['0.50-0.74', 1, 1, 0, 1], - ['0.75-1.00', 2, 1, 1, 0.5], - ]); - expect(report.labels.map((item) => [item.candidateId, item.predictedStatus, item.bucket])).toEqual([ - ['orders:orders.account_id->accounts:accounts.id', 'review', '0.50-0.74'], - ['orders:orders.customer_id->customers:customers.id', 'accepted', '0.75-1.00'], - ['orders:orders.note_id->notes:notes.id', 'rejected', '0.00-0.24'], - ['orders:orders.region_id->regions:regions.id', 'accepted', '0.75-1.00'], - ]); - }); - - it('keeps unscored labels visible without treating them as threshold predictions', () => { - const report = buildKtxRelationshipFeedbackCalibrationReport( - feedback([ - label({ - candidateId: 'orders:orders.note_id->notes:notes.id', - decision: 'rejected', - score: null, - confidence: 0.2, - fkScore: null, - }), - ]), - { - acceptThreshold: 0.85, - reviewThreshold: 0.55, - }, - ); - - expect(report.summary).toMatchObject({ - total: 1, - scored: 0, - unscored: 1, - predictedAccepted: 0, - predictedReview: 0, - predictedRejected: 0, - acceptedBandPrecision: null, - rejectedBandPrecision: null, - reviewBandAcceptedRate: null, - meanAcceptedScore: null, - meanRejectedScore: null, - }); - expect(report.labels[0]).toMatchObject({ - candidateId: 'orders:orders.note_id->notes:notes.id', - predictedStatus: 'unscored', - bucket: 'unscored', - }); - }); - - it('formats a stable markdown summary for human CLI output', () => { - const report = buildKtxRelationshipFeedbackCalibrationReport( - feedback([ - label({ candidateId: 'orders:orders.customer_id->customers:customers.id', decision: 'accepted', score: 0.91 }), - label({ candidateId: 'orders:orders.note_id->notes:notes.id', decision: 'rejected', score: 0.21 }), - ]), - { - acceptThreshold: 0.85, - reviewThreshold: 0.55, - }, - ); - - expect(formatKtxRelationshipFeedbackCalibrationMarkdown(report)).toContain( - 'KTX relationship feedback calibration', - ); - expect(formatKtxRelationshipFeedbackCalibrationMarkdown(report)).toContain('Total labels: 2'); - expect(formatKtxRelationshipFeedbackCalibrationMarkdown(report)).toContain('Accepted-band precision: 1.000'); - expect(formatKtxRelationshipFeedbackCalibrationMarkdown(report)).toContain( - '0.75-1.00: total=1 accepted=1 rejected=0 acceptanceRate=1.000', - ); - }); - - it('wraps the feedback exporter and preserves exporter warnings', async () => { - const project = { projectDir: '/tmp/ktx-project' } as KtxLocalProject; - const exportLocalRelationshipFeedbackLabels = vi.fn(async () => ({ - ...feedback([ - label({ candidateId: 'orders:orders.customer_id->customers:customers.id', decision: 'accepted', score: 0.91 }), - ]), - warnings: [{ path: 'raw-sources/broken/live-database/sync/enrichment/relationship-review-decisions.json', message: 'Unexpected token' }], - })); - - const report = await calibrateLocalRelationshipFeedbackLabels(project, { - connectionId: 'warehouse', - decision: 'all', - acceptThreshold: 0.9, - reviewThreshold: 0.5, - exportLocalRelationshipFeedbackLabels, - }); - - expect(exportLocalRelationshipFeedbackLabels).toHaveBeenCalledWith(project, { - connectionId: 'warehouse', - decision: 'all', - }); - expect(report.thresholds).toEqual({ accept: 0.9, review: 0.5 }); - expect(report.warnings).toEqual([ - { path: 'raw-sources/broken/live-database/sync/enrichment/relationship-review-decisions.json', message: 'Unexpected token' }, - ]); - }); -}); diff --git a/packages/context/src/scan/relationship-feedback-calibration.ts b/packages/context/src/scan/relationship-feedback-calibration.ts deleted file mode 100644 index 5d471907..00000000 --- a/packages/context/src/scan/relationship-feedback-calibration.ts +++ /dev/null @@ -1,300 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import { - exportLocalRelationshipFeedbackLabels, - type ExportLocalRelationshipFeedbackLabelsInput, - type ExportLocalRelationshipFeedbackLabelsResult, - type KtxRelationshipFeedbackExportWarning, - type KtxRelationshipFeedbackLabel, -} from './relationship-feedback-export.js'; -import type { KtxResolvedRelationshipStatus } from './relationship-graph-resolver.js'; -import type { KtxRelationshipReviewDecisionValue } from './relationship-review-decisions.js'; - -const DEFAULT_ACCEPT_THRESHOLD = 0.85; -const DEFAULT_REVIEW_THRESHOLD = 0.55; - -type CalibrationPredictedStatus = KtxResolvedRelationshipStatus | 'unscored'; - -interface Thresholds { - accept: number; - review: number; -} - -export interface BuildKtxRelationshipFeedbackCalibrationReportInput { - acceptThreshold?: number; - reviewThreshold?: number; -} - -export interface CalibrateLocalRelationshipFeedbackLabelsInput - extends ExportLocalRelationshipFeedbackLabelsInput, - BuildKtxRelationshipFeedbackCalibrationReportInput { - exportLocalRelationshipFeedbackLabels?: typeof exportLocalRelationshipFeedbackLabels; -} - -export interface KtxRelationshipFeedbackCalibrationBucket { - label: string; - minInclusive: number; - maxInclusive: number; - total: number; - accepted: number; - rejected: number; - acceptanceRate: number | null; -} - -export interface KtxRelationshipFeedbackCalibrationLabel { - candidateId: string; - decision: KtxRelationshipReviewDecisionValue; - previousStatus: KtxRelationshipFeedbackLabel['previousStatus']; - predictedStatus: CalibrationPredictedStatus; - bucket: string; - score: number | null; - pkScore: number | null; - fkScore: number | null; - connectionId: string; - runId: string; - fromTable: string; - fromColumns: string[]; - toTable: string; - toColumns: string[]; - source: string; - reasons: string[]; -} - -export interface KtxRelationshipFeedbackCalibrationReport { - generatedAt: string; - filters: ExportLocalRelationshipFeedbackLabelsResult['filters']; - thresholds: Thresholds; - summary: { - total: number; - scored: number; - unscored: number; - acceptedLabels: number; - rejectedLabels: number; - predictedAccepted: number; - predictedReview: number; - predictedRejected: number; - acceptedBandPrecision: number | null; - rejectedBandPrecision: number | null; - reviewBandAcceptedRate: number | null; - meanAcceptedScore: number | null; - meanRejectedScore: number | null; - }; - buckets: KtxRelationshipFeedbackCalibrationBucket[]; - labels: KtxRelationshipFeedbackCalibrationLabel[]; - warnings: KtxRelationshipFeedbackExportWarning[]; -} - -const BUCKETS = [ - { label: '0.00-0.24', minInclusive: 0, maxInclusive: 0.249999 }, - { label: '0.25-0.49', minInclusive: 0.25, maxInclusive: 0.499999 }, - { label: '0.50-0.74', minInclusive: 0.5, maxInclusive: 0.749999 }, - { label: '0.75-1.00', minInclusive: 0.75, maxInclusive: 1 }, -] as const; - -function thresholds(input: BuildKtxRelationshipFeedbackCalibrationReportInput): Thresholds { - return { - accept: input.acceptThreshold ?? DEFAULT_ACCEPT_THRESHOLD, - review: input.reviewThreshold ?? DEFAULT_REVIEW_THRESHOLD, - }; -} - -function roundMetric(value: number): number { - return Math.round(value * 1000) / 1000; -} - -function ratio(numerator: number, denominator: number): number | null { - return denominator === 0 ? null : roundMetric(numerator / denominator); -} - -function mean(values: readonly number[]): number | null { - if (values.length === 0) { - return null; - } - return roundMetric(values.reduce((sum, value) => sum + value, 0) / values.length); -} - -function scoreBucket(score: number | null): string { - if (score === null) { - return 'unscored'; - } - return BUCKETS.find((bucket) => score >= bucket.minInclusive && score <= bucket.maxInclusive)?.label ?? 'unscored'; -} - -function predictedStatus(score: number | null, currentThresholds: Thresholds): CalibrationPredictedStatus { - if (score === null) { - return 'unscored'; - } - if (score >= currentThresholds.accept) { - return 'accepted'; - } - if (score >= currentThresholds.review) { - return 'review'; - } - return 'rejected'; -} - -function calibrationLabel( - label: KtxRelationshipFeedbackLabel, - currentThresholds: Thresholds, -): KtxRelationshipFeedbackCalibrationLabel { - return { - candidateId: label.candidateId, - decision: label.decision, - previousStatus: label.previousStatus, - predictedStatus: predictedStatus(label.score, currentThresholds), - bucket: scoreBucket(label.score), - score: label.score, - pkScore: label.pkScore, - fkScore: label.fkScore, - connectionId: label.connectionId, - runId: label.runId, - fromTable: label.fromTable, - fromColumns: [...label.fromColumns], - toTable: label.toTable, - toColumns: [...label.toColumns], - source: label.source, - reasons: [...label.reasons], - }; -} - -function summarize( - labels: readonly KtxRelationshipFeedbackCalibrationLabel[], -): KtxRelationshipFeedbackCalibrationReport['summary'] { - const scored = labels.filter((label) => label.score !== null); - const predictedAccepted = scored.filter((label) => label.predictedStatus === 'accepted'); - const predictedReview = scored.filter((label) => label.predictedStatus === 'review'); - const predictedRejected = scored.filter((label) => label.predictedStatus === 'rejected'); - const acceptedLabels = labels.filter((label) => label.decision === 'accepted'); - const rejectedLabels = labels.filter((label) => label.decision === 'rejected'); - - return { - total: labels.length, - scored: scored.length, - unscored: labels.length - scored.length, - acceptedLabels: acceptedLabels.length, - rejectedLabels: rejectedLabels.length, - predictedAccepted: predictedAccepted.length, - predictedReview: predictedReview.length, - predictedRejected: predictedRejected.length, - acceptedBandPrecision: ratio( - predictedAccepted.filter((label) => label.decision === 'accepted').length, - predictedAccepted.length, - ), - rejectedBandPrecision: ratio( - predictedRejected.filter((label) => label.decision === 'rejected').length, - predictedRejected.length, - ), - reviewBandAcceptedRate: ratio( - predictedReview.filter((label) => label.decision === 'accepted').length, - predictedReview.length, - ), - meanAcceptedScore: mean(acceptedLabels.map((label) => label.score).filter((score): score is number => score !== null)), - meanRejectedScore: mean(rejectedLabels.map((label) => label.score).filter((score): score is number => score !== null)), - }; -} - -function buildBuckets( - labels: readonly KtxRelationshipFeedbackCalibrationLabel[], -): KtxRelationshipFeedbackCalibrationBucket[] { - return BUCKETS.map((bucket) => { - const bucketLabels = labels.filter((label) => label.bucket === bucket.label); - const accepted = bucketLabels.filter((label) => label.decision === 'accepted').length; - const rejected = bucketLabels.filter((label) => label.decision === 'rejected').length; - return { - label: bucket.label, - minInclusive: bucket.minInclusive, - maxInclusive: - bucket.maxInclusive === 0.249999 - ? 0.24 - : bucket.maxInclusive === 0.499999 - ? 0.49 - : bucket.maxInclusive === 0.749999 - ? 0.74 - : 1, - total: bucketLabels.length, - accepted, - rejected, - acceptanceRate: ratio(accepted, bucketLabels.length), - }; - }); -} - -export function buildKtxRelationshipFeedbackCalibrationReport( - feedback: ExportLocalRelationshipFeedbackLabelsResult, - input: BuildKtxRelationshipFeedbackCalibrationReportInput = {}, -): KtxRelationshipFeedbackCalibrationReport { - const currentThresholds = thresholds(input); - const labels = feedback.labels - .map((label) => calibrationLabel(label, currentThresholds)) - .sort( - (left, right) => - left.connectionId.localeCompare(right.connectionId) || - left.runId.localeCompare(right.runId) || - left.candidateId.localeCompare(right.candidateId), - ); - - return { - generatedAt: feedback.generatedAt, - filters: feedback.filters, - thresholds: currentThresholds, - summary: summarize(labels), - buckets: buildBuckets(labels), - labels, - warnings: [...feedback.warnings], - }; -} - -export async function calibrateLocalRelationshipFeedbackLabels( - project: KtxLocalProject, - input: CalibrateLocalRelationshipFeedbackLabelsInput = {}, -): Promise { - const exporter = input.exportLocalRelationshipFeedbackLabels ?? exportLocalRelationshipFeedbackLabels; - const feedback = await exporter(project, { - connectionId: input.connectionId, - decision: input.decision, - }); - return buildKtxRelationshipFeedbackCalibrationReport(feedback, input); -} - -function formatMetric(value: number | null): string { - return value === null ? 'n/a' : value.toFixed(3); -} - -export function formatKtxRelationshipFeedbackCalibrationMarkdown( - report: KtxRelationshipFeedbackCalibrationReport, -): string { - const lines = [ - 'KTX relationship feedback calibration', - `Generated: ${report.generatedAt}`, - `Filter connection: ${report.filters.connectionId ?? 'all'}`, - `Filter decision: ${report.filters.decision}`, - `Thresholds: accept=${report.thresholds.accept.toFixed(2)} review=${report.thresholds.review.toFixed(2)}`, - `Total labels: ${report.summary.total}`, - `Scored labels: ${report.summary.scored}`, - `Unscored labels: ${report.summary.unscored}`, - `Accepted labels: ${report.summary.acceptedLabels}`, - `Rejected labels: ${report.summary.rejectedLabels}`, - `Predicted accepted: ${report.summary.predictedAccepted}`, - `Predicted review: ${report.summary.predictedReview}`, - `Predicted rejected: ${report.summary.predictedRejected}`, - `Accepted-band precision: ${formatMetric(report.summary.acceptedBandPrecision)}`, - `Rejected-band precision: ${formatMetric(report.summary.rejectedBandPrecision)}`, - `Review-band accepted rate: ${formatMetric(report.summary.reviewBandAcceptedRate)}`, - `Mean accepted score: ${formatMetric(report.summary.meanAcceptedScore)}`, - `Mean rejected score: ${formatMetric(report.summary.meanRejectedScore)}`, - '', - 'Score buckets', - ...report.buckets.map( - (bucket) => - ` - ${bucket.label}: total=${bucket.total} accepted=${bucket.accepted} rejected=${bucket.rejected} acceptanceRate=${formatMetric(bucket.acceptanceRate)}`, - ), - ]; - - if (report.warnings.length > 0) { - lines.push('', 'Warnings'); - for (const warning of report.warnings.slice(0, 5)) { - lines.push(` - ${warning.path}: ${warning.message}`); - } - } - - return `${lines.join('\n')}\n`; -} diff --git a/packages/context/src/scan/relationship-feedback-export.test.ts b/packages/context/src/scan/relationship-feedback-export.test.ts deleted file mode 100644 index 512c29ad..00000000 --- a/packages/context/src/scan/relationship-feedback-export.test.ts +++ /dev/null @@ -1,270 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import { describe, expect, it, vi } from 'vitest'; -import { - exportLocalRelationshipFeedbackLabels, - formatKtxRelationshipFeedbackLabelsJsonl, -} from './relationship-feedback-export.js'; -import type { KtxRelationshipReviewDecisionArtifact } from './relationship-review-decisions.js'; - -function projectWithFiles(files: Record): KtxLocalProject { - const contentByPath = new Map( - Object.entries(files).map(([path, value]) => [ - path, - typeof value === 'string' ? value : `${JSON.stringify(value, null, 2)}\n`, - ]), - ); - return { - projectDir: '/tmp/ktx-project', - fileStore: { - async listFiles(path: string) { - return { - files: [...contentByPath.keys()].filter((file) => file.startsWith(`${path}/`)).sort(), - }; - }, - async readFile(path: string) { - const content = contentByPath.get(path); - if (!content) { - throw new Error(`missing file ${path}`); - } - return { content }; - }, - writeFile: vi.fn(), - deleteFile: vi.fn(), - getFileHistory: vi.fn(), - forWorktree: vi.fn(), - }, - } as unknown as KtxLocalProject; -} - -function decisionsArtifact(input: { - connectionId: string; - runId: string; - syncId: string; - decisions: KtxRelationshipReviewDecisionArtifact['decisions']; -}): KtxRelationshipReviewDecisionArtifact { - return { - connectionId: input.connectionId, - runId: input.runId, - syncId: input.syncId, - generatedAt: '2026-05-07T12:00:00.000Z', - decisions: input.decisions, - }; -} - -const acceptedOrderCustomer = { - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted' as const, - previousStatus: 'review' as const, - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decidedAt: '2026-05-07T12:00:00.000Z', - reviewer: 'Andrey', - note: 'Confirmed in warehouse docs', - from: { - tableId: 'orders', - columnIds: ['orders.customer_id'], - table: { catalog: null, db: 'public', name: 'orders' }, - columns: ['customer_id'], - }, - to: { - tableId: 'customers', - columnIds: ['customers.id'], - table: { catalog: null, db: 'public', name: 'customers' }, - columns: ['id'], - }, - relationshipType: 'many_to_one' as const, - source: 'deterministic_name', - score: 0.62, - confidence: 0.62, - pkScore: 0.91, - fkScore: 0.62, - reasons: ['fk_score_review'], -}; - -const rejectedOrderNote = { - candidateId: 'orders:orders.note_id->notes:notes.id', - decision: 'rejected' as const, - previousStatus: 'rejected' as const, - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decidedAt: '2026-05-07T12:05:00.000Z', - reviewer: 'Andrey', - note: null, - from: { - tableId: 'orders', - columnIds: ['orders.note_id'], - table: { catalog: null, db: 'public', name: 'orders' }, - columns: ['note_id'], - }, - to: { - tableId: 'notes', - columnIds: ['notes.id'], - table: { catalog: null, db: 'public', name: 'notes' }, - columns: ['id'], - }, - relationshipType: 'many_to_one' as const, - source: 'deterministic_name', - score: 0.2, - confidence: 0.2, - pkScore: 0.4, - fkScore: 0.2, - reasons: ['low_source_coverage'], -}; - -const acceptedInvoiceAccount = { - candidateId: 'invoices:invoices.account_id->accounts:accounts.id', - decision: 'accepted' as const, - previousStatus: 'accepted' as const, - connectionId: 'billing', - runId: 'scan-run-b', - syncId: 'sync-b', - decidedAt: '2026-05-07T12:10:00.000Z', - reviewer: 'ktx', - note: null, - from: { - tableId: 'invoices', - columnIds: ['invoices.account_id'], - table: { catalog: null, db: 'billing', name: 'invoices' }, - columns: ['account_id'], - }, - to: { - tableId: 'accounts', - columnIds: ['accounts.id'], - table: { catalog: null, db: 'billing', name: 'accounts' }, - columns: ['id'], - }, - relationshipType: 'many_to_one' as const, - source: 'formal_metadata', - score: 1, - confidence: 1, - pkScore: 1, - fkScore: 1, - reasons: ['formal_metadata_relationship'], -}; - -describe('relationship feedback export', () => { - it('exports stable labels from all relationship review decision artifacts', async () => { - const project = projectWithFiles({ - 'raw-sources/warehouse/live-database/sync-a/enrichment/relationship-review-decisions.json': decisionsArtifact({ - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decisions: [rejectedOrderNote, acceptedOrderCustomer], - }), - 'raw-sources/billing/live-database/sync-b/enrichment/relationship-review-decisions.json': decisionsArtifact({ - connectionId: 'billing', - runId: 'scan-run-b', - syncId: 'sync-b', - decisions: [acceptedInvoiceAccount], - }), - 'raw-sources/warehouse/live-database/sync-a/enrichment/relationships.json': { accepted: [], review: [], rejected: [] }, - }); - - const result = await exportLocalRelationshipFeedbackLabels(project, { - now: () => new Date('2026-05-07T13:00:00.000Z'), - }); - - expect(result.summary).toEqual({ - total: 3, - accepted: 2, - rejected: 1, - connections: 2, - runs: 2, - }); - expect(result.labels.map((label) => label.candidateId)).toEqual([ - 'invoices:invoices.account_id->accounts:accounts.id', - 'orders:orders.customer_id->customers:customers.id', - 'orders:orders.note_id->notes:notes.id', - ]); - expect(result.labels[0]).toMatchObject({ - schemaVersion: 1, - decision: 'accepted', - connectionId: 'billing', - source: 'formal_metadata', - fromTable: 'billing.invoices', - fromColumns: ['account_id'], - toTable: 'billing.accounts', - toColumns: ['id'], - artifactPath: 'raw-sources/billing/live-database/sync-b/enrichment/relationship-review-decisions.json', - }); - expect(result.warnings).toEqual([]); - }); - - it('filters labels by connection and decision', async () => { - const project = projectWithFiles({ - 'raw-sources/warehouse/live-database/sync-a/enrichment/relationship-review-decisions.json': decisionsArtifact({ - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decisions: [rejectedOrderNote, acceptedOrderCustomer], - }), - 'raw-sources/billing/live-database/sync-b/enrichment/relationship-review-decisions.json': decisionsArtifact({ - connectionId: 'billing', - runId: 'scan-run-b', - syncId: 'sync-b', - decisions: [acceptedInvoiceAccount], - }), - }); - - const result = await exportLocalRelationshipFeedbackLabels(project, { - connectionId: 'warehouse', - decision: 'rejected', - now: () => new Date('2026-05-07T13:00:00.000Z'), - }); - - expect(result.summary).toMatchObject({ total: 1, accepted: 0, rejected: 1 }); - expect(result.labels).toHaveLength(1); - expect(result.labels[0]?.candidateId).toBe('orders:orders.note_id->notes:notes.id'); - }); - - it('formats JSONL with one stable label object per line', async () => { - const project = projectWithFiles({ - 'raw-sources/warehouse/live-database/sync-a/enrichment/relationship-review-decisions.json': decisionsArtifact({ - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decisions: [acceptedOrderCustomer], - }), - }); - const result = await exportLocalRelationshipFeedbackLabels(project, { - now: () => new Date('2026-05-07T13:00:00.000Z'), - }); - - const lines = formatKtxRelationshipFeedbackLabelsJsonl(result).trim().split('\n').map((line) => JSON.parse(line)); - - expect(lines).toHaveLength(1); - expect(lines[0]).toMatchObject({ - schemaVersion: 1, - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted', - relationshipType: 'many_to_one', - }); - }); - - it('records parse warnings and continues exporting readable decision artifacts', async () => { - const project = projectWithFiles({ - 'raw-sources/warehouse/live-database/sync-a/enrichment/relationship-review-decisions.json': decisionsArtifact({ - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decisions: [acceptedOrderCustomer], - }), - 'raw-sources/broken/live-database/sync-b/enrichment/relationship-review-decisions.json': '{not-json', - }); - - const result = await exportLocalRelationshipFeedbackLabels(project, { - now: () => new Date('2026-05-07T13:00:00.000Z'), - }); - - expect(result.summary.total).toBe(1); - expect(result.warnings).toEqual([ - { - path: 'raw-sources/broken/live-database/sync-b/enrichment/relationship-review-decisions.json', - message: expect.any(String), - }, - ]); - expect(result.warnings[0]?.message.length).toBeGreaterThan(0); - }); -}); diff --git a/packages/context/src/scan/relationship-feedback-export.ts b/packages/context/src/scan/relationship-feedback-export.ts deleted file mode 100644 index d4690cef..00000000 --- a/packages/context/src/scan/relationship-feedback-export.ts +++ /dev/null @@ -1,179 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import type { - KtxRelationshipReviewDecisionArtifact, - KtxRelationshipReviewDecisionEntry, - KtxRelationshipReviewDecisionValue, -} from './relationship-review-decisions.js'; - -const DECISION_ARTIFACT_SUFFIX = '/enrichment/relationship-review-decisions.json'; -const FEEDBACK_SCHEMA_VERSION = 1; - -export type KtxRelationshipFeedbackDecisionFilter = KtxRelationshipReviewDecisionValue | 'all'; - -export interface ExportLocalRelationshipFeedbackLabelsInput { - connectionId?: string | null; - decision?: KtxRelationshipFeedbackDecisionFilter; - now?: () => Date; -} - -export interface KtxRelationshipFeedbackLabel { - schemaVersion: 1; - candidateId: string; - decision: KtxRelationshipReviewDecisionValue; - previousStatus: KtxRelationshipReviewDecisionEntry['previousStatus']; - connectionId: string; - runId: string; - syncId: string; - decidedAt: string; - reviewer: string; - note: string | null; - relationshipType: KtxRelationshipReviewDecisionEntry['relationshipType']; - source: string; - score: number | null; - confidence: number; - pkScore: number | null; - fkScore: number | null; - fromTable: string; - fromColumns: string[]; - toTable: string; - toColumns: string[]; - reasons: string[]; - artifactPath: string; -} - -export interface KtxRelationshipFeedbackExportWarning { - path: string; - message: string; -} - -export interface ExportLocalRelationshipFeedbackLabelsResult { - generatedAt: string; - filters: { - connectionId: string | null; - decision: KtxRelationshipFeedbackDecisionFilter; - }; - summary: { - total: number; - accepted: number; - rejected: number; - connections: number; - runs: number; - }; - labels: KtxRelationshipFeedbackLabel[]; - warnings: KtxRelationshipFeedbackExportWarning[]; -} - -function qualifiedTableName(entry: KtxRelationshipReviewDecisionEntry, side: 'from' | 'to'): string { - const table = entry[side].table; - return [table.catalog, table.db, table.name].filter((part): part is string => Boolean(part)).join('.'); -} - -function labelFromDecision(entry: KtxRelationshipReviewDecisionEntry, artifactPath: string): KtxRelationshipFeedbackLabel { - return { - schemaVersion: FEEDBACK_SCHEMA_VERSION, - candidateId: entry.candidateId, - decision: entry.decision, - previousStatus: entry.previousStatus, - connectionId: entry.connectionId, - runId: entry.runId, - syncId: entry.syncId, - decidedAt: entry.decidedAt, - reviewer: entry.reviewer, - note: entry.note, - relationshipType: entry.relationshipType, - source: entry.source, - score: entry.score, - confidence: entry.confidence, - pkScore: entry.pkScore, - fkScore: entry.fkScore, - fromTable: qualifiedTableName(entry, 'from'), - fromColumns: [...entry.from.columns], - toTable: qualifiedTableName(entry, 'to'), - toColumns: [...entry.to.columns], - reasons: [...entry.reasons], - artifactPath, - }; -} - -function sortLabels(labels: KtxRelationshipFeedbackLabel[]): KtxRelationshipFeedbackLabel[] { - return [...labels].sort((left, right) => { - return ( - left.connectionId.localeCompare(right.connectionId) || - left.runId.localeCompare(right.runId) || - left.candidateId.localeCompare(right.candidateId) || - left.decidedAt.localeCompare(right.decidedAt) - ); - }); -} - -function passesFilters( - label: KtxRelationshipFeedbackLabel, - filters: { connectionId: string | null; decision: KtxRelationshipFeedbackDecisionFilter }, -): boolean { - if (filters.connectionId && label.connectionId !== filters.connectionId) { - return false; - } - return filters.decision === 'all' || label.decision === filters.decision; -} - -function messageFromUnknownError(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} - -async function readDecisionLabels( - project: KtxLocalProject, - artifactPath: string, -): Promise { - const raw = await project.fileStore.readFile(artifactPath); - const parsed = JSON.parse(raw.content) as KtxRelationshipReviewDecisionArtifact; - const decisions = Array.isArray(parsed.decisions) ? parsed.decisions : []; - return decisions.map((entry) => labelFromDecision(entry, artifactPath)); -} - -function summarize(labels: KtxRelationshipFeedbackLabel[]): ExportLocalRelationshipFeedbackLabelsResult['summary'] { - return { - total: labels.length, - accepted: labels.filter((label) => label.decision === 'accepted').length, - rejected: labels.filter((label) => label.decision === 'rejected').length, - connections: new Set(labels.map((label) => label.connectionId)).size, - runs: new Set(labels.map((label) => `${label.connectionId}:${label.runId}`)).size, - }; -} - -export async function exportLocalRelationshipFeedbackLabels( - project: KtxLocalProject, - input: ExportLocalRelationshipFeedbackLabelsInput = {}, -): Promise { - const filters = { - connectionId: input.connectionId ?? null, - decision: input.decision ?? 'all', - }; - const listed = await project.fileStore.listFiles('raw-sources'); - const artifactPaths = listed.files.filter((path) => path.endsWith(DECISION_ARTIFACT_SUFFIX)).sort(); - const labels: KtxRelationshipFeedbackLabel[] = []; - const warnings: KtxRelationshipFeedbackExportWarning[] = []; - - for (const artifactPath of artifactPaths) { - try { - labels.push(...(await readDecisionLabels(project, artifactPath))); - } catch (error) { - warnings.push({ path: artifactPath, message: messageFromUnknownError(error) }); - } - } - - const filtered = sortLabels(labels.filter((label) => passesFilters(label, filters))); - return { - generatedAt: (input.now?.() ?? new Date()).toISOString(), - filters, - summary: summarize(filtered), - labels: filtered, - warnings, - }; -} - -export function formatKtxRelationshipFeedbackLabelsJsonl(result: ExportLocalRelationshipFeedbackLabelsResult): string { - if (result.labels.length === 0) { - return ''; - } - return `${result.labels.map((label) => JSON.stringify(label)).join('\n')}\n`; -} diff --git a/packages/context/src/scan/relationship-review-apply.test.ts b/packages/context/src/scan/relationship-review-apply.test.ts deleted file mode 100644 index 268c9598..00000000 --- a/packages/context/src/scan/relationship-review-apply.test.ts +++ /dev/null @@ -1,351 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import type { KtxLocalProject } from '../project/index.js'; -import { initKtxProject } from '../project/index.js'; -import { describe, expect, it, vi } from 'vitest'; -import { applyLocalScanRelationshipReviewDecisions } from './relationship-review-apply.js'; -import type { KtxRelationshipReviewDecisionArtifact } from './relationship-review-decisions.js'; -import type { ReadLocalScanRelationshipArtifactsResult } from './relationship-artifacts.js'; -import type { WriteLocalScanManifestShardsResult } from './local-enrichment-artifacts.js'; -import type { KtxSchemaSnapshot } from './types.js'; - -const acceptedDecisionArtifact: KtxRelationshipReviewDecisionArtifact = { - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - generatedAt: '2026-05-07T12:00:00.000Z', - decisions: [ - { - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted', - previousStatus: 'review', - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decidedAt: '2026-05-07T12:01:00.000Z', - reviewer: 'Andrey', - note: 'Customer link is valid.', - from: { - tableId: 'public.orders', - columnIds: ['public.orders.customer_id'], - table: { catalog: null, db: 'public', name: 'orders' }, - columns: ['customer_id'], - }, - to: { - tableId: 'public.customers', - columnIds: ['public.customers.id'], - table: { catalog: null, db: 'public', name: 'customers' }, - columns: ['id'], - }, - relationshipType: 'many_to_one', - source: 'deterministic_name', - score: 0.81, - confidence: 0.81, - pkScore: 0.93, - fkScore: 0.81, - reasons: ['review_threshold'], - }, - { - candidateId: 'orders:orders.note_id->notes:notes.id', - decision: 'rejected', - previousStatus: 'review', - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decidedAt: '2026-05-07T12:02:00.000Z', - reviewer: 'Andrey', - note: null, - from: { - tableId: 'public.orders', - columnIds: ['public.orders.note_id'], - table: { catalog: null, db: 'public', name: 'orders' }, - columns: ['note_id'], - }, - to: { - tableId: 'public.notes', - columnIds: ['public.notes.id'], - table: { catalog: null, db: 'public', name: 'notes' }, - columns: ['id'], - }, - relationshipType: 'many_to_one', - source: 'embedding_similarity', - score: 0.7, - confidence: 0.7, - pkScore: 0.7, - fkScore: 0.7, - reasons: ['review_threshold'], - }, - ], -}; - -const artifacts: ReadLocalScanRelationshipArtifactsResult = { - runId: 'scan-run-a', - connectionId: 'warehouse', - syncId: 'sync-a', - report: { - connectionId: 'warehouse', - driver: 'postgres', - syncId: 'sync-a', - runId: 'scan-run-a', - trigger: 'cli', - mode: 'relationships', - dryRun: false, - artifactPaths: { - rawSourcesDir: 'raw-sources/warehouse/live-database/sync-a', - reportPath: 'raw-sources/warehouse/live-database/sync-a/scan-report.json', - manifestShards: ['semantic-layer/warehouse/_schema/public.yaml'], - enrichmentArtifacts: ['raw-sources/warehouse/live-database/sync-a/enrichment/relationships.json'], - }, - diffSummary: { - tablesAdded: 0, - tablesModified: 0, - tablesDeleted: 0, - tablesUnchanged: 2, - columnsAdded: 0, - columnsModified: 0, - columnsDeleted: 0, - }, - manifestShardsWritten: 1, - structuralSyncStats: { - tablesCreated: 0, - tablesUpdated: 0, - tablesDeleted: 0, - columnsCreated: 0, - columnsUpdated: 0, - columnsDeleted: 0, - }, - enrichment: { - dataDictionary: 'skipped', - tableDescriptions: 'skipped', - columnDescriptions: 'skipped', - embeddings: 'skipped', - deterministicRelationships: 'completed', - llmRelationshipValidation: 'skipped', - statisticalValidation: 'completed', - }, - capabilityGaps: [], - warnings: [], - relationships: { accepted: 0, review: 1, rejected: 1, skipped: 0 }, - enrichmentState: { resumedStages: [], completedStages: ['relationships'], failedStages: [] }, - createdAt: '2026-05-07T12:00:00.000Z', - }, - relationships: { - connectionId: 'warehouse', - accepted: [], - review: [], - rejected: [], - skipped: [], - }, - diagnostics: null, - profile: null, - paths: { - relationships: 'raw-sources/warehouse/live-database/sync-a/enrichment/relationships.json', - diagnostics: null, - profile: null, - }, -}; - -const snapshot: KtxSchemaSnapshot = { - connectionId: 'warehouse', - driver: 'postgres', - extractedAt: '2026-05-07T12:00:00.000Z', - scope: { schemas: ['public'] }, - metadata: {}, - tables: [ - { - catalog: null, - db: 'public', - name: 'customers', - kind: 'table', - comment: null, - estimatedRows: 2, - foreignKeys: [], - columns: [ - { - name: 'id', - nativeType: 'integer', - normalizedType: 'integer', - dimensionType: 'number', - nullable: false, - primaryKey: false, - comment: null, - }, - ], - }, - { - catalog: null, - db: 'public', - name: 'orders', - kind: 'table', - comment: null, - estimatedRows: 2, - foreignKeys: [], - columns: [ - { - name: 'customer_id', - nativeType: 'integer', - normalizedType: 'integer', - dimensionType: 'number', - nullable: false, - primaryKey: false, - comment: null, - }, - ], - }, - ], -}; - -async function projectWithDecisions( - decisions = acceptedDecisionArtifact, -): Promise<{ project: KtxLocalProject; tempDir: string }> { - const tempDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-review-apply-')); - const project = await initKtxProject({ - projectDir: join(tempDir, 'project'), - }); - await project.fileStore.writeFile( - 'raw-sources/warehouse/live-database/sync-a/enrichment/relationship-review-decisions.json', - `${JSON.stringify(decisions)}\n`, - 'ktx', - 'ktx@example.com', - 'Seed relationship review decisions', - ); - return { project, tempDir }; -} - -function manifestResult(): WriteLocalScanManifestShardsResult { - return { - manifestShards: ['semantic-layer/warehouse/_schema/public.yaml'], - manifestShardsWritten: 1, - }; -} - -describe('relationship review apply', () => { - it('previews all accepted decisions without writing manifest shards', async () => { - const { project, tempDir } = await projectWithDecisions(); - const writeLocalScanManifestShards = vi.fn(async () => manifestResult()); - try { - const result = await applyLocalScanRelationshipReviewDecisions(project, { - runId: 'scan-run-a', - applyAllAccepted: true, - dryRun: true, - readLocalScanRelationshipArtifacts: vi.fn(async () => artifacts), - readLocalScanStructuralSnapshot: vi.fn(async () => snapshot), - writeLocalScanManifestShards, - }); - - expect(result).toMatchObject({ - runId: 'scan-run-a', - connectionId: 'warehouse', - syncId: 'sync-a', - dryRun: true, - selectedDecisions: 1, - appliedRelationships: 1, - manifestShards: [], - manifestShardsWritten: 0, - }); - expect(result.relationships[0]).toMatchObject({ - id: 'orders:orders.customer_id->customers:customers.id', - source: 'manual', - relationshipType: 'many_to_one', - confidence: 1, - }); - expect(writeLocalScanManifestShards).not.toHaveBeenCalled(); - } finally { - await rm(tempDir, { recursive: true, force: true }); - } - }); - - it('writes selected accepted decisions as manual manifest relationships', async () => { - const { project, tempDir } = await projectWithDecisions(); - const readLocalScanStructuralSnapshot = vi.fn(async () => snapshot); - const writeLocalScanManifestShards = vi.fn(async () => manifestResult()); - try { - const result = await applyLocalScanRelationshipReviewDecisions(project, { - runId: 'scan-run-a', - candidateIds: ['orders:orders.customer_id->customers:customers.id'], - readLocalScanRelationshipArtifacts: vi.fn(async () => artifacts), - readLocalScanStructuralSnapshot, - writeLocalScanManifestShards, - }); - - expect(readLocalScanStructuralSnapshot).toHaveBeenCalledWith({ - project: expect.any(Object), - connectionId: 'warehouse', - driver: 'postgres', - rawSourcesDir: 'raw-sources/warehouse/live-database/sync-a', - extractedAtFallback: '2026-05-07T12:00:00.000Z', - }); - expect(writeLocalScanManifestShards).toHaveBeenCalledWith({ - project: expect.any(Object), - connectionId: 'warehouse', - syncId: 'sync-a', - driver: 'postgres', - snapshot, - dryRun: false, - relationshipUpdate: { - connectionId: 'warehouse', - accepted: [ - expect.objectContaining({ - id: 'orders:orders.customer_id->customers:customers.id', - source: 'manual', - from: expect.objectContaining({ columns: ['customer_id'] }), - to: expect.objectContaining({ columns: ['id'] }), - }), - ], - rejected: [], - skipped: [], - }, - }); - expect(result.manifestShardsWritten).toBe(1); - } finally { - await rm(tempDir, { recursive: true, force: true }); - } - }); - - it('rejects ambiguous apply selection input', async () => { - const { project, tempDir } = await projectWithDecisions(); - try { - await expect( - applyLocalScanRelationshipReviewDecisions(project, { - runId: 'scan-run-a', - readLocalScanRelationshipArtifacts: vi.fn(async () => artifacts), - }), - ).rejects.toThrow('Pass --all-accepted or at least one --candidate to choose review decisions to apply'); - - await expect( - applyLocalScanRelationshipReviewDecisions(project, { - runId: 'scan-run-a', - applyAllAccepted: true, - candidateIds: ['orders:orders.customer_id->customers:customers.id'], - readLocalScanRelationshipArtifacts: vi.fn(async () => artifacts), - }), - ).rejects.toThrow('Use either --all-accepted or --candidate, not both'); - } finally { - await rm(tempDir, { recursive: true, force: true }); - } - }); - - it('refuses rejected decisions and missing candidate ids', async () => { - const { project, tempDir } = await projectWithDecisions(); - try { - await expect( - applyLocalScanRelationshipReviewDecisions(project, { - runId: 'scan-run-a', - candidateIds: ['orders:orders.note_id->notes:notes.id'], - readLocalScanRelationshipArtifacts: vi.fn(async () => artifacts), - }), - ).rejects.toThrow('Relationship review decision "orders:orders.note_id->notes:notes.id" is rejected, not accepted'); - - await expect( - applyLocalScanRelationshipReviewDecisions(project, { - runId: 'scan-run-a', - candidateIds: ['missing'], - readLocalScanRelationshipArtifacts: vi.fn(async () => artifacts), - }), - ).rejects.toThrow('Relationship review decision "missing" was not found for scan run "scan-run-a"'); - } finally { - await rm(tempDir, { recursive: true, force: true }); - } - }); -}); diff --git a/packages/context/src/scan/relationship-review-apply.ts b/packages/context/src/scan/relationship-review-apply.ts deleted file mode 100644 index 3afa6a7e..00000000 --- a/packages/context/src/scan/relationship-review-apply.ts +++ /dev/null @@ -1,231 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import { - readLocalScanRelationshipArtifacts, - type ReadLocalScanRelationshipArtifactsResult, -} from './relationship-artifacts.js'; -import { - readLocalScanStructuralSnapshot, - type ReadLocalScanStructuralSnapshotInput, -} from './local-structural-artifacts.js'; -import { - writeLocalScanManifestShards, - type WriteLocalScanManifestShardsInput, - type WriteLocalScanManifestShardsResult, -} from './local-enrichment-artifacts.js'; -import type { KtxEnrichedRelationship, KtxRelationshipUpdate } from './enrichment-types.js'; -import type { - KtxRelationshipReviewDecisionArtifact, - KtxRelationshipReviewDecisionEntry, -} from './relationship-review-decisions.js'; - -const DECISIONS_FILE = 'relationship-review-decisions.json'; - -export interface ApplyLocalScanRelationshipReviewDecisionsInput { - runId: string; - applyAllAccepted?: boolean; - candidateIds?: readonly string[]; - dryRun?: boolean; - readLocalScanRelationshipArtifacts?: typeof readLocalScanRelationshipArtifacts; - readLocalScanStructuralSnapshot?: ( - input: ReadLocalScanStructuralSnapshotInput, - ) => Promise; - writeLocalScanManifestShards?: ( - input: WriteLocalScanManifestShardsInput, - ) => Promise; -} - -export interface AppliedRelationshipReviewDecision { - candidateId: string; - decidedAt: string; - reviewer: string; - note: string | null; - relationship: KtxEnrichedRelationship; -} - -export interface ApplyLocalScanRelationshipReviewDecisionsResult { - runId: string; - connectionId: string; - syncId: string; - dryRun: boolean; - decisionsPath: string; - selectedDecisions: number; - appliedRelationships: number; - relationships: KtxEnrichedRelationship[]; - manifestShards: string[]; - manifestShardsWritten: number; -} - -function decisionsPathFromRelationshipsPath(relationshipsPath: string): string { - return relationshipsPath.replace(/relationships\.json$/u, DECISIONS_FILE); -} - -async function readDecisionArtifact( - project: KtxLocalProject, - path: string, - runId: string, -): Promise { - let raw: { content: string }; - try { - raw = await project.fileStore.readFile(path); - } catch { - throw new Error(`Relationship review decisions were not found for scan run "${runId}"`); - } - const parsed = JSON.parse(raw.content) as KtxRelationshipReviewDecisionArtifact; - return { - connectionId: parsed.connectionId, - runId: parsed.runId, - syncId: parsed.syncId, - generatedAt: parsed.generatedAt, - decisions: Array.isArray(parsed.decisions) ? parsed.decisions : [], - }; -} - -function assertSelection(input: ApplyLocalScanRelationshipReviewDecisionsInput): void { - const candidateIds = input.candidateIds ?? []; - if (input.applyAllAccepted === true && candidateIds.length > 0) { - throw new Error('Use either --all-accepted or --candidate, not both'); - } - if (input.applyAllAccepted !== true && candidateIds.length === 0) { - throw new Error('Pass --all-accepted or at least one --candidate to choose review decisions to apply'); - } -} - -function selectAcceptedDecisions( - artifact: KtxRelationshipReviewDecisionArtifact, - input: ApplyLocalScanRelationshipReviewDecisionsInput, -): KtxRelationshipReviewDecisionEntry[] { - assertSelection(input); - if (input.applyAllAccepted === true) { - return artifact.decisions.filter((decision) => decision.decision === 'accepted'); - } - - const decisionsById = new Map(artifact.decisions.map((decision) => [decision.candidateId, decision])); - const selected: KtxRelationshipReviewDecisionEntry[] = []; - for (const candidateId of input.candidateIds ?? []) { - const decision = decisionsById.get(candidateId); - if (!decision) { - throw new Error(`Relationship review decision "${candidateId}" was not found for scan run "${input.runId}"`); - } - if (decision.decision !== 'accepted') { - throw new Error(`Relationship review decision "${candidateId}" is ${decision.decision}, not accepted`); - } - selected.push(decision); - } - return selected; -} - -function tableId(table: KtxRelationshipReviewDecisionEntry['from']['table']): string { - return [table.catalog, table.db, table.name].filter((part): part is string => Boolean(part)).join('.'); -} - -function columnIds(table: KtxRelationshipReviewDecisionEntry['from']['table'], columns: readonly string[]): string[] { - const prefix = tableId(table); - return columns.map((column) => `${prefix}.${column}`); -} - -function relationshipFromDecision(decision: KtxRelationshipReviewDecisionEntry): KtxEnrichedRelationship { - return { - id: decision.candidateId, - source: 'manual', - from: { - tableId: tableId(decision.from.table), - columnIds: columnIds(decision.from.table, decision.from.columns), - table: decision.from.table, - columns: [...decision.from.columns], - }, - to: { - tableId: tableId(decision.to.table), - columnIds: columnIds(decision.to.table, decision.to.columns), - table: decision.to.table, - columns: [...decision.to.columns], - }, - relationshipType: decision.relationshipType, - confidence: 1, - isPrimaryKeyReference: true, - }; -} - -function relationshipUpdate( - connectionId: string, - relationships: readonly KtxEnrichedRelationship[], -): KtxRelationshipUpdate { - return { - connectionId, - accepted: [...relationships], - rejected: [], - skipped: [], - }; -} - -function assertApplyableArtifacts(artifacts: ReadLocalScanRelationshipArtifactsResult): string { - const rawSourcesDir = artifacts.report.artifactPaths.rawSourcesDir; - if (!rawSourcesDir) { - throw new Error(`Scan run "${artifacts.runId}" does not have raw source artifacts for manifest rewriting`); - } - return rawSourcesDir; -} - -export async function applyLocalScanRelationshipReviewDecisions( - project: KtxLocalProject, - input: ApplyLocalScanRelationshipReviewDecisionsInput, -): Promise { - const readArtifacts = input.readLocalScanRelationshipArtifacts ?? readLocalScanRelationshipArtifacts; - const artifacts = await readArtifacts(project, input.runId); - if (!artifacts) { - throw new Error(`Scan run "${input.runId}" was not found`); - } - - const decisionsPath = decisionsPathFromRelationshipsPath(artifacts.paths.relationships); - const decisions = await readDecisionArtifact(project, decisionsPath, input.runId); - const selected = selectAcceptedDecisions(decisions, input); - const relationships = selected.map((decision) => relationshipFromDecision(decision)); - const dryRun = input.dryRun === true; - - if (dryRun || relationships.length === 0) { - return { - runId: artifacts.runId, - connectionId: artifacts.connectionId, - syncId: artifacts.syncId, - dryRun, - decisionsPath, - selectedDecisions: selected.length, - appliedRelationships: relationships.length, - relationships, - manifestShards: [], - manifestShardsWritten: 0, - }; - } - - const rawSourcesDir = assertApplyableArtifacts(artifacts); - const readSnapshot = input.readLocalScanStructuralSnapshot ?? readLocalScanStructuralSnapshot; - const writeManifestShards = input.writeLocalScanManifestShards ?? writeLocalScanManifestShards; - const snapshot = await readSnapshot({ - project, - connectionId: artifacts.connectionId, - driver: artifacts.report.driver, - rawSourcesDir, - extractedAtFallback: artifacts.report.createdAt, - }); - const manifest = await writeManifestShards({ - project, - connectionId: artifacts.connectionId, - syncId: artifacts.syncId, - driver: artifacts.report.driver, - snapshot, - dryRun: false, - relationshipUpdate: relationshipUpdate(artifacts.connectionId, relationships), - }); - - return { - runId: artifacts.runId, - connectionId: artifacts.connectionId, - syncId: artifacts.syncId, - dryRun, - decisionsPath, - selectedDecisions: selected.length, - appliedRelationships: relationships.length, - relationships, - manifestShards: manifest.manifestShards, - manifestShardsWritten: manifest.manifestShardsWritten, - }; -} diff --git a/packages/context/src/scan/relationship-review-decisions.test.ts b/packages/context/src/scan/relationship-review-decisions.test.ts deleted file mode 100644 index 979c863b..00000000 --- a/packages/context/src/scan/relationship-review-decisions.test.ts +++ /dev/null @@ -1,363 +0,0 @@ -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; -import { runLocalStageOnlyIngest, type SourceAdapter } from '../ingest/index.js'; -import { initKtxProject, loadKtxProject } from '../project/index.js'; -import { describe, expect, it } from 'vitest'; -import { writeLocalScanRelationshipReviewDecision } from './relationship-review-decisions.js'; -import type { KtxRelationshipArtifact, KtxRelationshipDiagnosticsArtifact } from './relationship-diagnostics.js'; -import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js'; -import type { KtxScanReport } from './types.js'; - -const RUN_ID = 'scan-run-review'; -const SYNC_ID = '2026-05-07-100000-scan-run-review'; - -async function writeProjectFile(projectDir: string, relativePath: string, content: string): Promise { - const absolutePath = join(projectDir, relativePath); - await mkdir(dirname(absolutePath), { recursive: true }); - await writeFile(absolutePath, content, 'utf-8'); -} - -async function createProject(projectDir: string): Promise { - await initKtxProject({ projectDir }); - await writeFile( - join(projectDir, 'ktx.yaml'), - [ - 'connections:', - ' warehouse:', - ' driver: sqlite', - ' path: warehouse.db', - 'ingest:', - ' adapters:', - ' - live-database', - '', - ].join('\n'), - 'utf-8', - ); -} - -function liveDatabaseAdapter(): SourceAdapter { - return { - source: 'live-database', - skillNames: ['live_database_ingest'], - async fetch(_pullConfig, stagedDir) { - await mkdir(join(stagedDir, 'tables'), { recursive: true }); - await writeFile(join(stagedDir, 'connection.json'), '{"connectionId":"warehouse"}\n', 'utf-8'); - await writeFile(join(stagedDir, 'foreign-keys.json'), '{"foreignKeys":[]}\n', 'utf-8'); - await writeFile( - join(stagedDir, 'tables', 'orders.json'), - '{"name":"orders","db":"public","columns":[{"name":"id","type":"integer","nullable":false,"primaryKey":true}]}\n', - 'utf-8', - ); - }, - async detect(stagedDir) { - await writeFile(join(stagedDir, 'connection.json'), '{"connectionId":"warehouse"}\n', 'utf-8'); - return true; - }, - async chunk() { - return { - workUnits: [ - { - unitKey: 'live-database-public-orders', - rawFiles: ['tables/orders.json'], - dependencyPaths: ['connection.json', 'foreign-keys.json'], - peerFileIndex: [], - }, - ], - }; - }, - }; -} - -async function createLiveDatabaseRun(projectDir: string): Promise { - await createProject(projectDir); - const project = await loadKtxProject({ projectDir }); - await runLocalStageOnlyIngest({ - project, - adapters: [liveDatabaseAdapter()], - adapter: 'live-database', - connectionId: 'warehouse', - jobId: RUN_ID, - now: () => new Date('2026-05-07T10:00:00.000Z'), - }); -} - -function reviewRelationships(): KtxRelationshipArtifact { - return { - connectionId: 'warehouse', - accepted: [], - review: [ - { - id: 'orders:orders.customer_id->customers:customers.id', - status: 'review', - source: 'deterministic_name', - from: { - tableId: 'orders', - columnIds: ['orders.customer_id'], - table: { catalog: null, db: 'public', name: 'orders' }, - columns: ['customer_id'], - }, - to: { - tableId: 'customers', - columnIds: ['customers.id'], - table: { catalog: null, db: 'public', name: 'customers' }, - columns: ['id'], - }, - relationshipType: 'many_to_one', - confidence: 0.62, - pkScore: 0.91, - fkScore: 0.62, - score: 0.62, - evidence: { sources: ['table_suffix'] }, - validation: { status: 'passed' }, - graph: { reasons: ['fk_score_review'] }, - reasons: ['fk_score_review'], - }, - ], - rejected: [], - skipped: [], - }; -} - -function diagnostics(): KtxRelationshipDiagnosticsArtifact { - return { - connectionId: 'warehouse', - generatedAt: '2026-05-07T10:00:00.000Z', - summary: { accepted: 0, review: 1, rejected: 0, skipped: 0 }, - noAcceptedReason: 'relationship candidates require review before manifest writes', - candidateCountsBySource: { deterministic_name: 1 }, - validation: { available: true, sqlAvailable: true, queryCount: 3 }, - thresholds: { acceptThreshold: 0.85, reviewThreshold: 0.55 }, - policy: { - validationRequiredForManifest: true, - maxCandidatesPerColumn: 25, - profileSampleRows: 10000, - validationConcurrency: 4, - }, - warnings: [], - profileWarnings: [], - }; -} - -function profile(): KtxRelationshipProfileArtifact { - return { - connectionId: 'warehouse', - driver: 'sqlite', - sqlAvailable: true, - tables: [], - columns: {}, - queryCount: 3, - warnings: [], - }; -} - -function report(): KtxScanReport { - return { - connectionId: 'warehouse', - driver: 'sqlite', - syncId: SYNC_ID, - runId: RUN_ID, - trigger: 'cli', - mode: 'relationships', - dryRun: false, - artifactPaths: { - rawSourcesDir: `raw-sources/warehouse/live-database/${SYNC_ID}`, - reportPath: `raw-sources/warehouse/live-database/${SYNC_ID}/scan-report.json`, - manifestShards: [], - enrichmentArtifacts: [ - `raw-sources/warehouse/live-database/${SYNC_ID}/enrichment/relationships.json`, - `raw-sources/warehouse/live-database/${SYNC_ID}/enrichment/relationship-diagnostics.json`, - `raw-sources/warehouse/live-database/${SYNC_ID}/enrichment/relationship-profile.json`, - ], - }, - diffSummary: { - tablesAdded: 0, - tablesModified: 0, - tablesDeleted: 0, - tablesUnchanged: 2, - columnsAdded: 0, - columnsModified: 0, - columnsDeleted: 0, - }, - manifestShardsWritten: 0, - structuralSyncStats: { - tablesCreated: 0, - tablesUpdated: 0, - tablesDeleted: 0, - columnsCreated: 0, - columnsUpdated: 0, - columnsDeleted: 0, - }, - enrichment: { - dataDictionary: 'skipped', - tableDescriptions: 'skipped', - columnDescriptions: 'skipped', - embeddings: 'skipped', - deterministicRelationships: 'completed', - llmRelationshipValidation: 'skipped', - statisticalValidation: 'completed', - }, - relationships: { accepted: 0, review: 1, rejected: 0, skipped: 0 }, - enrichmentState: { - resumedStages: [], - completedStages: ['relationships'], - failedStages: [], - }, - warnings: [], - capabilityGaps: [], - createdAt: '2026-05-07T10:00:00.000Z', - }; -} - -async function writeScanArtifacts(projectDir: string): Promise { - await writeProjectFile( - projectDir, - `raw-sources/warehouse/live-database/${SYNC_ID}/scan-report.json`, - JSON.stringify(report(), null, 2), - ); - await writeProjectFile( - projectDir, - `raw-sources/warehouse/live-database/${SYNC_ID}/enrichment/relationships.json`, - JSON.stringify(reviewRelationships(), null, 2), - ); - await writeProjectFile( - projectDir, - `raw-sources/warehouse/live-database/${SYNC_ID}/enrichment/relationship-diagnostics.json`, - JSON.stringify(diagnostics(), null, 2), - ); - await writeProjectFile( - projectDir, - `raw-sources/warehouse/live-database/${SYNC_ID}/enrichment/relationship-profile.json`, - JSON.stringify(profile(), null, 2), - ); -} - -describe('relationship review decisions', () => { - it('writes an accepted decision beside the scan relationship artifacts', async () => { - const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-review-decisions-')); - try { - await createLiveDatabaseRun(projectDir); - await writeScanArtifacts(projectDir); - const project = await loadKtxProject({ projectDir }); - - const result = await writeLocalScanRelationshipReviewDecision(project, { - runId: 'scan-run-review', - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted', - reviewer: 'Andrey', - note: 'Matches the warehouse model', - decidedAt: '2026-05-07T12:00:00.000Z', - }); - - expect(result).not.toBeNull(); - if (!result) { - throw new Error('Expected relationship review decision to be written'); - } - expect(result.path).toBe( - `raw-sources/warehouse/live-database/${SYNC_ID}/enrichment/relationship-review-decisions.json`, - ); - expect(result.artifact.decisions).toHaveLength(1); - expect(result.decision).toMatchObject({ - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted', - previousStatus: 'review', - reviewer: 'Andrey', - note: 'Matches the warehouse model', - source: 'deterministic_name', - relationshipType: 'many_to_one', - score: 0.62, - reasons: ['fk_score_review'], - }); - await expect(project.fileStore.readFile(result.path)).resolves.toMatchObject({ - path: result.path, - content: expect.stringContaining('"decision": "accepted"'), - }); - } finally { - await rm(projectDir, { recursive: true, force: true }); - } - }); - - it('replaces the existing decision for the same candidate id', async () => { - const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-review-replace-')); - try { - await createLiveDatabaseRun(projectDir); - await writeScanArtifacts(projectDir); - const project = await loadKtxProject({ projectDir }); - - await writeLocalScanRelationshipReviewDecision(project, { - runId: 'scan-run-review', - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted', - reviewer: 'Andrey', - note: 'First decision', - decidedAt: '2026-05-07T12:00:00.000Z', - }); - const replacement = await writeLocalScanRelationshipReviewDecision(project, { - runId: 'scan-run-review', - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'rejected', - reviewer: 'Andrey', - note: 'Reviewed against source data and rejected', - decidedAt: '2026-05-07T12:05:00.000Z', - }); - - expect(replacement).not.toBeNull(); - if (!replacement) { - throw new Error('Expected replacement relationship review decision to be written'); - } - expect(replacement.artifact.decisions).toHaveLength(1); - expect(replacement.artifact.decisions[0]).toMatchObject({ - decision: 'rejected', - note: 'Reviewed against source data and rejected', - decidedAt: '2026-05-07T12:05:00.000Z', - }); - } finally { - await rm(projectDir, { recursive: true, force: true }); - } - }); - - it('returns null when the scan run does not exist', async () => { - const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-review-missing-run-')); - try { - await createProject(projectDir); - const project = await loadKtxProject({ projectDir }); - - await expect( - writeLocalScanRelationshipReviewDecision(project, { - runId: 'missing-run', - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted', - reviewer: 'Andrey', - note: null, - decidedAt: '2026-05-07T12:00:00.000Z', - }), - ).resolves.toBeNull(); - } finally { - await rm(projectDir, { recursive: true, force: true }); - } - }); - - it('rejects unknown candidate ids for an existing scan run', async () => { - const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-review-missing-candidate-')); - try { - await createLiveDatabaseRun(projectDir); - await writeScanArtifacts(projectDir); - const project = await loadKtxProject({ projectDir }); - - await expect( - writeLocalScanRelationshipReviewDecision(project, { - runId: 'scan-run-review', - candidateId: 'orders:orders.unknown_id->customers:customers.id', - decision: 'accepted', - reviewer: 'Andrey', - note: null, - decidedAt: '2026-05-07T12:00:00.000Z', - }), - ).rejects.toThrow( - 'Relationship candidate "orders:orders.unknown_id->customers:customers.id" was not found in scan run "scan-run-review"', - ); - } finally { - await rm(projectDir, { recursive: true, force: true }); - } - }); -}); diff --git a/packages/context/src/scan/relationship-review-decisions.ts b/packages/context/src/scan/relationship-review-decisions.ts deleted file mode 100644 index bf459d8a..00000000 --- a/packages/context/src/scan/relationship-review-decisions.ts +++ /dev/null @@ -1,182 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import type { KtxRelationshipType } from './enrichment-types.js'; -import { readLocalScanRelationshipArtifacts } from './relationship-artifacts.js'; -import type { - KtxRelationshipArtifactEdge, - KtxRelationshipArtifactEndpoint, -} from './relationship-diagnostics.js'; -import type { KtxResolvedRelationshipStatus } from './relationship-graph-resolver.js'; - -const LOCAL_AUTHOR = 'ktx'; -const LOCAL_AUTHOR_EMAIL = 'ktx@example.com'; -const DECISIONS_FILE = 'relationship-review-decisions.json'; - -export type KtxRelationshipReviewDecisionValue = 'accepted' | 'rejected'; - -export interface WriteLocalScanRelationshipReviewDecisionInput { - runId: string; - candidateId: string; - decision: KtxRelationshipReviewDecisionValue; - reviewer: string; - note: string | null; - decidedAt?: string; -} - -export interface KtxRelationshipReviewDecisionEntry { - candidateId: string; - decision: KtxRelationshipReviewDecisionValue; - previousStatus: KtxResolvedRelationshipStatus; - connectionId: string; - runId: string; - syncId: string; - decidedAt: string; - reviewer: string; - note: string | null; - from: KtxRelationshipArtifactEndpoint; - to: KtxRelationshipArtifactEndpoint; - relationshipType: KtxRelationshipType; - source: string; - score: number | null; - confidence: number; - pkScore: number | null; - fkScore: number | null; - reasons: string[]; -} - -export interface KtxRelationshipReviewDecisionArtifact { - connectionId: string; - runId: string; - syncId: string; - generatedAt: string; - decisions: KtxRelationshipReviewDecisionEntry[]; -} - -export interface WriteLocalScanRelationshipReviewDecisionResult { - path: string; - decision: KtxRelationshipReviewDecisionEntry; - artifact: KtxRelationshipReviewDecisionArtifact; -} - -function reviewDecisionPath(relationshipsPath: string): string { - return relationshipsPath.replace(/relationships\.json$/u, DECISIONS_FILE); -} - -function allCandidateEdges(result: Awaited>): KtxRelationshipArtifactEdge[] { - if (!result) { - return []; - } - return [...result.relationships.accepted, ...result.relationships.review, ...result.relationships.rejected]; -} - -async function readExistingDecisions( - project: KtxLocalProject, - path: string, - fallback: Omit, -): Promise { - try { - const raw = await project.fileStore.readFile(path); - const parsed = JSON.parse(raw.content) as KtxRelationshipReviewDecisionArtifact; - return { - connectionId: parsed.connectionId, - runId: parsed.runId, - syncId: parsed.syncId, - generatedAt: parsed.generatedAt, - decisions: Array.isArray(parsed.decisions) ? parsed.decisions : [], - }; - } catch { - return { ...fallback, decisions: [] }; - } -} - -function decisionEntry(input: { - candidate: KtxRelationshipArtifactEdge; - connectionId: string; - runId: string; - syncId: string; - decision: KtxRelationshipReviewDecisionValue; - reviewer: string; - note: string | null; - decidedAt: string; -}): KtxRelationshipReviewDecisionEntry { - return { - candidateId: input.candidate.id, - decision: input.decision, - previousStatus: input.candidate.status, - connectionId: input.connectionId, - runId: input.runId, - syncId: input.syncId, - decidedAt: input.decidedAt, - reviewer: input.reviewer, - note: input.note, - from: input.candidate.from, - to: input.candidate.to, - relationshipType: input.candidate.relationshipType, - source: input.candidate.source, - score: input.candidate.score, - confidence: input.candidate.confidence, - pkScore: input.candidate.pkScore, - fkScore: input.candidate.fkScore, - reasons: [...input.candidate.reasons], - }; -} - -function upsertDecision( - existing: readonly KtxRelationshipReviewDecisionEntry[], - next: KtxRelationshipReviewDecisionEntry, -): KtxRelationshipReviewDecisionEntry[] { - return [...existing.filter((item) => item.candidateId !== next.candidateId), next].sort((left, right) => - left.candidateId.localeCompare(right.candidateId), - ); -} - -export async function writeLocalScanRelationshipReviewDecision( - project: KtxLocalProject, - input: WriteLocalScanRelationshipReviewDecisionInput, -): Promise { - const artifacts = await readLocalScanRelationshipArtifacts(project, input.runId); - if (!artifacts) { - return null; - } - - const candidate = allCandidateEdges(artifacts).find((edge) => edge.id === input.candidateId); - if (!candidate) { - throw new Error(`Relationship candidate "${input.candidateId}" was not found in scan run "${input.runId}"`); - } - - const decidedAt = input.decidedAt ?? new Date().toISOString(); - const path = reviewDecisionPath(artifacts.paths.relationships); - const fallback = { - connectionId: artifacts.connectionId, - runId: artifacts.runId, - syncId: artifacts.syncId, - generatedAt: decidedAt, - }; - const existing = await readExistingDecisions(project, path, fallback); - const decision = decisionEntry({ - candidate, - connectionId: artifacts.connectionId, - runId: artifacts.runId, - syncId: artifacts.syncId, - decision: input.decision, - reviewer: input.reviewer, - note: input.note, - decidedAt, - }); - const artifact: KtxRelationshipReviewDecisionArtifact = { - connectionId: artifacts.connectionId, - runId: artifacts.runId, - syncId: artifacts.syncId, - generatedAt: decidedAt, - decisions: upsertDecision(existing.decisions, decision), - }; - - await project.fileStore.writeFile( - path, - `${JSON.stringify(artifact, null, 2)}\n`, - LOCAL_AUTHOR, - LOCAL_AUTHOR_EMAIL, - `scan(live-database): record relationship review decision runId=${input.runId}`, - ); - - return { path, decision, artifact }; -} diff --git a/packages/context/src/scan/relationship-threshold-advice.test.ts b/packages/context/src/scan/relationship-threshold-advice.test.ts deleted file mode 100644 index aceb19c2..00000000 --- a/packages/context/src/scan/relationship-threshold-advice.test.ts +++ /dev/null @@ -1,241 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import { describe, expect, it, vi } from 'vitest'; -import { - adviseLocalRelationshipFeedbackThresholds, - buildKtxRelationshipThresholdAdviceReport, - formatKtxRelationshipThresholdAdviceMarkdown, -} from './relationship-threshold-advice.js'; -import type { - ExportLocalRelationshipFeedbackLabelsResult, - KtxRelationshipFeedbackLabel, -} from './relationship-feedback-export.js'; - -function label( - input: Partial & Pick, -): KtxRelationshipFeedbackLabel { - return { - schemaVersion: 1, - previousStatus: 'review', - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decidedAt: '2026-05-07T12:00:00.000Z', - reviewer: 'Andrey', - note: null, - relationshipType: 'many_to_one', - source: 'deterministic_name', - confidence: input.score ?? 0, - pkScore: input.pkScore ?? null, - fkScore: input.fkScore ?? input.score, - fromTable: 'public.orders', - fromColumns: ['customer_id'], - toTable: 'public.customers', - toColumns: ['id'], - reasons: [], - artifactPath: 'raw-sources/warehouse/live-database/sync-a/enrichment/relationship-review-decisions.json', - ...input, - }; -} - -function feedback(labels: KtxRelationshipFeedbackLabel[]): ExportLocalRelationshipFeedbackLabelsResult { - return { - generatedAt: '2026-05-07T13:00:00.000Z', - filters: { connectionId: null, decision: 'all' }, - summary: { - total: labels.length, - accepted: labels.filter((item) => item.decision === 'accepted').length, - rejected: labels.filter((item) => item.decision === 'rejected').length, - connections: new Set(labels.map((item) => item.connectionId)).size, - runs: new Set(labels.map((item) => `${item.connectionId}:${item.runId}`)).size, - }, - labels, - warnings: [], - }; -} - -describe('relationship threshold advice', () => { - it('selects the highest-quality threshold candidate when enough labels exist', () => { - const report = buildKtxRelationshipThresholdAdviceReport( - feedback([ - label({ - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted', - score: 0.91, - pkScore: 0.97, - fkScore: 0.91, - }), - label({ - candidateId: 'orders:orders.account_id->accounts:accounts.id', - decision: 'accepted', - score: 0.61, - pkScore: 0.88, - fkScore: 0.61, - }), - label({ - candidateId: 'orders:orders.note_id->notes:notes.id', - decision: 'rejected', - score: 0.21, - pkScore: 0.4, - fkScore: 0.21, - }), - label({ - candidateId: 'orders:orders.region_id->regions:regions.id', - decision: 'rejected', - score: 0.88, - pkScore: 0.9, - fkScore: 0.88, - }), - ]), - { - acceptThresholds: [0.9, 0.85], - reviewThresholds: [0.55], - minTotalLabels: 4, - minAcceptedLabels: 2, - minRejectedLabels: 2, - minAcceptedBandPrecision: 0.75, - minAcceptedOrReviewRecall: 0.75, - minRejectedBandPrecision: 0.75, - }, - ); - - expect(report.status).toBe('ready'); - expect(report.summary).toMatchObject({ - totalLabels: 4, - scoredLabels: 4, - acceptedLabels: 2, - rejectedLabels: 2, - eligibleCandidates: 1, - }); - expect(report.recommended).toMatchObject({ - acceptThreshold: 0.9, - reviewThreshold: 0.55, - eligible: true, - acceptedBandPrecision: 1, - acceptedRecall: 0.5, - acceptedOrReviewRecall: 1, - rejectedBandPrecision: 1, - rejectedRecall: 1, - falseAcceptedRejectedLabels: 0, - falseRejectedAcceptedLabels: 0, - }); - expect(report.candidates.map((candidate) => [candidate.acceptThreshold, candidate.reviewThreshold, candidate.eligible])).toEqual([ - [0.9, 0.55, true], - [0.85, 0.55, false], - ]); - }); - - it('reports insufficient labels without hiding evaluated candidates', () => { - const report = buildKtxRelationshipThresholdAdviceReport( - feedback([ - label({ candidateId: 'orders:orders.customer_id->customers:customers.id', decision: 'accepted', score: 0.91 }), - label({ candidateId: 'orders:orders.note_id->notes:notes.id', decision: 'rejected', score: 0.21 }), - ]), - { - acceptThresholds: [0.9], - reviewThresholds: [0.55], - minTotalLabels: 10, - minAcceptedLabels: 5, - minRejectedLabels: 5, - }, - ); - - expect(report.status).toBe('insufficient_labels'); - expect(report.recommended).toBeNull(); - expect(report.summary).toMatchObject({ - totalLabels: 2, - scoredLabels: 2, - acceptedLabels: 1, - rejectedLabels: 1, - eligibleCandidates: 1, - }); - expect(report.reasons).toEqual([ - 'Need at least 10 scored labels; found 2.', - 'Need at least 5 accepted labels; found 1.', - 'Need at least 5 rejected labels; found 1.', - ]); - expect(report.candidates).toHaveLength(1); - }); - - it('reports no eligible thresholds when label counts pass but quality gates fail', () => { - const report = buildKtxRelationshipThresholdAdviceReport( - feedback([ - label({ candidateId: 'a', decision: 'accepted', score: 0.92 }), - label({ candidateId: 'b', decision: 'accepted', score: 0.58 }), - label({ candidateId: 'c', decision: 'rejected', score: 0.91 }), - label({ candidateId: 'd', decision: 'rejected', score: 0.2 }), - ]), - { - acceptThresholds: [0.9], - reviewThresholds: [0.55], - minTotalLabels: 4, - minAcceptedLabels: 2, - minRejectedLabels: 2, - minAcceptedBandPrecision: 0.9, - }, - ); - - expect(report.status).toBe('no_eligible_thresholds'); - expect(report.recommended).toBeNull(); - expect(report.reasons).toEqual(['No threshold candidate met the precision and recall gates.']); - expect(report.candidates[0]).toMatchObject({ - acceptThreshold: 0.9, - reviewThreshold: 0.55, - eligible: false, - acceptedBandPrecision: 0.5, - }); - }); - - it('wraps the feedback exporter and preserves warnings', async () => { - const project = { projectDir: '/tmp/ktx-project' } as KtxLocalProject; - const exportLocalRelationshipFeedbackLabels = vi.fn(async () => ({ - ...feedback([]), - warnings: [ - { - path: 'raw-sources/broken/live-database/sync/enrichment/relationship-review-decisions.json', - message: 'Unexpected token', - }, - ], - })); - - const report = await adviseLocalRelationshipFeedbackThresholds(project, { - connectionId: 'warehouse', - exportLocalRelationshipFeedbackLabels, - minTotalLabels: 1, - }); - - expect(exportLocalRelationshipFeedbackLabels).toHaveBeenCalledWith(project, { - connectionId: 'warehouse', - decision: 'all', - }); - expect(report.warnings).toEqual([ - { - path: 'raw-sources/broken/live-database/sync/enrichment/relationship-review-decisions.json', - message: 'Unexpected token', - }, - ]); - }); - - it('formats a stable human-readable report', () => { - const report = buildKtxRelationshipThresholdAdviceReport( - feedback([ - label({ candidateId: 'orders:orders.customer_id->customers:customers.id', decision: 'accepted', score: 0.91 }), - label({ candidateId: 'orders:orders.account_id->accounts:accounts.id', decision: 'accepted', score: 0.61 }), - label({ candidateId: 'orders:orders.note_id->notes:notes.id', decision: 'rejected', score: 0.21 }), - label({ candidateId: 'orders:orders.region_id->regions:regions.id', decision: 'rejected', score: 0.88 }), - ]), - { - acceptThresholds: [0.9], - reviewThresholds: [0.55], - minTotalLabels: 4, - minAcceptedLabels: 2, - minRejectedLabels: 2, - minAcceptedBandPrecision: 0.75, - }, - ); - - expect(formatKtxRelationshipThresholdAdviceMarkdown(report)).toContain('KTX relationship threshold advice'); - expect(formatKtxRelationshipThresholdAdviceMarkdown(report)).toContain('Status: ready'); - expect(formatKtxRelationshipThresholdAdviceMarkdown(report)).toContain('Recommended: accept=0.90 review=0.55'); - expect(formatKtxRelationshipThresholdAdviceMarkdown(report)).toContain('acceptedPrecision=1.000'); - }); -}); diff --git a/packages/context/src/scan/relationship-threshold-advice.ts b/packages/context/src/scan/relationship-threshold-advice.ts deleted file mode 100644 index d94abc2e..00000000 --- a/packages/context/src/scan/relationship-threshold-advice.ts +++ /dev/null @@ -1,335 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import { - exportLocalRelationshipFeedbackLabels, - type ExportLocalRelationshipFeedbackLabelsInput, - type ExportLocalRelationshipFeedbackLabelsResult, - type KtxRelationshipFeedbackExportWarning, - type KtxRelationshipFeedbackLabel, -} from './relationship-feedback-export.js'; -import type { KtxResolvedRelationshipStatus } from './relationship-graph-resolver.js'; - -const DEFAULT_ACCEPT_THRESHOLDS = [0.95, 0.9, 0.85, 0.8, 0.75] as const; -const DEFAULT_REVIEW_THRESHOLDS = [0.65, 0.6, 0.55, 0.5, 0.45] as const; - -type AdvicePredictedStatus = KtxResolvedRelationshipStatus; -export type KtxRelationshipThresholdAdviceStatus = 'ready' | 'insufficient_labels' | 'no_eligible_thresholds'; - -export interface BuildKtxRelationshipThresholdAdviceReportInput { - acceptThresholds?: readonly number[]; - reviewThresholds?: readonly number[]; - minTotalLabels?: number; - minAcceptedLabels?: number; - minRejectedLabels?: number; - minAcceptedBandPrecision?: number; - minAcceptedOrReviewRecall?: number; - minRejectedBandPrecision?: number; -} - -export interface AdviseLocalRelationshipFeedbackThresholdsInput - extends Omit, - BuildKtxRelationshipThresholdAdviceReportInput { - exportLocalRelationshipFeedbackLabels?: typeof exportLocalRelationshipFeedbackLabels; -} - -export interface KtxRelationshipThresholdAdviceCandidate { - acceptThreshold: number; - reviewThreshold: number; - eligible: boolean; - predictedAccepted: number; - predictedReview: number; - predictedRejected: number; - acceptedBandPrecision: number | null; - acceptedRecall: number | null; - acceptedOrReviewRecall: number | null; - rejectedBandPrecision: number | null; - rejectedRecall: number | null; - falseAcceptedRejectedLabels: number; - falseRejectedAcceptedLabels: number; -} - -export interface KtxRelationshipThresholdAdviceReport { - generatedAt: string; - filters: ExportLocalRelationshipFeedbackLabelsResult['filters']; - status: KtxRelationshipThresholdAdviceStatus; - gates: { - minTotalLabels: number; - minAcceptedLabels: number; - minRejectedLabels: number; - minAcceptedBandPrecision: number; - minAcceptedOrReviewRecall: number; - minRejectedBandPrecision: number; - }; - summary: { - totalLabels: number; - scoredLabels: number; - unscoredLabels: number; - acceptedLabels: number; - rejectedLabels: number; - evaluatedCandidates: number; - eligibleCandidates: number; - }; - recommended: KtxRelationshipThresholdAdviceCandidate | null; - candidates: KtxRelationshipThresholdAdviceCandidate[]; - reasons: string[]; - warnings: KtxRelationshipFeedbackExportWarning[]; -} - -interface ResolvedAdviceInput { - acceptThresholds: number[]; - reviewThresholds: number[]; - minTotalLabels: number; - minAcceptedLabels: number; - minRejectedLabels: number; - minAcceptedBandPrecision: number; - minAcceptedOrReviewRecall: number; - minRejectedBandPrecision: number; -} - -function resolveInput(input: BuildKtxRelationshipThresholdAdviceReportInput): ResolvedAdviceInput { - return { - acceptThresholds: [...(input.acceptThresholds ?? DEFAULT_ACCEPT_THRESHOLDS)].sort((left, right) => right - left), - reviewThresholds: [...(input.reviewThresholds ?? DEFAULT_REVIEW_THRESHOLDS)].sort((left, right) => right - left), - minTotalLabels: input.minTotalLabels ?? 20, - minAcceptedLabels: input.minAcceptedLabels ?? 5, - minRejectedLabels: input.minRejectedLabels ?? 5, - minAcceptedBandPrecision: input.minAcceptedBandPrecision ?? 0.9, - minAcceptedOrReviewRecall: input.minAcceptedOrReviewRecall ?? 0.8, - minRejectedBandPrecision: input.minRejectedBandPrecision ?? 0.8, - }; -} - -function roundMetric(value: number): number { - return Math.round(value * 1000) / 1000; -} - -function ratio(numerator: number, denominator: number): number | null { - return denominator === 0 ? null : roundMetric(numerator / denominator); -} - -function prediction(score: number, acceptThreshold: number, reviewThreshold: number): AdvicePredictedStatus { - if (score >= acceptThreshold) { - return 'accepted'; - } - if (score >= reviewThreshold) { - return 'review'; - } - return 'rejected'; -} - -function isMetricAtLeast(value: number | null, minimum: number): boolean { - return value !== null && value >= minimum; -} - -function thresholdCandidate( - labels: readonly KtxRelationshipFeedbackLabel[], - acceptThreshold: number, - reviewThreshold: number, - gates: ResolvedAdviceInput, -): KtxRelationshipThresholdAdviceCandidate { - const scored = labels.filter((label): label is KtxRelationshipFeedbackLabel & { score: number } => label.score !== null); - const acceptedLabels = scored.filter((label) => label.decision === 'accepted'); - const rejectedLabels = scored.filter((label) => label.decision === 'rejected'); - const predictions = scored.map((label) => ({ - label, - predictedStatus: prediction(label.score, acceptThreshold, reviewThreshold), - })); - const predictedAccepted = predictions.filter((item) => item.predictedStatus === 'accepted'); - const predictedReview = predictions.filter((item) => item.predictedStatus === 'review'); - const predictedRejected = predictions.filter((item) => item.predictedStatus === 'rejected'); - const acceptedBandPrecision = ratio( - predictedAccepted.filter((item) => item.label.decision === 'accepted').length, - predictedAccepted.length, - ); - const acceptedOrReviewRecall = ratio( - predictions.filter((item) => item.label.decision === 'accepted' && item.predictedStatus !== 'rejected').length, - acceptedLabels.length, - ); - const rejectedBandPrecision = ratio( - predictedRejected.filter((item) => item.label.decision === 'rejected').length, - predictedRejected.length, - ); - - return { - acceptThreshold, - reviewThreshold, - eligible: - predictedAccepted.length > 0 && - predictedRejected.length > 0 && - isMetricAtLeast(acceptedBandPrecision, gates.minAcceptedBandPrecision) && - isMetricAtLeast(acceptedOrReviewRecall, gates.minAcceptedOrReviewRecall) && - isMetricAtLeast(rejectedBandPrecision, gates.minRejectedBandPrecision), - predictedAccepted: predictedAccepted.length, - predictedReview: predictedReview.length, - predictedRejected: predictedRejected.length, - acceptedBandPrecision, - acceptedRecall: ratio( - predictedAccepted.filter((item) => item.label.decision === 'accepted').length, - acceptedLabels.length, - ), - acceptedOrReviewRecall, - rejectedBandPrecision, - rejectedRecall: ratio( - predictions.filter((item) => item.label.decision === 'rejected' && item.predictedStatus !== 'accepted').length, - rejectedLabels.length, - ), - falseAcceptedRejectedLabels: predictedAccepted.filter((item) => item.label.decision === 'rejected').length, - falseRejectedAcceptedLabels: predictedRejected.filter((item) => item.label.decision === 'accepted').length, - }; -} - -function metricRank(value: number | null): number { - return value ?? -1; -} - -function sortCandidates( - candidates: readonly KtxRelationshipThresholdAdviceCandidate[], -): KtxRelationshipThresholdAdviceCandidate[] { - return [...candidates].sort( - (left, right) => - Number(right.eligible) - Number(left.eligible) || - metricRank(right.acceptedBandPrecision) - metricRank(left.acceptedBandPrecision) || - metricRank(right.acceptedOrReviewRecall) - metricRank(left.acceptedOrReviewRecall) || - metricRank(right.rejectedBandPrecision) - metricRank(left.rejectedBandPrecision) || - right.acceptThreshold - left.acceptThreshold || - right.reviewThreshold - left.reviewThreshold, - ); -} - -function labelGateReasons(labels: readonly KtxRelationshipFeedbackLabel[], gates: ResolvedAdviceInput): string[] { - const scored = labels.filter((label) => label.score !== null); - const accepted = scored.filter((label) => label.decision === 'accepted'); - const rejected = scored.filter((label) => label.decision === 'rejected'); - const reasons: string[] = []; - if (scored.length < gates.minTotalLabels) { - reasons.push(`Need at least ${gates.minTotalLabels} scored labels; found ${scored.length}.`); - } - if (accepted.length < gates.minAcceptedLabels) { - reasons.push(`Need at least ${gates.minAcceptedLabels} accepted labels; found ${accepted.length}.`); - } - if (rejected.length < gates.minRejectedLabels) { - reasons.push(`Need at least ${gates.minRejectedLabels} rejected labels; found ${rejected.length}.`); - } - return reasons; -} - -export function buildKtxRelationshipThresholdAdviceReport( - feedback: ExportLocalRelationshipFeedbackLabelsResult, - input: BuildKtxRelationshipThresholdAdviceReportInput = {}, -): KtxRelationshipThresholdAdviceReport { - const gates = resolveInput(input); - const scored = feedback.labels.filter((label) => label.score !== null); - const acceptedLabels = scored.filter((label) => label.decision === 'accepted'); - const rejectedLabels = scored.filter((label) => label.decision === 'rejected'); - const candidates = sortCandidates( - gates.acceptThresholds.flatMap((acceptThreshold) => - gates.reviewThresholds.flatMap((reviewThreshold) => - acceptThreshold > reviewThreshold - ? [thresholdCandidate(feedback.labels, acceptThreshold, reviewThreshold, gates)] - : [], - ), - ), - ); - const labelReasons = labelGateReasons(feedback.labels, gates); - const eligibleCandidates = candidates.filter((candidate) => candidate.eligible); - const status: KtxRelationshipThresholdAdviceStatus = - labelReasons.length > 0 ? 'insufficient_labels' : eligibleCandidates.length > 0 ? 'ready' : 'no_eligible_thresholds'; - const reasons = - status === 'insufficient_labels' - ? labelReasons - : status === 'no_eligible_thresholds' - ? ['No threshold candidate met the precision and recall gates.'] - : []; - - return { - generatedAt: feedback.generatedAt, - filters: feedback.filters, - status, - gates: { - minTotalLabels: gates.minTotalLabels, - minAcceptedLabels: gates.minAcceptedLabels, - minRejectedLabels: gates.minRejectedLabels, - minAcceptedBandPrecision: gates.minAcceptedBandPrecision, - minAcceptedOrReviewRecall: gates.minAcceptedOrReviewRecall, - minRejectedBandPrecision: gates.minRejectedBandPrecision, - }, - summary: { - totalLabels: feedback.labels.length, - scoredLabels: scored.length, - unscoredLabels: feedback.labels.length - scored.length, - acceptedLabels: acceptedLabels.length, - rejectedLabels: rejectedLabels.length, - evaluatedCandidates: candidates.length, - eligibleCandidates: eligibleCandidates.length, - }, - recommended: status === 'ready' ? eligibleCandidates[0] ?? null : null, - candidates, - reasons, - warnings: [...feedback.warnings], - }; -} - -export async function adviseLocalRelationshipFeedbackThresholds( - project: KtxLocalProject, - input: AdviseLocalRelationshipFeedbackThresholdsInput = {}, -): Promise { - const exporter = input.exportLocalRelationshipFeedbackLabels ?? exportLocalRelationshipFeedbackLabels; - const feedback = await exporter(project, { - connectionId: input.connectionId, - decision: 'all', - }); - return buildKtxRelationshipThresholdAdviceReport(feedback, input); -} - -function formatMetric(value: number | null): string { - return value === null ? 'n/a' : value.toFixed(3); -} - -function candidateLine(candidate: KtxRelationshipThresholdAdviceCandidate): string { - return [ - `accept=${candidate.acceptThreshold.toFixed(2)}`, - `review=${candidate.reviewThreshold.toFixed(2)}`, - `eligible=${candidate.eligible ? 'yes' : 'no'}`, - `acceptedPrecision=${formatMetric(candidate.acceptedBandPrecision)}`, - `acceptedRecall=${formatMetric(candidate.acceptedRecall)}`, - `acceptedOrReviewRecall=${formatMetric(candidate.acceptedOrReviewRecall)}`, - `rejectedPrecision=${formatMetric(candidate.rejectedBandPrecision)}`, - `rejectedRecall=${formatMetric(candidate.rejectedRecall)}`, - `falseAcceptedRejected=${candidate.falseAcceptedRejectedLabels}`, - `falseRejectedAccepted=${candidate.falseRejectedAcceptedLabels}`, - ].join(' '); -} - -export function formatKtxRelationshipThresholdAdviceMarkdown(report: KtxRelationshipThresholdAdviceReport): string { - const lines = [ - 'KTX relationship threshold advice', - `Generated: ${report.generatedAt}`, - `Filter connection: ${report.filters.connectionId ?? 'all'}`, - `Status: ${report.status}`, - `Labels: total=${report.summary.totalLabels} scored=${report.summary.scoredLabels} accepted=${report.summary.acceptedLabels} rejected=${report.summary.rejectedLabels}`, - `Gates: minTotal=${report.gates.minTotalLabels} minAccepted=${report.gates.minAcceptedLabels} minRejected=${report.gates.minRejectedLabels} acceptedPrecision=${report.gates.minAcceptedBandPrecision.toFixed(3)} acceptedOrReviewRecall=${report.gates.minAcceptedOrReviewRecall.toFixed(3)} rejectedPrecision=${report.gates.minRejectedBandPrecision.toFixed(3)}`, - `Evaluated candidates: ${report.summary.evaluatedCandidates}`, - `Eligible candidates: ${report.summary.eligibleCandidates}`, - `Recommended: ${ - report.recommended - ? `accept=${report.recommended.acceptThreshold.toFixed(2)} review=${report.recommended.reviewThreshold.toFixed(2)}` - : 'none' - }`, - ]; - - if (report.reasons.length > 0) { - lines.push('', 'Reasons', ...report.reasons.map((reason) => ` - ${reason}`)); - } - - if (report.candidates.length > 0) { - lines.push('', 'Top candidates', ...report.candidates.slice(0, 5).map((candidate) => ` - ${candidateLine(candidate)}`)); - } - - if (report.warnings.length > 0) { - lines.push('', 'Warnings'); - for (const warning of report.warnings.slice(0, 5)) { - lines.push(` - ${warning.path}: ${warning.message}`); - } - } - - return `${lines.join('\n')}\n`; -} diff --git a/packages/context/src/search/index.ts b/packages/context/src/search/index.ts deleted file mode 100644 index 9cec3602..00000000 --- a/packages/context/src/search/index.ts +++ /dev/null @@ -1,44 +0,0 @@ -export type { - AssertSearchBackendCapabilitiesInput, - AssertSearchBackendConformanceCaseInput, - ExpectedSearchBackendConformanceLane, - SearchBackendConformanceDictionaryMatch, - SearchBackendConformanceLane, - SearchBackendConformanceResult, -} from './backend-conformance.js'; -export { - assertSearchBackendCapabilities, - assertSearchBackendConformanceCase, -} from './backend-conformance.js'; -export { createKtxDiscoverDataService } from './discover.js'; -export type { - KtxDiscoverDataInput, - KtxDiscoverDataKind, - KtxDiscoverDataMatchedOn, - KtxDiscoverDataRef, - KtxDiscoverDataResponse, - KtxDiscoverDataServiceOptions, -} from './discover.js'; -export { HybridSearchCore } from './hybrid-search-core.js'; -export { defaultLaneCandidatePoolLimit, normalizeSearchQuery } from './query.js'; -export { - compareFusedSearchCandidates, - DEFAULT_RRF_K, - DEFAULT_SEARCH_LANE_WEIGHTS, - rrfContribution, -} from './rrf.js'; -export type { - FusedSearchCandidate, - HybridSearchOptions, - HybridSearchResult, - NormalizedSearchQuery, - SearchBackendCapabilities, - SearchCandidate, - SearchCandidateGenerator, - SearchCandidateGeneratorArgs, - SearchLaneBreakdown, - SearchLaneName, - SearchLaneResult, - SearchLaneStatus, - SearchResultHydrator, -} from './types.js'; diff --git a/packages/context/src/skills/index.ts b/packages/context/src/skills/index.ts deleted file mode 100644 index 13f06853..00000000 --- a/packages/context/src/skills/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export type { FrontmatterFields, SkillCaller, SkillMetadata, SkillsRegistryServiceOptions } from './skills-registry.service.js'; -export { SkillsRegistryService } from './skills-registry.service.js'; diff --git a/packages/context/src/sl/index.ts b/packages/context/src/sl/index.ts deleted file mode 100644 index 600a5a93..00000000 --- a/packages/context/src/sl/index.ts +++ /dev/null @@ -1,44 +0,0 @@ -export type { SlValidationResult, SlValidatorPort } from './sl-validator.port.js'; -export type { - SemanticLayerQueryExecutionResult, - SemanticLayerQueryInput, - SemanticLayerSource, - SlDictionaryMatch, - SlSearchLaneSummary, - SlSearchMatchReason, - SlSearchMetadata, -} from './types.js'; -export type { - KtxConnectionInfo, - KtxQueryResult, - SlConnectionCatalogPort, - SlPythonPort, - SlSourcesIndexPort, -} from './ports.js'; -export { DEFAULT_PRIORITY, resolveDescription } from './descriptions.js'; -export { isOverlaySource, sourceDefinitionSchema, sourceOverlaySchema } from './schemas.js'; -export { - composeOverlay, - enrichColumnsFromManifest, - findDanglingSegmentRefs, - SemanticLayerService, -} from './semantic-layer.service.js'; -export { loadLatestSlDictionaryEntries } from './sl-dictionary-profile.js'; -export type { SlDictionaryEntry } from './sl-dictionary-profile.js'; -export { createKtxDictionarySearchService } from './dictionary-search.js'; -export type { - KtxDictionarySearchCoverage, - KtxDictionarySearchInput, - KtxDictionarySearchMatch, - KtxDictionarySearchMiss, - KtxDictionarySearchMissReason, - KtxDictionarySearchResponse, - KtxDictionarySearchSearchedConnection, - KtxDictionarySearchStatus, - KtxDictionarySearchValueResult, -} from './dictionary-search.js'; -export { buildSemanticLayerSourceSearchText, SlSearchService } from './sl-search.service.js'; -export { SqliteSlSourcesIndex, type SqliteSlSourcesIndexOptions } from './sqlite-sl-sources-index.js'; -export * from './local-sl.js'; -export * from './local-query.js'; -export * from './tools/index.js'; diff --git a/packages/context/src/sl/tools/index.ts b/packages/context/src/sl/tools/index.ts deleted file mode 100644 index 915f91ad..00000000 --- a/packages/context/src/sl/tools/index.ts +++ /dev/null @@ -1,11 +0,0 @@ -export type { BaseSemanticLayerToolDeps, SemanticLayerStructured } from './base-semantic-layer.tool.js'; -export { BaseSemanticLayerTool, sourceDefinitionSchema } from './base-semantic-layer.tool.js'; -export type { SlDiscoverySettings } from './sl-discover.tool.js'; -export { SlDiscoverTool } from './sl-discover.tool.js'; -export { SlEditSourceTool } from './sl-edit-source.tool.js'; -export { SlReadSourceTool } from './sl-read-source.tool.js'; -export { SlRollbackTool } from './sl-rollback.tool.js'; -export { SlValidateTool, validateSemanticLayerEndpoint } from './sl-validate.tool.js'; -export { SlWriteSourceTool } from './sl-write-source.tool.js'; -export type { SlValidationDeps, SourceValidationResult } from './sl-warehouse-validation.js'; -export { revertSourceToPreHead, validateSingleSource } from './sl-warehouse-validation.js'; diff --git a/packages/context/src/sql-analysis/index.ts b/packages/context/src/sql-analysis/index.ts deleted file mode 100644 index c01a8aaa..00000000 --- a/packages/context/src/sql-analysis/index.ts +++ /dev/null @@ -1,13 +0,0 @@ -export { createHttpSqlAnalysisPort } from './http-sql-analysis-port.js'; -export type { HttpSqlAnalysisPortOptions, KtxSqlAnalysisHttpJsonRunner } from './http-sql-analysis-port.js'; -export type { - SqlAnalysisBatchItem, - SqlAnalysisBatchResult, - SqlAnalysisClause, - SqlAnalysisDialect, - SqlAnalysisFingerprintResult, - SqlAnalysisLiteralSlot, - SqlAnalysisLiteralSlotType, - SqlAnalysisPort, - SqlReadOnlyValidationResult, -} from './ports.js'; diff --git a/packages/context/src/tools/index.ts b/packages/context/src/tools/index.ts deleted file mode 100644 index c6a334d5..00000000 --- a/packages/context/src/tools/index.ts +++ /dev/null @@ -1,45 +0,0 @@ -export type { GitAuthor, GitAuthorResolverPort } from './authors.js'; -export { SYSTEM_GIT_AUTHOR } from './authors.js'; -export type { - MethodologyEntry, - ToolContext, - ToolOutput, - ToolProgressRelayPort, - ToolTimingTrackerPort, -} from './base-tool.js'; -export { BaseTool } from './base-tool.js'; -export { ContextCandidateMarkTool } from './context-candidate-mark.tool.js'; -export { ContextCandidateWriteTool } from './context-candidate-write.tool.js'; -export { ContextEvidenceNeighborsTool } from './context-evidence-neighbors.tool.js'; -export { ContextEvidenceReadTool } from './context-evidence-read.tool.js'; -export { ContextEvidenceSearchTool } from './context-evidence-search.tool.js'; -export type { - ContextCandidateInsertResult, - ContextCandidateStatusResult, - ContextEvidenceChunkForCandidate, - ContextEvidenceChunkForRead, - ContextEvidenceChunkReadResult, - ContextEvidenceDocumentForRead, - ContextEvidenceNeighborResult, - ContextEvidenceReadResult, - ContextEvidenceSearchArgs, - ContextEvidenceSearchResult, - ContextEvidenceToolStorePort, -} from './context-evidence-tool-store.js'; -export type { ToolFailure } from './context-ingest-metadata.js'; -export { ingestMetadataRequired, resolveIngestMetadata } from './context-ingest-metadata.js'; -export type { SqlEdit } from './sql-edit-replacer.js'; -export { applySqlEdits } from './sql-edit-replacer.js'; -export type { IngestToolMetadata, MemoryAction, ToolSession } from './tool-session.js'; -export { validateActionRawPaths } from './action-raw-paths.js'; -export { validateActionTargetConnection } from './action-target-connection.js'; -export type { TouchedSlSource, TouchedSlSourceSet } from './touched-sl-sources.js'; -export { - addTouchedSlSource, - createTouchedSlSources, - deleteTouchedSlSource, - hasTouchedSlSource, - listTouchedSlSources, - touchedSlSourceCount, - touchedSlSourceNamesForConnection, -} from './touched-sl-sources.js'; diff --git a/packages/context/src/wiki/index.ts b/packages/context/src/wiki/index.ts deleted file mode 100644 index 17d37399..00000000 --- a/packages/context/src/wiki/index.ts +++ /dev/null @@ -1,37 +0,0 @@ -export { buildKnowledgeSearchText } from './knowledge-search-text.js'; -export { - assertFlatWikiKey, - invalidFlatWikiKeyMessage, - isFlatWikiKey, - suggestFlatWikiKey, - validateFlatWikiKey, -} from './keys.js'; -export { KnowledgeWikiService } from './knowledge-wiki.service.js'; -export * from './local-knowledge.js'; -export type { - KnowledgeEventPort, - KnowledgeGitDiffPort, - KnowledgeIndexPort, - KnowledgeIndexPageListing, - UpsertPageParams, - WikiFileStorePort, -} from './ports.js'; -export type { - ExistingKnowledgeIndexPage, - SqliteKnowledgeIndexOptions, - SqliteKnowledgeIndexPage, - SqliteKnowledgeIndexSearchResult, - WikiSqliteLaneCandidate, -} from './sqlite-knowledge-index.js'; -export { SqliteKnowledgeIndex } from './sqlite-knowledge-index.js'; -export * from './tools/index.js'; -export type { - HistoricSqlWikiUsageFrontmatter, - WikiFrontmatter, - WikiPage, - WikiPageWithScope, - WikiScope, - WikiSearchLaneSummary, - WikiSearchMatchReason, - WikiSearchMetadata, -} from './types.js'; diff --git a/packages/context/src/wiki/tools/index.ts b/packages/context/src/wiki/tools/index.ts deleted file mode 100644 index 8cc5ac9c..00000000 --- a/packages/context/src/wiki/tools/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export { WikiListTagsTool } from './wiki-list-tags.tool.js'; -export { WikiReadTool } from './wiki-read.tool.js'; -export { WikiRemoveTool } from './wiki-remove.tool.js'; -export { WikiSearchTool } from './wiki-search.tool.js'; -export { WikiWriteTool } from './wiki-write.tool.js'; diff --git a/packages/context/tsconfig.json b/packages/context/tsconfig.json deleted file mode 100644 index 965e6978..00000000 --- a/packages/context/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*.ts"], - "exclude": ["dist", "node_modules"] -} diff --git a/packages/context/vitest.config.ts b/packages/context/vitest.config.ts deleted file mode 100644 index 2339ffd3..00000000 --- a/packages/context/vitest.config.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { defineConfig } from 'vitest/config'; - -export default defineConfig({ - test: { - root: '.', - include: ['src/**/*.test.ts'], - }, -}); diff --git a/packages/llm/package.json b/packages/llm/package.json deleted file mode 100644 index 277e0a98..00000000 --- a/packages/llm/package.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "name": "@ktx/llm", - "version": "0.0.0-private", - "description": "Canonical KTX LLM and embedding provider package", - "private": true, - "type": "module", - "engines": { - "node": ">=22.0.0" - }, - "main": "dist/index.js", - "types": "dist/index.d.ts", - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - }, - "./package.json": "./package.json" - }, - "files": [ - "dist" - ], - "scripts": { - "build": "tsc -p tsconfig.json", - "test": "vitest run", - "type-check": "tsc -p tsconfig.json --noEmit" - }, - "dependencies": { - "@ai-sdk/anthropic": "3.0.77", - "@ai-sdk/devtools": "0.0.17", - "@ai-sdk/google-vertex": "^4.0.128", - "ai": "^6.0.180", - "openai": "^6.37.0" - }, - "devDependencies": { - "@types/node": "^25.7.0", - "@vitest/coverage-v8": "^4.1.6", - "typescript": "^6.0.3", - "vitest": "^4.1.6" - }, - "license": "Apache-2.0", - "repository": { - "type": "git", - "url": "git+https://github.com/kaelio/ktx.git", - "directory": "packages/llm" - }, - "bugs": { - "url": "https://github.com/kaelio/ktx/issues" - }, - "homepage": "https://github.com/kaelio/ktx#readme" -} diff --git a/packages/llm/src/index.ts b/packages/llm/src/index.ts deleted file mode 100644 index ab2ad341..00000000 --- a/packages/llm/src/index.ts +++ /dev/null @@ -1,31 +0,0 @@ -export { createKtxEmbeddingProvider } from './embedding-provider.js'; -export { runKtxEmbeddingHealthCheck } from './embedding-health.js'; -export { KtxMessageBuilder, splitKtxSystemMessages } from './message-builder.js'; -export type { KtxSplitSystemMessagesResult } from './message-builder.js'; -export type { KtxEmbeddingHealthCheckOptions, KtxEmbeddingHealthCheckResult } from './embedding-health.js'; -export type { KtxEmbeddingProviderDeps } from './embedding-provider.js'; -export type { KtxLlmHealthCheckDeps, KtxLlmHealthCheckOptions, KtxLlmHealthCheckResult } from './model-health.js'; -export { runKtxLlmHealthCheck } from './model-health.js'; -export { - createKtxLlmProvider, - isAnthropicProtocolModel, - modelIdFromLanguageModel, - type KtxLlmProviderFactoryDeps, -} from './model-provider.js'; -export type { - KtxEmbeddingBackend, - KtxEmbeddingConfig, - KtxEmbeddingProvider, - KtxEmbeddingTokenUsageEvent, - KtxJsonValue, - KtxLlmBackend, - KtxLlmConfig, - KtxLlmProvider, - KtxModelRole, - KtxPromptCacheTtl, - KtxPromptCachingConfig, - KtxPromptParts, - KtxProviderOptions, - KtxTokenUsageEvent, -} from './types.js'; -export { KTX_MODEL_ROLES } from './types.js'; diff --git a/packages/llm/src/package-exports.test.ts b/packages/llm/src/package-exports.test.ts deleted file mode 100644 index cf080325..00000000 --- a/packages/llm/src/package-exports.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -describe('@ktx/llm package exports', () => { - it('exports the canonical LLM and embedding surfaces', async () => { - const llm = await import('./index.js'); - - expect(llm.KTX_MODEL_ROLES).toEqual([ - 'default', - 'triage', - 'candidateExtraction', - 'curator', - 'reconcile', - 'repair', - ]); - expect(llm.createKtxLlmProvider).toBeTypeOf('function'); - expect(llm.KtxMessageBuilder).toBeTypeOf('function'); - expect(llm.createKtxEmbeddingProvider).toBeTypeOf('function'); - }); -}); diff --git a/packages/llm/tsconfig.json b/packages/llm/tsconfig.json deleted file mode 100644 index 965e6978..00000000 --- a/packages/llm/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*.ts"], - "exclude": ["dist", "node_modules"] -} diff --git a/packages/llm/vitest.config.ts b/packages/llm/vitest.config.ts deleted file mode 100644 index 2339ffd3..00000000 --- a/packages/llm/vitest.config.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { defineConfig } from 'vitest/config'; - -export default defineConfig({ - test: { - root: '.', - include: ['src/**/*.test.ts'], - }, -}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ac2398ca..d513e057 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -22,6 +22,12 @@ importers: '@biomejs/biome': specifier: ^2.4.15 version: 2.4.15 + '@electric-sql/pglite': + specifier: ^0.4.5 + version: 0.4.5 + '@electric-sql/pglite-socket': + specifier: ^0.1.5 + version: 0.1.5(@electric-sql/pglite@0.4.5) '@semantic-release/commit-analyzer': specifier: ^13.0.1 version: 13.0.1(semantic-release@25.0.3(typescript@6.0.3)) @@ -52,6 +58,9 @@ importers: knip: specifier: ^6.12.2 version: 6.12.2(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0) + pg: + specifier: ^8.20.0 + version: 8.20.0 semantic-release: specifier: ^25.0.3 version: 25.0.3(typescript@6.0.3) @@ -107,254 +116,30 @@ importers: packages/cli: dependencies: - '@clack/prompts': - specifier: 1.4.0 - version: 1.4.0 - '@commander-js/extra-typings': - specifier: 14.0.0 - version: 14.0.0(commander@14.0.3) - '@ktx/connector-bigquery': - specifier: workspace:* - version: file:packages/connector-bigquery(js-yaml@4.1.1)(ws@8.20.1) - '@ktx/connector-clickhouse': - specifier: workspace:* - version: file:packages/connector-clickhouse(js-yaml@4.1.1)(ws@8.20.1) - '@ktx/connector-mysql': - specifier: workspace:* - version: file:packages/connector-mysql(@types/node@24.12.2)(js-yaml@4.1.1)(ws@8.20.1) - '@ktx/connector-postgres': - specifier: workspace:* - version: file:packages/connector-postgres(js-yaml@4.1.1)(ws@8.20.1) - '@ktx/connector-snowflake': - specifier: workspace:* - version: file:packages/connector-snowflake(asn1.js@5.4.1)(js-yaml@4.1.1)(ws@8.20.1) - '@ktx/connector-sqlite': - specifier: workspace:* - version: file:packages/connector-sqlite(js-yaml@4.1.1)(ws@8.20.1) - '@ktx/connector-sqlserver': - specifier: workspace:* - version: file:packages/connector-sqlserver(js-yaml@4.1.1)(ws@8.20.1) - '@ktx/context': - specifier: workspace:* - version: link:../context - '@ktx/llm': - specifier: workspace:* - version: link:../llm - '@modelcontextprotocol/sdk': - specifier: ^1.29.0 - version: 1.29.0(zod@4.4.3) - commander: - specifier: 14.0.3 - version: 14.0.3 - fflate: - specifier: ^0.8.2 - version: 0.8.2 - ink: - specifier: ^7.0.2 - version: 7.0.2(@types/react@19.2.14)(react@19.2.6) - react: - specifier: ^19.2.6 - version: 19.2.6 - zod: - specifier: ^4.4.3 - version: 4.4.3 - devDependencies: - '@types/better-sqlite3': - specifier: ^7.6.13 - version: 7.6.13 - '@types/node': - specifier: ^24.3.0 - version: 24.12.2 - '@types/react': - specifier: ^19.2.14 - version: 19.2.14 - '@vitest/coverage-v8': - specifier: ^4.1.6 - version: 4.1.6(vitest@4.1.6) - better-sqlite3: - specifier: ^12.10.0 - version: 12.10.0 - ink-testing-library: - specifier: ^4.0.0 - version: 4.0.0(@types/react@19.2.14) - typescript: - specifier: ^6.0.3 - version: 6.0.3 - vitest: - specifier: ^4.1.6 - version: 4.1.6(@opentelemetry/api@1.9.0)(@types/node@24.12.2)(@vitest/coverage-v8@4.1.6)(vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) - - packages/connector-bigquery: - dependencies: - '@google-cloud/bigquery': - specifier: ^8.3.1 - version: 8.3.1 - '@ktx/context': - specifier: workspace:* - version: file:packages/context(js-yaml@4.1.1) - devDependencies: - '@types/node': - specifier: ^24.3.0 - version: 24.12.2 - '@vitest/coverage-v8': - specifier: ^4.1.6 - version: 4.1.6(vitest@4.1.6) - typescript: - specifier: ^6.0.3 - version: 6.0.3 - vitest: - specifier: ^4.1.6 - version: 4.1.6(@opentelemetry/api@1.9.0)(@types/node@24.12.2)(@vitest/coverage-v8@4.1.6)(vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) - - packages/connector-clickhouse: - dependencies: - '@clickhouse/client': - specifier: ^1.18.4 - version: 1.18.4 - '@ktx/context': - specifier: workspace:* - version: file:packages/context(js-yaml@4.1.1) - devDependencies: - '@types/node': - specifier: ^24.3.0 - version: 24.12.2 - '@vitest/coverage-v8': - specifier: ^4.1.6 - version: 4.1.6(vitest@4.1.6) - typescript: - specifier: ^6.0.3 - version: 6.0.3 - vitest: - specifier: ^4.1.6 - version: 4.1.6(@opentelemetry/api@1.9.0)(@types/node@24.12.2)(@vitest/coverage-v8@4.1.6)(vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) - - packages/connector-mysql: - dependencies: - '@ktx/context': - specifier: workspace:* - version: file:packages/context(js-yaml@4.1.1) - mysql2: - specifier: ^3.22.3 - version: 3.22.3(@types/node@24.12.2) - devDependencies: - '@types/node': - specifier: ^24.3.0 - version: 24.12.2 - '@vitest/coverage-v8': - specifier: ^4.1.6 - version: 4.1.6(vitest@4.1.6) - typescript: - specifier: ^6.0.3 - version: 6.0.3 - vitest: - specifier: ^4.1.6 - version: 4.1.6(@opentelemetry/api@1.9.0)(@types/node@24.12.2)(@vitest/coverage-v8@4.1.6)(vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) - - packages/connector-postgres: - dependencies: - '@ktx/context': - specifier: workspace:* - version: file:packages/context(js-yaml@4.1.1) - pg: - specifier: ^8.20.0 - version: 8.20.0 - devDependencies: - '@types/node': - specifier: ^24.3.0 - version: 24.12.2 - '@types/pg': - specifier: ^8.20.0 - version: 8.20.0 - '@vitest/coverage-v8': - specifier: ^4.1.6 - version: 4.1.6(vitest@4.1.6) - typescript: - specifier: ^6.0.3 - version: 6.0.3 - vitest: - specifier: ^4.1.6 - version: 4.1.6(@opentelemetry/api@1.9.0)(@types/node@24.12.2)(@vitest/coverage-v8@4.1.6)(vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) - - packages/connector-snowflake: - dependencies: - '@ktx/context': - specifier: workspace:* - version: file:packages/context(js-yaml@4.1.1) - snowflake-sdk: - specifier: ^2.4.1 - version: 2.4.1(asn1.js@5.4.1) - devDependencies: - '@types/node': - specifier: ^24.3.0 - version: 24.12.2 - '@vitest/coverage-v8': - specifier: ^4.1.6 - version: 4.1.6(vitest@4.1.6) - typescript: - specifier: ^6.0.3 - version: 6.0.3 - vitest: - specifier: ^4.1.6 - version: 4.1.6(@opentelemetry/api@1.9.0)(@types/node@24.12.2)(@vitest/coverage-v8@4.1.6)(vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) - - packages/connector-sqlite: - dependencies: - '@ktx/context': - specifier: workspace:* - version: file:packages/context(js-yaml@4.1.1) - better-sqlite3: - specifier: ^12.10.0 - version: 12.10.0 - devDependencies: - '@types/better-sqlite3': - specifier: ^7.6.13 - version: 7.6.13 - '@types/node': - specifier: ^24.3.0 - version: 24.12.2 - '@vitest/coverage-v8': - specifier: ^4.1.6 - version: 4.1.6(vitest@4.1.6) - typescript: - specifier: ^6.0.3 - version: 6.0.3 - vitest: - specifier: ^4.1.6 - version: 4.1.6(@opentelemetry/api@1.9.0)(@types/node@24.12.2)(@vitest/coverage-v8@4.1.6)(vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) - - packages/connector-sqlserver: - dependencies: - '@ktx/context': - specifier: workspace:* - version: file:packages/context(js-yaml@4.1.1) - mssql: - specifier: ^12.5.2 - version: 12.5.2(@azure/core-client@1.10.1) - devDependencies: - '@types/mssql': - specifier: ^12.3.0 - version: 12.3.0(@azure/core-client@1.10.1) - '@types/node': - specifier: ^24.3.0 - version: 24.12.2 - '@vitest/coverage-v8': - specifier: ^4.1.6 - version: 4.1.6(vitest@4.1.6) - typescript: - specifier: ^6.0.3 - version: 6.0.3 - vitest: - specifier: ^4.1.6 - version: 4.1.6(@opentelemetry/api@1.9.0)(@types/node@24.12.2)(@vitest/coverage-v8@4.1.6)(vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) - - packages/context: - dependencies: + '@ai-sdk/anthropic': + specifier: 3.0.77 + version: 3.0.77(zod@4.4.3) + '@ai-sdk/devtools': + specifier: 0.0.17 + version: 0.0.17 + '@ai-sdk/google-vertex': + specifier: ^4.0.128 + version: 4.0.128(zod@4.4.3) '@anthropic-ai/claude-agent-sdk': specifier: 0.3.142 version: 0.3.142(zod@4.4.3) - '@ktx/llm': - specifier: workspace:* - version: link:../llm + '@clack/prompts': + specifier: 1.4.0 + version: 1.4.0 + '@clickhouse/client': + specifier: ^1.18.4 + version: 1.18.4 + '@commander-js/extra-typings': + specifier: 14.0.0 + version: 14.0.0(commander@14.0.3) + '@google-cloud/bigquery': + specifier: ^8.3.1 + version: 8.3.1 '@looker/sdk': specifier: ^26.8.0 version: 26.8.0 @@ -376,24 +161,48 @@ importers: better-sqlite3: specifier: ^12.10.0 version: 12.10.0 + commander: + specifier: 14.0.3 + version: 14.0.3 + fflate: + specifier: ^0.8.2 + version: 0.8.2 handlebars: specifier: ^4.7.9 version: 4.7.9 + ink: + specifier: ^7.0.2 + version: 7.0.2(@types/react@19.2.14)(react@19.2.6) lookml-parser: specifier: 7.1.0 version: 7.1.0(js-yaml@4.1.1) minimatch: specifier: ^10.2.5 version: 10.2.5 + mssql: + specifier: ^12.5.2 + version: 12.5.2(@azure/core-client@1.10.1) + mysql2: + specifier: ^3.22.3 + version: 3.22.3(@types/node@24.12.2) + openai: + specifier: ^6.37.0 + version: 6.37.0(ws@8.20.1)(zod@4.4.3) p-limit: specifier: ^7.3.0 version: 7.3.0 pg: specifier: ^8.20.0 version: 8.20.0 + react: + specifier: ^19.2.6 + version: 19.2.6 simple-git: specifier: 3.36.0 version: 3.36.0 + snowflake-sdk: + specifier: ^2.4.1 + version: 2.4.1(asn1.js@5.4.1) yaml: specifier: ^2.9.0 version: 2.9.0 @@ -410,49 +219,27 @@ importers: '@types/better-sqlite3': specifier: ^7.6.13 version: 7.6.13 + '@types/mssql': + specifier: ^12.3.0 + version: 12.3.0(@azure/core-client@1.10.1) '@types/node': specifier: ^24.3.0 version: 24.12.2 '@types/pg': specifier: ^8.20.0 version: 8.20.0 + '@types/react': + specifier: ^19.2.14 + version: 19.2.14 '@vitest/coverage-v8': specifier: ^4.1.6 version: 4.1.6(vitest@4.1.6) ajv: specifier: 8.20.0 version: 8.20.0 - typescript: - specifier: ^6.0.3 - version: 6.0.3 - vitest: - specifier: ^4.1.6 - version: 4.1.6(@opentelemetry/api@1.9.0)(@types/node@24.12.2)(@vitest/coverage-v8@4.1.6)(vite@8.0.10(@types/node@24.12.2)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) - - packages/llm: - dependencies: - '@ai-sdk/anthropic': - specifier: 3.0.77 - version: 3.0.77(zod@4.4.3) - '@ai-sdk/devtools': - specifier: 0.0.17 - version: 0.0.17 - '@ai-sdk/google-vertex': - specifier: ^4.0.128 - version: 4.0.128(zod@4.4.3) - ai: - specifier: ^6.0.180 - version: 6.0.180(zod@4.4.3) - openai: - specifier: ^6.37.0 - version: 6.37.0(ws@8.20.1)(zod@4.4.3) - devDependencies: - '@types/node': - specifier: ^24.3.0 - version: 24.12.2 - '@vitest/coverage-v8': - specifier: ^4.1.6 - version: 4.1.6(vitest@4.1.6) + ink-testing-library: + specifier: ^4.0.0 + version: 4.0.0(@types/react@19.2.14) typescript: specifier: ^6.0.3 version: 6.0.3 @@ -617,10 +404,6 @@ packages: resolution: {integrity: sha512-oDJJ7rM1osvfBdfZuhQ5DM6lHD9iuypL9m2LsEiA/lB8xuE5uPYsftNDcS0J9VRXFSvYTqC14K7Y5vMMKMg0vw==} engines: {node: '>=20.0.0'} - '@aws-sdk/core@3.974.7': - resolution: {integrity: sha512-YhRC90ofz5oolTJZlA8voU/oUrCB2azi8Usx51k8hhB5LpWbYQMMXKUqSqkoL0Cru+RQJgWTHpAfEDDIwfUhJw==} - engines: {node: '>=20.0.0'} - '@aws-sdk/core@3.974.8': resolution: {integrity: sha512-njR2qoG6ZuB0kvAS2FyICsFZJ6gmCcf2X/7JcD14sUvGDm26wiZ5BrA6LOiUxKFEF+IVe7kdroxyE00YlkiYsw==} engines: {node: '>=20.0.0'} @@ -629,66 +412,34 @@ packages: resolution: {integrity: sha512-QUagVVBbC8gODCF6e1aV0mE2TXWB9Opz4k8EJFdNrujUVQm5R4AjJa1mpOqzwOuROBzqJU9zawzig7M96L8Ejg==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-env@3.972.33': - resolution: {integrity: sha512-bJV7eViSJV6GSuuN+VIdNVPdwPsNSf75BiC2v5alPrjR/OCcqgKwSZInKbDFz9mNeizldsyf67jt6YSIiv53Cw==} - engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-env@3.972.34': resolution: {integrity: sha512-XT0jtf8Fw9JE6ppsQeoNnZRiG+jqRixMT1v1ZR17G60UvVdsQmTG8nbEyHuEPfMxDXEhfdARaM/XiEhca4lGHQ==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-http@3.972.35': - resolution: {integrity: sha512-x/BQGEIdq0oI+4WxLjKmnQvT7CnF9r8ezdGt7wXwxb7ckHXQz0Zmgxt8v3Ne0JaT3R5YefmuybHX6E8EnsDXyA==} - engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-http@3.972.36': resolution: {integrity: sha512-DPoGWfy7J7RKxvbf5kOKIGQkD2ek3dbKgzKIGrnLuvZBz5myU+Im/H6pmc14QcnFbqHMqxvtWSgRDSJW3qXLQg==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-ini@3.972.37': - resolution: {integrity: sha512-eUTpmWfd/BKsq9medhCRcu+GRAhFP2Zrn7/2jKDHHOOjCkhrMoTp/t4cEthqFoG7gE0VGp5wUxrXTdvBCmSmJg==} - engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-ini@3.972.38': resolution: {integrity: sha512-oDzUBu2MGJFgoar05sPMCwSrhw44ASyccrHzj66vO69OZqi7I6hZZxXfuPLC8OCzW7C+sU+bI73XHij41yekgQ==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-login@3.972.37': - resolution: {integrity: sha512-Ty68y8ISSC+g5Q3D0K8uAaoINwvfaOslnNpsF/LgVUxyosYXHawcK2yV4HLXDVugiTTYLQfJfcw0ce5meAGkKw==} - engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-login@3.972.38': resolution: {integrity: sha512-g1NosS8qe4OF++G2UFCM5ovSkgipC7YYor5KCWatG0UoMSO5YFj9C8muePlyVmOBV/WTI16Jo3/s1NUo/o1Bww==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-node@3.972.38': - resolution: {integrity: sha512-BQ9XYnBDVxR2HuV5huXYQYF/PZMTsY+EnwfGnCU2cA8Zw63XpkOtPY8WqiMIZMQCrKPQQEiFURS/o9CIolRLqg==} - engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-node@3.972.39': resolution: {integrity: sha512-HEswDQyxUtadoZ/bJsPPENHg7R0Lzym5LuMksJeHvqhCOpP+rtkDLKI4/ZChH4w3cf5kG8n6bZuI8PzajoiqMg==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-process@3.972.33': - resolution: {integrity: sha512-yfjGksI9WQbdMObb0VeLXqzTLI+a0qXLJT9gCDiv0+X/xjPpI3mTz6a5FibrhpuEKIe0gSgvs3MaoFZy5cx4WA==} - engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-process@3.972.34': resolution: {integrity: sha512-T3IFs4EVmVi1dVN5RciFnklCANSzvrQd/VuHY9ThHSQmYkTogjcGkoJEr+oNUPQZnso52183088NqysMPji1/Q==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-sso@3.972.37': - resolution: {integrity: sha512-fpwE+20ntpp3i9Xb9vUuQfXLDKYHH+5I2V+ZG96SX1nBzrruhy10RXDgmN7t1etOz3c55stlA3TeQASUA451NQ==} - engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-sso@3.972.38': resolution: {integrity: sha512-5ZxG+t0+3Q3QPh8KEjX6syskhgNf7I0MN7oGioTf6Lm1NTjfP7sIcYGNsthXC2qR8vcD3edNZwCr2ovfSSWuRA==} engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-web-identity@3.972.37': - resolution: {integrity: sha512-aryawqyebf+3WhAFNHfF62rekFpYtVcVN7dQ89qnAWsa4n5hJst8qBG6gXC24WHtW7Nnhkf9ScYnjwo0Brn3bw==} - engines: {node: '>=20.0.0'} - '@aws-sdk/credential-provider-web-identity@3.972.38': resolution: {integrity: sha512-lYHFF30DGI20jZcYX8cm6Ns0V7f1dDN6g/MBDLTyD/5iw+bXs3yBr2iAiHDkx4RFU5JgsnZvCHYKiRVPRdmOgw==} engines: {node: '>=20.0.0'} @@ -725,10 +476,6 @@ packages: resolution: {integrity: sha512-+zz6f79Kj9V5qFK2P+D8Ehjnw4AhphAlCAsPjUqEcInA9umtSSKMrHbSagEeOIsDNuvVrH98bjRHcyQukTrhaQ==} engines: {node: '>=20.0.0'} - '@aws-sdk/middleware-sdk-s3@3.972.36': - resolution: {integrity: sha512-YhPix+0x/MdQrb1Ug1GDKeS5fqylIy+naz800asX8II4jqfTk2KY2KhmmYCwZcky8YWtRQQwWCGdoqeAnip8Uw==} - engines: {node: '>=20.0.0'} - '@aws-sdk/middleware-sdk-s3@3.972.37': resolution: {integrity: sha512-Km7M+i8DrLArVzrid1gfxeGhYHBd3uxvE77g0s5a52zPSVosxzQBnJ0gwWb6NIp/DOk8gsBMhi7V+cpJG0ndTA==} engines: {node: '>=20.0.0'} @@ -737,18 +484,10 @@ packages: resolution: {integrity: sha512-Gli9A0u8EVVb+5bFDGS/QbSVg28w/wpEidg1ggVcSj65BDTdGR6punsOcVjqdiu1i42WHWo51MCvARPIIz9juw==} engines: {node: '>=20.0.0'} - '@aws-sdk/middleware-user-agent@3.972.37': - resolution: {integrity: sha512-N1oNpdiLoVAWYD3WFBnUi3LlfoDA06ZHo4ozyjbsJNLvILzvt//0CnR8N+CZ0NWeYgVB/5V59ivixHCWCx2ALw==} - engines: {node: '>=20.0.0'} - '@aws-sdk/middleware-user-agent@3.972.38': resolution: {integrity: sha512-iz+B29TXcAZsJpwB+AwG/TTGA5l/VnmMZ2UxtiySOZjI6gCdmviXPwdgzcmuazMy16rXoPY4mYCGe7zdNKfx5A==} engines: {node: '>=20.0.0'} - '@aws-sdk/nested-clients@3.997.5': - resolution: {integrity: sha512-jGFr6DxtcMTmzOkG/a0jCZYv4BBDmeNYVeO+/memSoDkYCJu4Y58xviYmzwJfYyIVSts+X/BVjJm1uGBnwHEMg==} - engines: {node: '>=20.0.0'} - '@aws-sdk/nested-clients@3.997.6': resolution: {integrity: sha512-WBDnqatJl+kGObpfmfSxqnXeYTu3Me8wx8WCtvoxX3pfWrrTv8I4WTMSSs7PZqcRcVh8WeUKMgGFjMG+52SR1w==} engines: {node: '>=20.0.0'} @@ -757,18 +496,10 @@ packages: resolution: {integrity: sha512-CvJ2ZIjK/jVD/lbOpowBVElJyC1YxLTIJ13yM0AEo0t2v7swOzGjSA6lJGH+DwZXQhcjUjoYwc8bVYCX5MDr1A==} engines: {node: '>=20.0.0'} - '@aws-sdk/signature-v4-multi-region@3.996.24': - resolution: {integrity: sha512-amP7tLikppN940wbBFISYqiuzVmpzMS9U3mcgtmVLjX4fdWI/SNCvrXv6ZxfVzTT4cT0rPKOLhFah2xLwzREWw==} - engines: {node: '>=20.0.0'} - '@aws-sdk/signature-v4-multi-region@3.996.25': resolution: {integrity: sha512-+CMIt3e1VzlklAECmG+DtP1sV8iKq25FuA0OKpnJ4KA0kxUtd7CgClY7/RU6VzJBQwbN4EJ9Ue6plvqx1qGadw==} engines: {node: '>=20.0.0'} - '@aws-sdk/token-providers@3.1039.0': - resolution: {integrity: sha512-NMSFL2HwkAOoCeLCQiqoOq5pT3vVbSjww2QZTuYgYknVwhhv125PSDzZIcL5EYnlxuPWjEOdauZK+FspkZDVdw==} - engines: {node: '>=20.0.0'} - '@aws-sdk/token-providers@3.1041.0': resolution: {integrity: sha512-Th7kPI6YPtvJUcdznooXJMy+9rQWjmEF81LxaJssngBzuysK4a/x+l8kjm1zb7nYsUPbndnBdUnwng/3PLvtGw==} engines: {node: '>=20.0.0'} @@ -792,15 +523,6 @@ packages: '@aws-sdk/util-user-agent-browser@3.972.10': resolution: {integrity: sha512-FAzqXvfEssGdSIz8ejatan0bOdx1qefBWKF/gWmVBXIP1HkS7v/wjjaqrAGGKvyihrXTXW00/2/1nTJtxpXz7g==} - '@aws-sdk/util-user-agent-node@3.973.23': - resolution: {integrity: sha512-gGwq8L2Euw0aNG6Ey4EktiAo3fSCVoDy1CaBIthd+oeaKHPXUrNaApMewQ6La5Hv0lcznOtECZaNvYyc5LXXfA==} - engines: {node: '>=20.0.0'} - peerDependencies: - aws-crt: '>=1.0.0' - peerDependenciesMeta: - aws-crt: - optional: true - '@aws-sdk/util-user-agent-node@3.973.24': resolution: {integrity: sha512-ZWwlkjcIp7cEL8ZfTpTAPNkwx25p7xol0xlKoWVVf22+nsjwmLcHYtTPjIV1cSpmB/b6DaK4cb1fSkvCXHgRdw==} engines: {node: '>=20.0.0'} @@ -1420,42 +1142,6 @@ packages: '@js-joda/core@5.7.0': resolution: {integrity: sha512-WBu4ULVVxySLLzK1Ppq+OdfP+adRS4ntmDQT915rzDJ++i95gc2jZkM5B6LWEAwN3lGXpfie3yPABozdD3K3Vg==} - '@ktx/connector-bigquery@file:packages/connector-bigquery': - resolution: {directory: packages/connector-bigquery, type: directory} - engines: {node: '>=22.0.0'} - - '@ktx/connector-clickhouse@file:packages/connector-clickhouse': - resolution: {directory: packages/connector-clickhouse, type: directory} - engines: {node: '>=22.0.0'} - - '@ktx/connector-mysql@file:packages/connector-mysql': - resolution: {directory: packages/connector-mysql, type: directory} - engines: {node: '>=22.0.0'} - - '@ktx/connector-postgres@file:packages/connector-postgres': - resolution: {directory: packages/connector-postgres, type: directory} - engines: {node: '>=22.0.0'} - - '@ktx/connector-snowflake@file:packages/connector-snowflake': - resolution: {directory: packages/connector-snowflake, type: directory} - engines: {node: '>=22.0.0'} - - '@ktx/connector-sqlite@file:packages/connector-sqlite': - resolution: {directory: packages/connector-sqlite, type: directory} - engines: {node: '>=22.0.0'} - - '@ktx/connector-sqlserver@file:packages/connector-sqlserver': - resolution: {directory: packages/connector-sqlserver, type: directory} - engines: {node: '>=22.0.0'} - - '@ktx/context@file:packages/context': - resolution: {directory: packages/context, type: directory} - engines: {node: '>=22.0.0'} - - '@ktx/llm@file:packages/llm': - resolution: {directory: packages/llm, type: directory} - engines: {node: '>=22.0.0'} - '@kwsites/file-exists@1.1.1': resolution: {integrity: sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==} @@ -6528,23 +6214,6 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/core@3.974.7': - dependencies: - '@aws-sdk/types': 3.973.8 - '@aws-sdk/xml-builder': 3.972.22 - '@smithy/core': 3.23.17 - '@smithy/node-config-provider': 4.3.14 - '@smithy/property-provider': 4.2.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/signature-v4': 5.3.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - '@smithy/util-base64': 4.3.2 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-retry': 4.3.6 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - '@aws-sdk/core@3.974.8': dependencies: '@aws-sdk/types': 3.973.8 @@ -6567,14 +6236,6 @@ snapshots: '@smithy/types': 4.14.1 tslib: 2.8.1 - '@aws-sdk/credential-provider-env@3.972.33': - dependencies: - '@aws-sdk/core': 3.974.7 - '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - '@aws-sdk/credential-provider-env@3.972.34': dependencies: '@aws-sdk/core': 3.974.8 @@ -6583,19 +6244,6 @@ snapshots: '@smithy/types': 4.14.1 tslib: 2.8.1 - '@aws-sdk/credential-provider-http@3.972.35': - dependencies: - '@aws-sdk/core': 3.974.7 - '@aws-sdk/types': 3.973.8 - '@smithy/fetch-http-handler': 5.3.17 - '@smithy/node-http-handler': 4.6.1 - '@smithy/property-provider': 4.2.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - '@smithy/util-stream': 4.5.25 - tslib: 2.8.1 - '@aws-sdk/credential-provider-http@3.972.36': dependencies: '@aws-sdk/core': 3.974.8 @@ -6609,25 +6257,6 @@ snapshots: '@smithy/util-stream': 4.5.25 tslib: 2.8.1 - '@aws-sdk/credential-provider-ini@3.972.37': - dependencies: - '@aws-sdk/core': 3.974.7 - '@aws-sdk/credential-provider-env': 3.972.33 - '@aws-sdk/credential-provider-http': 3.972.35 - '@aws-sdk/credential-provider-login': 3.972.37 - '@aws-sdk/credential-provider-process': 3.972.33 - '@aws-sdk/credential-provider-sso': 3.972.37 - '@aws-sdk/credential-provider-web-identity': 3.972.37 - '@aws-sdk/nested-clients': 3.997.5 - '@aws-sdk/types': 3.973.8 - '@smithy/credential-provider-imds': 4.2.14 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - '@aws-sdk/credential-provider-ini@3.972.38': dependencies: '@aws-sdk/core': 3.974.8 @@ -6647,19 +6276,6 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/credential-provider-login@3.972.37': - dependencies: - '@aws-sdk/core': 3.974.7 - '@aws-sdk/nested-clients': 3.997.5 - '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - '@aws-sdk/credential-provider-login@3.972.38': dependencies: '@aws-sdk/core': 3.974.8 @@ -6673,23 +6289,6 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/credential-provider-node@3.972.38': - dependencies: - '@aws-sdk/credential-provider-env': 3.972.33 - '@aws-sdk/credential-provider-http': 3.972.35 - '@aws-sdk/credential-provider-ini': 3.972.37 - '@aws-sdk/credential-provider-process': 3.972.33 - '@aws-sdk/credential-provider-sso': 3.972.37 - '@aws-sdk/credential-provider-web-identity': 3.972.37 - '@aws-sdk/types': 3.973.8 - '@smithy/credential-provider-imds': 4.2.14 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - '@aws-sdk/credential-provider-node@3.972.39': dependencies: '@aws-sdk/credential-provider-env': 3.972.34 @@ -6707,15 +6306,6 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/credential-provider-process@3.972.33': - dependencies: - '@aws-sdk/core': 3.974.7 - '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - '@aws-sdk/credential-provider-process@3.972.34': dependencies: '@aws-sdk/core': 3.974.8 @@ -6725,19 +6315,6 @@ snapshots: '@smithy/types': 4.14.1 tslib: 2.8.1 - '@aws-sdk/credential-provider-sso@3.972.37': - dependencies: - '@aws-sdk/core': 3.974.7 - '@aws-sdk/nested-clients': 3.997.5 - '@aws-sdk/token-providers': 3.1039.0 - '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - '@aws-sdk/credential-provider-sso@3.972.38': dependencies: '@aws-sdk/core': 3.974.8 @@ -6751,18 +6328,6 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/credential-provider-web-identity@3.972.37': - dependencies: - '@aws-sdk/core': 3.974.7 - '@aws-sdk/nested-clients': 3.997.5 - '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - '@aws-sdk/credential-provider-web-identity@3.972.38': dependencies: '@aws-sdk/core': 3.974.8 @@ -6846,23 +6411,6 @@ snapshots: '@smithy/types': 4.14.1 tslib: 2.8.1 - '@aws-sdk/middleware-sdk-s3@3.972.36': - dependencies: - '@aws-sdk/core': 3.974.7 - '@aws-sdk/types': 3.973.8 - '@aws-sdk/util-arn-parser': 3.972.3 - '@smithy/core': 3.23.17 - '@smithy/node-config-provider': 4.3.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/signature-v4': 5.3.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - '@smithy/util-config-provider': 4.2.2 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-stream': 4.5.25 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - '@aws-sdk/middleware-sdk-s3@3.972.37': dependencies: '@aws-sdk/core': 3.974.8 @@ -6886,17 +6434,6 @@ snapshots: '@smithy/types': 4.14.1 tslib: 2.8.1 - '@aws-sdk/middleware-user-agent@3.972.37': - dependencies: - '@aws-sdk/core': 3.974.7 - '@aws-sdk/types': 3.973.8 - '@aws-sdk/util-endpoints': 3.996.8 - '@smithy/core': 3.23.17 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - '@smithy/util-retry': 4.3.6 - tslib: 2.8.1 - '@aws-sdk/middleware-user-agent@3.972.38': dependencies: '@aws-sdk/core': 3.974.8 @@ -6908,50 +6445,6 @@ snapshots: '@smithy/util-retry': 4.3.6 tslib: 2.8.1 - '@aws-sdk/nested-clients@3.997.5': - dependencies: - '@aws-crypto/sha256-browser': 5.2.0 - '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/core': 3.974.7 - '@aws-sdk/middleware-host-header': 3.972.10 - '@aws-sdk/middleware-logger': 3.972.10 - '@aws-sdk/middleware-recursion-detection': 3.972.11 - '@aws-sdk/middleware-user-agent': 3.972.37 - '@aws-sdk/region-config-resolver': 3.972.13 - '@aws-sdk/signature-v4-multi-region': 3.996.24 - '@aws-sdk/types': 3.973.8 - '@aws-sdk/util-endpoints': 3.996.8 - '@aws-sdk/util-user-agent-browser': 3.972.10 - '@aws-sdk/util-user-agent-node': 3.973.23 - '@smithy/config-resolver': 4.4.17 - '@smithy/core': 3.23.17 - '@smithy/fetch-http-handler': 5.3.17 - '@smithy/hash-node': 4.2.14 - '@smithy/invalid-dependency': 4.2.14 - '@smithy/middleware-content-length': 4.2.14 - '@smithy/middleware-endpoint': 4.4.32 - '@smithy/middleware-retry': 4.5.7 - '@smithy/middleware-serde': 4.2.20 - '@smithy/middleware-stack': 4.2.14 - '@smithy/node-config-provider': 4.3.14 - '@smithy/node-http-handler': 4.6.1 - '@smithy/protocol-http': 5.3.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - '@smithy/url-parser': 4.2.14 - '@smithy/util-base64': 4.3.2 - '@smithy/util-body-length-browser': 4.2.2 - '@smithy/util-body-length-node': 4.2.3 - '@smithy/util-defaults-mode-browser': 4.3.49 - '@smithy/util-defaults-mode-node': 4.2.54 - '@smithy/util-endpoints': 3.4.2 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-retry': 4.3.6 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - '@aws-sdk/nested-clients@3.997.6': dependencies: '@aws-crypto/sha256-browser': 5.2.0 @@ -7004,15 +6497,6 @@ snapshots: '@smithy/types': 4.14.1 tslib: 2.8.1 - '@aws-sdk/signature-v4-multi-region@3.996.24': - dependencies: - '@aws-sdk/middleware-sdk-s3': 3.972.36 - '@aws-sdk/types': 3.973.8 - '@smithy/protocol-http': 5.3.14 - '@smithy/signature-v4': 5.3.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - '@aws-sdk/signature-v4-multi-region@3.996.25': dependencies: '@aws-sdk/middleware-sdk-s3': 3.972.37 @@ -7022,18 +6506,6 @@ snapshots: '@smithy/types': 4.14.1 tslib: 2.8.1 - '@aws-sdk/token-providers@3.1039.0': - dependencies: - '@aws-sdk/core': 3.974.7 - '@aws-sdk/nested-clients': 3.997.5 - '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - '@aws-sdk/token-providers@3.1041.0': dependencies: '@aws-sdk/core': 3.974.8 @@ -7074,15 +6546,6 @@ snapshots: bowser: 2.14.1 tslib: 2.8.1 - '@aws-sdk/util-user-agent-node@3.973.23': - dependencies: - '@aws-sdk/middleware-user-agent': 3.972.37 - '@aws-sdk/types': 3.973.8 - '@smithy/node-config-provider': 4.3.14 - '@smithy/types': 4.14.1 - '@smithy/util-config-provider': 4.2.2 - tslib: 2.8.1 - '@aws-sdk/util-user-agent-node@3.973.24': dependencies: '@aws-sdk/middleware-user-agent': 3.972.38 @@ -7142,11 +6605,6 @@ snapshots: '@azure/core-client': 1.10.1 '@azure/core-rest-pipeline': 1.23.0 - '@azure/core-http-compat@2.4.0(@azure/core-rest-pipeline@1.23.0)': - dependencies: - '@azure/abort-controller': 2.1.2 - '@azure/core-rest-pipeline': 1.23.0 - '@azure/core-lro@2.7.2': dependencies: '@azure/abort-controller': 2.1.2 @@ -7218,24 +6676,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@azure/keyvault-keys@4.10.0': - dependencies: - '@azure-rest/core-client': 2.6.0 - '@azure/abort-controller': 2.1.2 - '@azure/core-auth': 1.10.1 - '@azure/core-http-compat': 2.4.0(@azure/core-rest-pipeline@1.23.0) - '@azure/core-lro': 2.7.2 - '@azure/core-paging': 1.6.2 - '@azure/core-rest-pipeline': 1.23.0 - '@azure/core-tracing': 1.3.1 - '@azure/core-util': 1.13.1 - '@azure/keyvault-common': 2.1.0 - '@azure/logger': 1.3.0 - tslib: 2.8.1 - transitivePeerDependencies: - - '@azure/core-client' - - supports-color - '@azure/keyvault-keys@4.10.0(@azure/core-client@1.10.1)': dependencies: '@azure-rest/core-client': 2.6.0 @@ -7666,164 +7106,6 @@ snapshots: '@js-joda/core@5.7.0': {} - '@ktx/connector-bigquery@file:packages/connector-bigquery(js-yaml@4.1.1)(ws@8.20.1)': - dependencies: - '@google-cloud/bigquery': 8.3.1 - '@ktx/context': file:packages/context(js-yaml@4.1.1)(ws@8.20.1) - transitivePeerDependencies: - - '@cfworker/json-schema' - - js-yaml - - pg-native - - supports-color - - ws - - '@ktx/connector-clickhouse@file:packages/connector-clickhouse(js-yaml@4.1.1)(ws@8.20.1)': - dependencies: - '@clickhouse/client': 1.18.4 - '@ktx/context': file:packages/context(js-yaml@4.1.1)(ws@8.20.1) - transitivePeerDependencies: - - '@cfworker/json-schema' - - js-yaml - - pg-native - - supports-color - - ws - - '@ktx/connector-mysql@file:packages/connector-mysql(@types/node@24.12.2)(js-yaml@4.1.1)(ws@8.20.1)': - dependencies: - '@ktx/context': file:packages/context(js-yaml@4.1.1)(ws@8.20.1) - mysql2: 3.22.3(@types/node@24.12.2) - transitivePeerDependencies: - - '@cfworker/json-schema' - - '@types/node' - - js-yaml - - pg-native - - supports-color - - ws - - '@ktx/connector-postgres@file:packages/connector-postgres(js-yaml@4.1.1)(ws@8.20.1)': - dependencies: - '@ktx/context': file:packages/context(js-yaml@4.1.1)(ws@8.20.1) - pg: 8.20.0 - transitivePeerDependencies: - - '@cfworker/json-schema' - - js-yaml - - pg-native - - supports-color - - ws - - '@ktx/connector-snowflake@file:packages/connector-snowflake(asn1.js@5.4.1)(js-yaml@4.1.1)(ws@8.20.1)': - dependencies: - '@ktx/context': file:packages/context(js-yaml@4.1.1)(ws@8.20.1) - snowflake-sdk: 2.4.1(asn1.js@5.4.1) - transitivePeerDependencies: - - '@cfworker/json-schema' - - asn1.js - - aws-crt - - debug - - js-yaml - - pg-native - - supports-color - - ws - - '@ktx/connector-sqlite@file:packages/connector-sqlite(js-yaml@4.1.1)(ws@8.20.1)': - dependencies: - '@ktx/context': file:packages/context(js-yaml@4.1.1)(ws@8.20.1) - better-sqlite3: 12.10.0 - transitivePeerDependencies: - - '@cfworker/json-schema' - - js-yaml - - pg-native - - supports-color - - ws - - '@ktx/connector-sqlserver@file:packages/connector-sqlserver(js-yaml@4.1.1)(ws@8.20.1)': - dependencies: - '@ktx/context': file:packages/context(js-yaml@4.1.1)(ws@8.20.1) - mssql: 12.5.2 - transitivePeerDependencies: - - '@azure/core-client' - - '@cfworker/json-schema' - - js-yaml - - pg-native - - supports-color - - ws - - '@ktx/context@file:packages/context(js-yaml@4.1.1)': - dependencies: - '@anthropic-ai/claude-agent-sdk': 0.3.142(zod@4.4.3) - '@ktx/llm': file:packages/llm(zod@4.4.3) - '@looker/sdk': 26.8.0 - '@looker/sdk-node': 26.8.0 - '@looker/sdk-rtl': 21.6.5 - '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) - '@notionhq/client': 5.21.0 - ai: 6.0.180(zod@4.4.3) - better-sqlite3: 12.10.0 - handlebars: 4.7.9 - lookml-parser: 7.1.0(js-yaml@4.1.1) - minimatch: 10.2.5 - p-limit: 7.3.0 - pg: 8.20.0 - simple-git: 3.36.0 - yaml: 2.9.0 - zod: 4.4.3 - transitivePeerDependencies: - - '@cfworker/json-schema' - - js-yaml - - pg-native - - supports-color - - ws - - '@ktx/context@file:packages/context(js-yaml@4.1.1)(ws@8.20.1)': - dependencies: - '@anthropic-ai/claude-agent-sdk': 0.3.142(zod@4.4.3) - '@ktx/llm': file:packages/llm(ws@8.20.1)(zod@4.4.3) - '@looker/sdk': 26.8.0 - '@looker/sdk-node': 26.8.0 - '@looker/sdk-rtl': 21.6.5 - '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) - '@notionhq/client': 5.21.0 - ai: 6.0.180(zod@4.4.3) - better-sqlite3: 12.10.0 - handlebars: 4.7.9 - lookml-parser: 7.1.0(js-yaml@4.1.1) - minimatch: 10.2.5 - p-limit: 7.3.0 - pg: 8.20.0 - simple-git: 3.36.0 - yaml: 2.9.0 - zod: 4.4.3 - transitivePeerDependencies: - - '@cfworker/json-schema' - - js-yaml - - pg-native - - supports-color - - ws - - '@ktx/llm@file:packages/llm(ws@8.20.1)(zod@4.4.3)': - dependencies: - '@ai-sdk/anthropic': 3.0.77(zod@4.4.3) - '@ai-sdk/devtools': 0.0.17 - '@ai-sdk/google-vertex': 4.0.128(zod@4.4.3) - ai: 6.0.180(zod@4.4.3) - openai: 6.37.0(ws@8.20.1)(zod@4.4.3) - transitivePeerDependencies: - - supports-color - - ws - - zod - - '@ktx/llm@file:packages/llm(zod@4.4.3)': - dependencies: - '@ai-sdk/anthropic': 3.0.77(zod@4.4.3) - '@ai-sdk/devtools': 0.0.17 - '@ai-sdk/google-vertex': 4.0.128(zod@4.4.3) - ai: 6.0.180(zod@4.4.3) - openai: 6.37.0(zod@4.4.3) - transitivePeerDependencies: - - supports-color - - ws - - zod - '@kwsites/file-exists@1.1.1': dependencies: debug: 4.4.3 @@ -11538,17 +10820,6 @@ snapshots: ms@2.1.3: {} - mssql@12.5.2: - dependencies: - '@tediousjs/connection-string': 1.1.0 - commander: 11.1.0 - debug: 4.4.3 - tarn: 3.0.2 - tedious: 19.2.1 - transitivePeerDependencies: - - '@azure/core-client' - - supports-color - mssql@12.5.2(@azure/core-client@1.10.1): dependencies: '@tediousjs/connection-string': 1.1.0 @@ -11725,10 +10996,6 @@ snapshots: ws: 8.20.1 zod: 4.4.3 - openai@6.37.0(zod@4.4.3): - optionalDependencies: - zod: 4.4.3 - oxc-parser@0.128.0: dependencies: '@oxc-project/types': 0.128.0 @@ -12468,7 +11735,7 @@ snapshots: '@aws-crypto/sha256-js': 5.2.0 '@aws-sdk/client-s3': 3.1045.0 '@aws-sdk/client-sts': 3.1045.0 - '@aws-sdk/credential-provider-node': 3.972.38 + '@aws-sdk/credential-provider-node': 3.972.39 '@aws-sdk/ec2-metadata-service': 3.1045.0 '@azure/identity': 4.13.1 '@azure/storage-blob': 12.26.0 @@ -12670,22 +11937,6 @@ snapshots: tarn@3.0.2: {} - tedious@19.2.1: - dependencies: - '@azure/core-auth': 1.10.1 - '@azure/identity': 4.13.1 - '@azure/keyvault-keys': 4.10.0 - '@js-joda/core': 5.7.0 - '@types/node': 24.12.2 - bl: 6.1.6 - iconv-lite: 0.7.2 - js-md4: 0.3.2 - native-duplexpair: 1.0.0 - sprintf-js: 1.1.3 - transitivePeerDependencies: - - '@azure/core-client' - - supports-color - tedious@19.2.1(@azure/core-client@1.10.1): dependencies: '@azure/core-auth': 1.10.1 diff --git a/python/ktx-daemon/README.md b/python/ktx-daemon/README.md index bb429c5e..95a3ffbf 100644 --- a/python/ktx-daemon/README.md +++ b/python/ktx-daemon/README.md @@ -4,7 +4,7 @@ It supports portable compute in two modes: -- One-shot commands, used by default by `@ktx/context`. +- One-shot commands, used by default by the `@kaelio/ktx` CLI. - An explicit HTTP server for long-running local MCP sessions. ## One-shot semantic query diff --git a/python/ktx-daemon/pyproject.toml b/python/ktx-daemon/pyproject.toml index 4f21f9de..8cb78f36 100644 --- a/python/ktx-daemon/pyproject.toml +++ b/python/ktx-daemon/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ktx-daemon" -version = "0.0.0+private" +version = "0.4.1" description = "Portable compute package for KTX semantic-layer operations" readme = "README.md" requires-python = ">=3.13" diff --git a/python/ktx-sl/pyproject.toml b/python/ktx-sl/pyproject.toml index 51e97a83..cfa7a014 100644 --- a/python/ktx-sl/pyproject.toml +++ b/python/ktx-sl/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ktx-sl" -version = "0.0.0+private" +version = "0.4.1" description = "Agent-first semantic layer engine with aggregate locality" readme = "README.md" requires-python = ">=3.13" diff --git a/release-policy.json b/release-policy.json index 813e09d3..534f5ad9 100644 --- a/release-policy.json +++ b/release-policy.json @@ -1,6 +1,5 @@ { "schemaVersion": 1, - "publicNpmPackageVersion": "0.4.1", "releaseMode": "npm-public-release-ready", "npm": { "publish": true, diff --git a/scripts/anti-fixture-conditional.test.mjs b/scripts/anti-fixture-conditional.test.mjs index 1ab4da1e..b4a3241c 100644 --- a/scripts/anti-fixture-conditional.test.mjs +++ b/scripts/anti-fixture-conditional.test.mjs @@ -5,21 +5,21 @@ import { describe, it } from 'node:test'; const KTX_ROOT = new URL('../', import.meta.url); const RELATIONSHIP_RUNTIME_SOURCES = Object.freeze([ - 'packages/context/src/scan/relationship-benchmarks.ts', - 'packages/context/src/scan/relationship-budget.ts', - 'packages/context/src/scan/relationship-candidates.ts', - 'packages/context/src/scan/relationship-composite-candidates.ts', - 'packages/context/src/scan/relationship-graph-resolver.ts', - 'packages/context/src/scan/relationship-locality.ts', - 'packages/context/src/scan/relationship-name-similarity.ts', - 'packages/context/src/scan/relationship-discovery.ts', - 'packages/context/src/scan/relationship-profiling.ts', - 'packages/context/src/scan/relationship-scoring.ts', - 'packages/context/src/scan/relationship-validation.ts', + 'packages/cli/src/context/scan/relationship-benchmarks.ts', + 'packages/cli/src/context/scan/relationship-budget.ts', + 'packages/cli/src/context/scan/relationship-candidates.ts', + 'packages/cli/src/context/scan/relationship-composite-candidates.ts', + 'packages/cli/src/context/scan/relationship-graph-resolver.ts', + 'packages/cli/src/context/scan/relationship-locality.ts', + 'packages/cli/src/context/scan/relationship-name-similarity.ts', + 'packages/cli/src/context/scan/relationship-discovery.ts', + 'packages/cli/src/context/scan/relationship-profiling.ts', + 'packages/cli/src/context/scan/relationship-scoring.ts', + 'packages/cli/src/context/scan/relationship-validation.ts', ]); async function checkedInFixtureIds() { - const fixtureRoot = new URL('packages/context/test/fixtures/relationship-benchmarks/', KTX_ROOT); + const fixtureRoot = new URL('packages/cli/src/test/fixtures/relationship-benchmarks/', KTX_ROOT); const entries = await readdir(fixtureRoot, { withFileTypes: true }); return entries .filter((entry) => entry.isDirectory()) diff --git a/scripts/build-adventureworks-oltp-fixture.mjs b/scripts/build-adventureworks-oltp-fixture.mjs index 2a57fe85..07ef3bec 100644 --- a/scripts/build-adventureworks-oltp-fixture.mjs +++ b/scripts/build-adventureworks-oltp-fixture.mjs @@ -7,7 +7,7 @@ import { expectedLinksFromSnapshot, normalizeSqliteType } from './build-benchmar const scriptDir = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.resolve(scriptDir, '..'); -const require = createRequire(new URL('../packages/context/package.json', import.meta.url)); +const require = createRequire(new URL('../packages/cli/package.json', import.meta.url)); const Database = require('better-sqlite3'); const { stringify: yamlStringify } = require('yaml'); @@ -224,7 +224,7 @@ async function main() { } const source = JSON.parse(readFileSync(path.join(scriptDir, 'adventureworks-oltp-source.json'), 'utf8')); - const { KtxSqlServerScanConnector } = await import('../packages/connector-sqlserver/dist/index.js'); + const { KtxSqlServerScanConnector } = await import('../packages/cli/dist/connectors/sqlserver/index.js'); const connector = new KtxSqlServerScanConnector({ connectionId: fixtureId, connection: { diff --git a/scripts/build-benchmark-snapshot.mjs b/scripts/build-benchmark-snapshot.mjs index 718c997a..d2dd7acd 100644 --- a/scripts/build-benchmark-snapshot.mjs +++ b/scripts/build-benchmark-snapshot.mjs @@ -5,7 +5,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; const scriptDir = path.dirname(fileURLToPath(import.meta.url)); -const require = createRequire(new URL('../packages/context/package.json', import.meta.url)); +const require = createRequire(new URL('../packages/cli/package.json', import.meta.url)); const Database = require('better-sqlite3'); const { stringify: yamlStringify } = require('yaml'); diff --git a/scripts/build-benchmark-snapshot.test.mjs b/scripts/build-benchmark-snapshot.test.mjs index 26ac6419..6e3f5189 100644 --- a/scripts/build-benchmark-snapshot.test.mjs +++ b/scripts/build-benchmark-snapshot.test.mjs @@ -4,7 +4,7 @@ import { createRequire } from 'node:module'; import { describe, it } from 'node:test'; import { buildBenchmarkSnapshot } from './build-benchmark-snapshot.mjs'; -const require = createRequire(new URL('../packages/context/package.json', import.meta.url)); +const require = createRequire(new URL('../packages/cli/package.json', import.meta.url)); const Database = require('better-sqlite3'); describe('buildBenchmarkSnapshot', () => { @@ -252,12 +252,12 @@ describe('buildBenchmarkSnapshot', () => { ]); }); - it('exposes relationship benchmarks as an explicit context package script', async () => { - const packageJson = JSON.parse(await readFile(new URL('../packages/context/package.json', import.meta.url), 'utf8')); + it('exposes relationship benchmarks as an explicit CLI package script', async () => { + const packageJson = JSON.parse(await readFile(new URL('../packages/cli/package.json', import.meta.url), 'utf8')); assert.equal( packageJson.scripts['relationships:benchmarks:test'], - 'KTX_RUN_RELATIONSHIP_BENCHMARKS=1 vitest run src/scan/relationship-benchmarks.test.ts', + 'KTX_RUN_RELATIONSHIP_BENCHMARKS=1 vitest run src/context/scan/relationship-benchmarks.test.ts', ); }); }); diff --git a/scripts/build-evidence-fusion-adversarial-fixtures.mjs b/scripts/build-evidence-fusion-adversarial-fixtures.mjs index 282a6477..46d9b6e2 100644 --- a/scripts/build-evidence-fusion-adversarial-fixtures.mjs +++ b/scripts/build-evidence-fusion-adversarial-fixtures.mjs @@ -9,7 +9,7 @@ import { buildBenchmarkSnapshot, writeFixtureFiles } from './build-benchmark-sna const scriptDir = path.dirname(fileURLToPath(import.meta.url)); const ktxRoot = path.resolve(scriptDir, '..'); const fixtureRoot = path.join(ktxRoot, 'packages', 'context', 'test', 'fixtures', 'relationship-benchmarks'); -const require = createRequire(new URL('../packages/context/package.json', import.meta.url)); +const require = createRequire(new URL('../packages/cli/package.json', import.meta.url)); const Database = require('better-sqlite3'); const { stringify: yamlStringify } = require('yaml'); diff --git a/scripts/build-public-npm-package.mjs b/scripts/build-public-npm-package.mjs deleted file mode 100644 index 63551d38..00000000 --- a/scripts/build-public-npm-package.mjs +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env node - -import { execFile } from 'node:child_process'; -import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; -import { dirname, join, resolve } from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; -import { promisify } from 'node:util'; - -import { - PUBLIC_NPM_PACKAGE_NAME, - publicNpmPackageVersion, -} from './public-npm-release-metadata.mjs'; - -const execFileAsync = promisify(execFile); - -export const PUBLIC_NPM_PACKAGE_VERSION = publicNpmPackageVersion(); -export { PUBLIC_NPM_PACKAGE_NAME }; - -export function publicNpmPackageTarballName(version = PUBLIC_NPM_PACKAGE_VERSION) { - return `kaelio-ktx-${version}.tgz`; -} - -export const PUBLIC_BUNDLED_WORKSPACE_PACKAGES = [ - '@ktx/llm', - '@ktx/context', - '@ktx/connector-bigquery', - '@ktx/connector-clickhouse', - '@ktx/connector-mysql', - '@ktx/connector-postgres', - '@ktx/connector-snowflake', - '@ktx/connector-sqlite', - '@ktx/connector-sqlserver', -]; - -export const PUBLIC_BUNDLED_WORKSPACE_PACKAGE_ROOTS = { - '@ktx/llm': 'packages/llm', - '@ktx/context': 'packages/context', - '@ktx/connector-bigquery': 'packages/connector-bigquery', - '@ktx/connector-clickhouse': 'packages/connector-clickhouse', - '@ktx/connector-mysql': 'packages/connector-mysql', - '@ktx/connector-postgres': 'packages/connector-postgres', - '@ktx/connector-snowflake': 'packages/connector-snowflake', - '@ktx/connector-sqlite': 'packages/connector-sqlite', - '@ktx/connector-sqlserver': 'packages/connector-sqlserver', -}; - -function scriptRootDir() { - return resolve(dirname(fileURLToPath(import.meta.url)), '..'); -} - -export function publicNpmPackageLayout(rootDir = scriptRootDir(), version = PUBLIC_NPM_PACKAGE_VERSION) { - return { - rootDir, - packageVersion: version, - cliPackageRoot: join(rootDir, 'packages', 'cli'), - packRoot: join(rootDir, 'dist', 'public-npm-package'), - npmDir: join(rootDir, 'dist', 'artifacts', 'npm'), - tarballPath: join(rootDir, 'dist', 'artifacts', 'npm', publicNpmPackageTarballName(version)), - }; -} - -async function readJson(path) { - return JSON.parse(await readFile(path, 'utf8')); -} - -async function writeJson(path, value) { - await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); -} - -function sortedObject(entries) { - return Object.fromEntries([...entries].sort(([left], [right]) => left.localeCompare(right))); -} - -function isWorkspacePackageName(name) { - return name.startsWith('@ktx/'); -} - -function parseCaretVersion(value) { - const match = /^\^(\d+)\.(\d+)\.(\d+)$/.exec(value); - if (!match) { - return null; - } - return { - major: Number(match[1]), - minor: Number(match[2]), - patch: Number(match[3]), - }; -} - -function compareParsedVersions(left, right) { - return left.major - right.major || left.minor - right.minor || left.patch - right.patch; -} - -function mergeDependencyVersion(name, previous, next) { - if (previous === next) { - return previous; - } - - const previousCaret = parseCaretVersion(previous); - const nextCaret = parseCaretVersion(next); - if (previousCaret && nextCaret && previousCaret.major === nextCaret.major) { - return compareParsedVersions(previousCaret, nextCaret) >= 0 ? previous : next; - } - - throw new Error(`Incompatible dependency versions for ${name}: ${previous} and ${next}`); -} - -export function collectPublicDependencies(packageJsons) { - const dependencies = new Map(); - - for (const packageJson of packageJsons) { - for (const [name, version] of Object.entries(packageJson.dependencies ?? {})) { - if (isWorkspacePackageName(name)) { - continue; - } - const previous = dependencies.get(name); - dependencies.set(name, previous ? mergeDependencyVersion(name, previous, version) : version); - } - } - - return sortedObject(dependencies); -} - -export function publicNpmPackageJson(cliPackageJson, dependencies, version = PUBLIC_NPM_PACKAGE_VERSION) { - return { - name: PUBLIC_NPM_PACKAGE_NAME, - version, - description: 'Standalone KTX context layer for database agents', - private: false, - type: 'module', - engines: cliPackageJson.engines ?? { node: '>=22.0.0' }, - bin: { ktx: './dist/bin.js' }, - main: cliPackageJson.main ?? 'dist/index.js', - types: cliPackageJson.types ?? 'dist/index.d.ts', - exports: cliPackageJson.exports ?? { - '.': { - types: './dist/index.d.ts', - import: './dist/index.js', - default: './dist/index.js', - }, - './package.json': './package.json', - }, - files: ['dist', 'assets'], - dependencies, - bundledDependencies: PUBLIC_BUNDLED_WORKSPACE_PACKAGES, - license: cliPackageJson.license ?? 'Apache-2.0', - repository: { - type: 'git', - url: 'https://github.com/Kaelio/ktx', - }, - bugs: { - url: 'https://github.com/Kaelio/ktx/issues', - }, - homepage: 'https://github.com/Kaelio/ktx#readme', - }; -} - -function bundledWorkspacePackageJson(packageJson) { - return { - name: packageJson.name, - version: packageJson.version ?? PUBLIC_NPM_PACKAGE_VERSION, - private: true, - type: packageJson.type ?? 'module', - main: packageJson.main, - types: packageJson.types, - exports: packageJson.exports, - files: packageJson.files, - license: packageJson.license ?? 'Apache-2.0', - }; -} - -async function copyPackageFileEntries(sourceRoot, targetRoot, packageJson) { - for (const entry of packageJson.files ?? ['dist']) { - await cp(join(sourceRoot, entry), join(targetRoot, entry), { - recursive: true, - force: true, - }); - } -} - -async function copyCliPackage(layout, cliPackageJson, dependencies) { - await copyPackageFileEntries(layout.cliPackageRoot, layout.packRoot, cliPackageJson); - await writeJson( - join(layout.packRoot, 'package.json'), - publicNpmPackageJson(cliPackageJson, dependencies, layout.packageVersion), - ); -} - -async function copyBundledWorkspacePackage(rootDir, packageName, packageJson) { - const packageRoot = PUBLIC_BUNDLED_WORKSPACE_PACKAGE_ROOTS[packageName]; - if (!packageRoot) { - throw new Error(`Missing bundled workspace package root for ${packageName}`); - } - - const sourceRoot = join(rootDir, packageRoot); - const targetRoot = join(rootDir, 'dist', 'public-npm-package', 'node_modules', ...packageName.split('/')); - await mkdir(targetRoot, { recursive: true }); - await copyPackageFileEntries(sourceRoot, targetRoot, packageJson); - await writeJson(join(targetRoot, 'package.json'), bundledWorkspacePackageJson(packageJson)); -} - -export async function createPublicNpmPackageTree(layout = publicNpmPackageLayout()) { - const cliPackageJson = await readJson(join(layout.cliPackageRoot, 'package.json')); - const bundledPackageJsons = await Promise.all( - PUBLIC_BUNDLED_WORKSPACE_PACKAGES.map(async (packageName) => { - const packageRoot = PUBLIC_BUNDLED_WORKSPACE_PACKAGE_ROOTS[packageName]; - const packageJson = await readJson(join(layout.rootDir, packageRoot, 'package.json')); - if (packageJson.name !== packageName) { - throw new Error(`Unexpected package name in ${packageRoot}/package.json: ${packageJson.name}`); - } - return packageJson; - }), - ); - const dependencies = collectPublicDependencies([cliPackageJson, ...bundledPackageJsons]); - - await rm(layout.packRoot, { recursive: true, force: true }); - await mkdir(layout.packRoot, { recursive: true }); - await mkdir(layout.npmDir, { recursive: true }); - await copyCliPackage(layout, cliPackageJson, dependencies); - - for (const packageJson of bundledPackageJsons) { - await copyBundledWorkspacePackage(layout.rootDir, packageJson.name, packageJson); - } - - return { - layout, - packageJson: publicNpmPackageJson(cliPackageJson, dependencies, layout.packageVersion), - bundledPackages: PUBLIC_BUNDLED_WORKSPACE_PACKAGES, - }; -} - -export function publicNpmPackCommand(layout = publicNpmPackageLayout()) { - return { - command: 'pnpm', - args: ['--config.node-linker=hoisted', 'pack', '--out', layout.tarballPath], - cwd: layout.packRoot, - }; -} - -export async function buildPublicNpmPackage(layout = publicNpmPackageLayout()) { - await createPublicNpmPackageTree(layout); - const pack = publicNpmPackCommand(layout); - await execFileAsync(pack.command, pack.args, { - cwd: pack.cwd, - encoding: 'utf8', - maxBuffer: 10 * 1024 * 1024, - }); - return layout.tarballPath; -} - -async function main() { - const tarball = await buildPublicNpmPackage(); - process.stdout.write(`Built ${PUBLIC_NPM_PACKAGE_NAME} package: ${tarball}\n`); -} - -if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { - try { - await main(); - } catch (error) { - process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); - process.exitCode = 1; - } -} diff --git a/scripts/build-public-npm-package.test.mjs b/scripts/build-public-npm-package.test.mjs deleted file mode 100644 index b69d7437..00000000 --- a/scripts/build-public-npm-package.test.mjs +++ /dev/null @@ -1,286 +0,0 @@ -import assert from 'node:assert/strict'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, it } from 'node:test'; - -import { - PUBLIC_BUNDLED_WORKSPACE_PACKAGES, - PUBLIC_NPM_PACKAGE_NAME, - PUBLIC_NPM_PACKAGE_VERSION, - collectPublicDependencies, - createPublicNpmPackageTree, - publicNpmPackageJson, - publicNpmPackageLayout, - publicNpmPackageTarballName, - publicNpmPackCommand, -} from './build-public-npm-package.mjs'; - -async function writeJson(path, value) { - await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); -} - -async function writePackage(root, packageRoot, packageJson, files = {}) { - const absoluteRoot = join(root, packageRoot); - await mkdir(absoluteRoot, { recursive: true }); - await writeJson(join(absoluteRoot, 'package.json'), packageJson); - - for (const [relativePath, contents] of Object.entries(files)) { - const target = join(absoluteRoot, relativePath); - await mkdir(join(target, '..'), { recursive: true }); - await writeFile(target, contents); - } -} - -async function writeWorkspaceFixture(root) { - await writePackage( - root, - 'packages/cli', - { - name: '@ktx/cli', - version: '0.0.0-private', - description: 'CLI wrapper for KTX', - type: 'module', - engines: { node: '>=22.0.0' }, - bin: { ktx: './dist/bin.js' }, - main: 'dist/index.js', - types: 'dist/index.d.ts', - exports: { - '.': { - types: './dist/index.d.ts', - import: './dist/index.js', - default: './dist/index.js', - }, - './package.json': './package.json', - }, - files: ['dist', 'assets'], - dependencies: { - '@clack/prompts': '1.3.0', - '@ktx/context': 'workspace:*', - commander: '14.0.3', - }, - license: 'Apache-2.0', - repository: { - type: 'git', - url: 'git+https://github.com/kaelio/ktx.git', - directory: 'packages/cli', - }, - }, - { - 'dist/bin.js': '#!/usr/bin/env node\n', - 'dist/index.js': 'export const cli = true;\n', - 'dist/index.d.ts': 'export declare const cli: true;\n', - 'assets/python/manifest.json': '{"schemaVersion":1}\n', - }, - ); - - await writePackage( - root, - 'packages/context', - { - name: '@ktx/context', - version: '0.0.0-private', - type: 'module', - main: 'dist/index.js', - exports: { '.': './dist/index.js' }, - files: ['dist', 'prompts', 'skills'], - dependencies: { - '@ktx/llm': 'workspace:*', - yaml: '^2.8.2', - }, - }, - { - 'dist/index.js': 'export const context = true;\n', - 'prompts/system.md': 'prompt\n', - 'skills/sl/SKILL.md': 'skill\n', - }, - ); - - await writePackage( - root, - 'packages/llm', - { - name: '@ktx/llm', - version: '0.0.0-private', - type: 'module', - main: 'dist/index.js', - exports: { '.': './dist/index.js' }, - files: ['dist'], - dependencies: { - ai: '^6.0.168', - }, - }, - { - 'dist/index.js': 'export const llm = true;\n', - }, - ); - - for (const packageName of PUBLIC_BUNDLED_WORKSPACE_PACKAGES.filter((name) => name.startsWith('@ktx/connector-'))) { - const directory = packageName.replace('@ktx/', ''); - await writePackage( - root, - `packages/${directory}`, - { - name: packageName, - version: '0.0.0-private', - type: 'module', - main: 'dist/index.js', - exports: { '.': './dist/index.js' }, - files: ['dist'], - dependencies: { - '@ktx/context': 'workspace:*', - }, - }, - { - 'dist/index.js': `export const name = ${JSON.stringify(packageName)};\n`, - }, - ); - } -} - -describe('publicNpmPackageLayout', () => { - it('uses the public npm release version for the tarball name', () => { - const layout = publicNpmPackageLayout('/repo/ktx'); - - assert.match(PUBLIC_NPM_PACKAGE_VERSION, /^\d+\.\d+\.\d+/); - assert.equal(publicNpmPackageTarballName(), `kaelio-ktx-${PUBLIC_NPM_PACKAGE_VERSION}.tgz`); - assert.equal( - layout.tarballPath, - `/repo/ktx/dist/artifacts/npm/kaelio-ktx-${PUBLIC_NPM_PACKAGE_VERSION}.tgz`, - ); - }); -}); - -describe('collectPublicDependencies', () => { - it('unions external runtime dependencies and omits workspace packages', () => { - assert.deepEqual( - collectPublicDependencies([ - { - name: '@ktx/cli', - dependencies: { - '@ktx/context': 'workspace:*', - commander: '14.0.3', - zod: '^4.4.3', - }, - }, - { - name: '@ktx/context', - dependencies: { - '@ktx/llm': 'workspace:*', - commander: '14.0.3', - yaml: '^2.8.2', - zod: '^4.1.13', - }, - }, - ]), - { - commander: '14.0.3', - yaml: '^2.8.2', - zod: '^4.4.3', - }, - ); - }); - - it('fails on incompatible external dependency ranges', () => { - assert.throws( - () => - collectPublicDependencies([ - { name: '@ktx/cli', dependencies: { zod: '^4.4.3' } }, - { name: '@ktx/context', dependencies: { zod: '^3.25.0' } }, - ]), - /Incompatible dependency versions for zod/, - ); - }); -}); - -describe('publicNpmPackageJson', () => { - it('does not bundle the removed PostHog connector package', () => { - assert.equal(PUBLIC_BUNDLED_WORKSPACE_PACKAGES.includes('@ktx/connector-posthog'), false); - }); - - it('describes the public @kaelio/ktx binary package', () => { - const packageJson = publicNpmPackageJson( - { - name: '@ktx/cli', - version: '0.0.0-private', - engines: { node: '>=22.0.0' }, - bin: { ktx: './dist/bin.js' }, - main: 'dist/index.js', - types: 'dist/index.d.ts', - exports: { '.': './dist/index.js', './package.json': './package.json' }, - license: 'Apache-2.0', - }, - { commander: '14.0.3' }, - ); - - assert.equal(packageJson.name, PUBLIC_NPM_PACKAGE_NAME); - assert.equal(packageJson.version, PUBLIC_NPM_PACKAGE_VERSION); - assert.equal(packageJson.private, false); - assert.deepEqual(packageJson.bin, { ktx: './dist/bin.js' }); - assert.deepEqual(packageJson.dependencies, { commander: '14.0.3' }); - assert.deepEqual(packageJson.bundledDependencies, PUBLIC_BUNDLED_WORKSPACE_PACKAGES); - assert.deepEqual(packageJson.files, ['dist', 'assets']); - assert.deepEqual(packageJson.repository, { - type: 'git', - url: 'https://github.com/Kaelio/ktx', - }); - assert.deepEqual(packageJson.bugs, { - url: 'https://github.com/Kaelio/ktx/issues', - }); - assert.equal(packageJson.homepage, 'https://github.com/Kaelio/ktx#readme'); - }); -}); - -describe('createPublicNpmPackageTree', () => { - it('copies CLI files, assets, and bundled internal workspace packages', async () => { - const root = await mkdtemp(join(tmpdir(), 'ktx-public-npm-test-')); - try { - await writeWorkspaceFixture(root); - const layout = publicNpmPackageLayout(root); - - const result = await createPublicNpmPackageTree(layout); - - assert.equal(result.packageJson.name, '@kaelio/ktx'); - assert.equal(result.packageJson.dependencies.commander, '14.0.3'); - assert.equal(result.packageJson.dependencies.yaml, '^2.8.2'); - assert.equal(result.packageJson.dependencies.ai, '^6.0.168'); - assert.equal( - await readFile(join(layout.packRoot, 'assets', 'python', 'manifest.json'), 'utf8'), - '{"schemaVersion":1}\n', - ); - assert.equal( - await readFile(join(layout.packRoot, 'node_modules', '@ktx', 'context', 'dist', 'index.js'), 'utf8'), - 'export const context = true;\n', - ); - assert.equal( - await readFile(join(layout.packRoot, 'node_modules', '@ktx', 'context', 'prompts', 'system.md'), 'utf8'), - 'prompt\n', - ); - - const bundledContextJson = JSON.parse( - await readFile(join(layout.packRoot, 'node_modules', '@ktx', 'context', 'package.json'), 'utf8'), - ); - assert.equal(bundledContextJson.private, true); - assert.equal(bundledContextJson.dependencies, undefined); - } finally { - await rm(root, { recursive: true, force: true }); - } - }); -}); - -describe('publicNpmPackCommand', () => { - it('packs the assembled public package with pnpm', () => { - const layout = publicNpmPackageLayout('/repo/ktx'); - - assert.deepEqual(publicNpmPackCommand(layout), { - command: 'pnpm', - args: [ - '--config.node-linker=hoisted', - 'pack', - '--out', - `/repo/ktx/dist/artifacts/npm/kaelio-ktx-${PUBLIC_NPM_PACKAGE_VERSION}.tgz`, - ], - cwd: '/repo/ktx/dist/public-npm-package', - }); - }); -}); diff --git a/scripts/build-python-runtime-wheel.test.mjs b/scripts/build-python-runtime-wheel.test.mjs index bf25af9a..67b85175 100644 --- a/scripts/build-python-runtime-wheel.test.mjs +++ b/scripts/build-python-runtime-wheel.test.mjs @@ -52,7 +52,10 @@ describe('runtimeWheelPyproject', () => { const pyproject = runtimeWheelPyproject(); assert.match(pyproject, /name = "kaelio-ktx"/); - assert.match(pyproject, new RegExp(`version = "${RUNTIME_WHEEL_PACKAGE_VERSION.replace(/\./g, '\\.')}"`)); + assert.match( + pyproject, + new RegExp(`version = "${RUNTIME_WHEEL_PACKAGE_VERSION.replace(/[.+]/g, (char) => `\\${char}`)}"`), + ); assert.match(pyproject, /ktx-daemon = "ktx_daemon\.__main__:main"/); assert.match(pyproject, /packages = \["semantic_layer", "ktx_daemon"\]/); assert.match(pyproject, /\[project\.optional-dependencies\]/); diff --git a/scripts/check-boundaries.mjs b/scripts/check-boundaries.mjs index 1a2adf4f..5766ac78 100644 --- a/scripts/check-boundaries.mjs +++ b/scripts/check-boundaries.mjs @@ -5,12 +5,12 @@ import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; const codeExtensions = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.py']); -const runtimeAssetPatterns = [/^packages\/[^/]+\/prompts\/.+\.md$/, /^packages\/[^/]+\/skills\/.+\.md$/]; +const runtimeAssetPatterns = [/^packages\/cli\/src\/prompts\/.+\.md$/, /^packages\/cli\/src\/skills\/.+\.md$/]; const identifierSkipPrefixes = ['docs/', 'docs-site/', 'examples/', 'python/ktx-sl/plans/', 'python/ktx-sl/openspec/']; const identifierAllowPatterns = [ /^packages\/cli\/src\/(?:index|managed-local-embeddings|managed-python-command|managed-python-daemon|managed-python-runtime|release-version|runtime)(?:\.test)?\.ts$/, /^python\/ktx-daemon\/src\/ktx_daemon\/__init__\.py$/, - /^scripts\/(?:build-public-npm-package|build-python-runtime-wheel|local-embeddings-runtime-smoke|package-artifacts|public-npm-release-metadata|published-package-smoke|release-readiness)(?:\.test)?\.mjs$/, + /^scripts\/(?:build-python-runtime-wheel|local-embeddings-runtime-smoke|package-artifacts|public-npm-release-metadata|published-package-smoke|release-readiness)(?:\.test)?\.mjs$/, /^scripts\/semantic-release-config\.cjs$/, ]; const forbiddenIdentifierTerms = ['kae' + 'lio', 'Kae' + 'lio', 'KAE' + 'LIO_']; @@ -85,7 +85,7 @@ function scansForAppImports(relativePath) { } function scansForLlmBoundaries(relativePath) { - return isCodeSource(relativePath) && relativePath.startsWith('packages/context/src/'); + return isCodeSource(relativePath) && relativePath.startsWith('packages/cli/src/context/'); } function isTestSource(relativePath) { @@ -133,7 +133,7 @@ export function scanFileContent(relativePath, content) { violations.push({ file: normalizedPath, kind: 'llm-boundary', - message: `Forbidden ${llmBoundaryPattern.label}; use @ktx/llm`, + message: `Forbidden ${llmBoundaryPattern.label}; use packages/cli/src/llm`, }); } } @@ -145,7 +145,7 @@ export function scanFileContent(relativePath, content) { violations.push({ file: normalizedPath, kind: 'llm-boundary', - message: `Forbidden ${llmBoundaryPattern.label}; use getModel(role) inside @ktx/context`, + message: `Forbidden ${llmBoundaryPattern.label}; use getModel(role) inside context modules`, }); } } diff --git a/scripts/check-boundaries.test.mjs b/scripts/check-boundaries.test.mjs index 9fb8999d..25cd0f85 100644 --- a/scripts/check-boundaries.test.mjs +++ b/scripts/check-boundaries.test.mjs @@ -17,8 +17,8 @@ describe('scanFileContent', () => { const pythonAppPath = `${['python', 'service'].join('-')}/app/api/endpoints/semantic_layer.py`; const violations = [ - ...scanFileContent('packages/context/src/index.ts', `import { orpc } from '${serverAlias}';`), - ...scanFileContent('packages/context/src/index.ts', `import "${pythonAppPath}";`), + ...scanFileContent('packages/cli/src/context/index.ts', `import { orpc } from '${serverAlias}';`), + ...scanFileContent('packages/cli/src/context/index.ts', `import "${pythonAppPath}";`), ]; assert.deepEqual( @@ -28,7 +28,7 @@ describe('scanFileContent', () => { }); it('rejects forbidden product identifiers in code source files', () => { - const violations = scanFileContent('packages/context/src/index.ts', `export const owner = '${lowerProductName()}';`); + const violations = scanFileContent('packages/cli/src/context/index.ts', `export const owner = '${lowerProductName()}';`); assert.equal(violations.length, 1); assert.equal(violations[0]?.kind, 'identifier'); @@ -36,24 +36,24 @@ describe('scanFileContent', () => { it('rejects forbidden product identifiers in shipped runtime prompt assets', () => { const violations = scanFileContent( - 'packages/context/prompts/memory_agent_bundle_ingest_work_unit.md', + 'packages/cli/src/prompts/memory_agent_bundle_ingest_work_unit.md', `Write output for ${productName()}.`, ); assert.equal(violations.length, 1); assert.equal(violations[0]?.kind, 'identifier'); - assert.equal(violations[0]?.file, 'packages/context/prompts/memory_agent_bundle_ingest_work_unit.md'); + assert.equal(violations[0]?.file, 'packages/cli/src/prompts/memory_agent_bundle_ingest_work_unit.md'); }); it('rejects forbidden product identifiers in shipped runtime skill assets', () => { const violations = scanFileContent( - 'packages/context/skills/metabase_ingest/SKILL.md', + 'packages/cli/src/skills/metabase_ingest/SKILL.md', `Use ${productName()} project conventions.`, ); assert.equal(violations.length, 1); assert.equal(violations[0]?.kind, 'identifier'); - assert.equal(violations[0]?.file, 'packages/context/skills/metabase_ingest/SKILL.md'); + assert.equal(violations[0]?.file, 'packages/cli/src/skills/metabase_ingest/SKILL.md'); }); it('allows product identifiers in docs, examples, and transition metadata', () => { @@ -69,7 +69,7 @@ describe('scanFileContent', () => { const name = lowerProductName(); assert.equal(scanFileContent('packages/cli/src/setup.test.ts', `project: ${name}-dev`).length, 0); - assert.equal(scanFileContent('packages/context/src/ingest/importer.test.ts', `email: system@${name}.dev`).length, 0); + assert.equal(scanFileContent('packages/cli/src/context/ingest/importer.test.ts', `email: system@${name}.dev`).length, 0); assert.equal(scanFileContent('python/ktx-daemon/tests/test_package.py', `${name}-ktx`).length, 0); }); @@ -87,23 +87,23 @@ describe('scanFileContent', () => { it('allows clean source files and clean runtime prompt assets', () => { assert.deepEqual( - scanFileContent('packages/context/src/index.ts', "export const packageName = '@ktx/context';"), + scanFileContent('packages/cli/src/context/index.ts', "export const packageName = 'ktx';"), [], ); assert.deepEqual( - scanFileContent('packages/context/prompts/memory_agent_bundle_ingest_work_unit.md', 'Write output for KTX.'), + scanFileContent('packages/cli/src/prompts/memory_agent_bundle_ingest_work_unit.md', 'Write output for KTX.'), [], ); }); - it('rejects context-owned LLM provider construction outside @ktx/llm', () => { + it('rejects context-owned LLM provider construction outside llm modules', () => { const violations = [ ...scanFileContent( - 'packages/context/src/agent/local-llm-provider.ts', + 'packages/cli/src/context/agent/local-llm-provider.ts', "import { createAnthropic } from '@ai-sdk/anthropic';", ), - ...scanFileContent('packages/context/src/scan/local-ai-gateway-enrichment.ts', "import { createGateway } from 'ai';"), - ...scanFileContent('packages/context/src/core/local-embedding-provider.ts', "import { embedMany } from 'ai';"), + ...scanFileContent('packages/cli/src/context/scan/local-ai-gateway-enrichment.ts', "import { createGateway } from 'ai';"), + ...scanFileContent('packages/cli/src/context/core/local-embedding-provider.ts', "import { embedMany } from 'ai';"), ]; assert.deepEqual( @@ -114,9 +114,9 @@ describe('scanFileContent', () => { it('rejects old KTX LLM port declarations in context', () => { const violations = [ - ...scanFileContent('packages/context/src/agent/agent-runner.service.ts', 'export interface LlmProviderPort {}'), - ...scanFileContent('packages/context/src/scan/types.ts', 'export interface KtxScanLlmPort {}'), - ...scanFileContent('packages/context/src/agent/gateway-llm-provider.ts', 'export function createGatewayLlmProvider() {}'), + ...scanFileContent('packages/cli/src/context/agent/agent-runner.service.ts', 'export interface LlmProviderPort {}'), + ...scanFileContent('packages/cli/src/context/scan/types.ts', 'export interface KtxScanLlmPort {}'), + ...scanFileContent('packages/cli/src/context/agent/gateway-llm-provider.ts', 'export function createGatewayLlmProvider() {}'), ]; assert.deepEqual( @@ -127,7 +127,7 @@ describe('scanFileContent', () => { it('rejects getModelByName calls in context production source', () => { const violations = scanFileContent( - 'packages/context/src/ingest/page-triage/page-triage.service.ts', + 'packages/cli/src/context/ingest/page-triage/page-triage.service.ts', "const model = this.deps.llmProvider.getModelByName('claude-sonnet-4-6');", ); @@ -135,14 +135,14 @@ describe('scanFileContent', () => { assert.equal(violations[0]?.kind, 'llm-boundary'); assert.equal( violations[0]?.message, - 'Forbidden context getModelByName call; use getModel(role) inside @ktx/context', + 'Forbidden context getModelByName call; use getModel(role) inside context modules', ); }); it('allows role-driven getModel calls, test calls, and provider shape declarations', () => { assert.deepEqual( scanFileContent( - 'packages/context/src/ingest/page-triage/page-triage.service.ts', + 'packages/cli/src/context/ingest/page-triage/page-triage.service.ts', "const model = this.deps.llmProvider.getModel('triage');", ), [], @@ -150,7 +150,7 @@ describe('scanFileContent', () => { assert.deepEqual( scanFileContent( - 'packages/context/src/ingest/page-triage/page-triage.service.test.ts', + 'packages/cli/src/context/ingest/page-triage/page-triage.service.test.ts', "const model = this.deps.llmProvider.getModelByName('test-model');", ), [], @@ -158,7 +158,7 @@ describe('scanFileContent', () => { assert.deepEqual( scanFileContent( - 'packages/context/src/scan/local-enrichment.ts', + 'packages/cli/src/context/scan/local-enrichment.ts', 'return { getModel() { return model; }, getModelByName() { return model; } };', ), [], diff --git a/scripts/examples-docs.test.mjs b/scripts/examples-docs.test.mjs index a1f89847..281777e6 100644 --- a/scripts/examples-docs.test.mjs +++ b/scripts/examples-docs.test.mjs @@ -144,18 +144,13 @@ describe('standalone example docs', () => { assert.doesNotMatch(orbitConfig, legacyPublicAdapter); }); - it('lists every workspace package in the contributor docs', async () => { + it('lists the consolidated workspace layout in the contributor docs', async () => { const contributing = await readText('docs-site/content/docs/community/contributing.mdx'); - assert.match(contributing, /cli\/\s+# CLI entry point/); - assert.match(contributing, /context\/\s+# Core context engine/); - assert.match(contributing, /llm\/\s+# LLM client abstraction/); - assert.match(contributing, /connector-bigquery\/\s+# BigQuery connector/); - assert.match(contributing, /connector-mysql\/\s+# MySQL connector/); - assert.match(contributing, /connector-postgres\/\s+# PostgreSQL connector/); - assert.match(contributing, /connector-snowflake\/\s+# Snowflake connector/); - assert.match(contributing, /connector-sqlite\/\s+# SQLite connector/); - assert.match(contributing, /connector-sqlserver\/\s+# SQL Server connector/); + assert.match(contributing, /cli\/\s+# CLI package and published npm package source/); + assert.match(contributing, /src\/context\/\s+# Core context engine/); + assert.match(contributing, /src\/llm\/\s+# LLM client abstraction/); + assert.match(contributing, /src\/connectors\/\s+# Database connectors/); assert.match(contributing, /ktx-sl\/\s+# Semantic layer/); assert.match(contributing, /ktx-daemon\/\s+# Daemon/); }); diff --git a/scripts/installed-live-database-smoke.mjs b/scripts/installed-live-database-smoke.mjs index 3675016e..a11e38d2 100644 --- a/scripts/installed-live-database-smoke.mjs +++ b/scripts/installed-live-database-smoke.mjs @@ -263,8 +263,7 @@ async function assertPathExists(path, label) { } async function prepareCleanInstall(layout, cleanInstallDir) { - await assertPathExists(layout.contextTarball, '@ktx/context tarball'); - await assertPathExists(layout.cliTarball, '@ktx/cli tarball'); + await assertPathExists(layout.cliTarball, 'CLI tarball'); await mkdir(cleanInstallDir, { recursive: true }); await writeFile(join(cleanInstallDir, 'package.json'), `${JSON.stringify(npmSmokePackageJson(layout), null, 2)}\n`); await writeFile(join(cleanInstallDir, 'pnpm-workspace.yaml'), npmSmokePnpmWorkspaceYaml()); diff --git a/scripts/link-dev-cli.test.mjs b/scripts/link-dev-cli.test.mjs index db4f9f4e..c3268541 100644 --- a/scripts/link-dev-cli.test.mjs +++ b/scripts/link-dev-cli.test.mjs @@ -13,7 +13,7 @@ test('linkDevCli writes a ktx-dev launcher by default', async () => { execText: async (command, args) => { assert.equal(command, 'ktx-dev'); assert.deepEqual(args, ['--version']); - return '@ktx/cli 0.0.0-private'; + return '@kaelio/ktx 0.0.0-private'; }, writeFile: async (path, content) => writes.push({ path, content }), chmod: async (path, mode) => chmods.push({ path, mode }), @@ -34,7 +34,7 @@ test('linkDevCli can explicitly write ktx when requested', async () => { binaryName: 'ktx', globalBin: '/pnpm/bin', binPath: '/workspace/ktx/packages/cli/dist/bin.js', - execText: async () => '@ktx/cli 0.0.0-private', + execText: async () => '@kaelio/ktx 0.0.0-private', writeFile: async (path, content) => writes.push({ path, content }), chmod: async () => undefined, access: async () => undefined, diff --git a/scripts/local-embeddings-runtime-smoke.mjs b/scripts/local-embeddings-runtime-smoke.mjs index c6d98178..539eaa60 100644 --- a/scripts/local-embeddings-runtime-smoke.mjs +++ b/scripts/local-embeddings-runtime-smoke.mjs @@ -8,7 +8,7 @@ import { promisify } from 'node:util'; import { PUBLIC_NPM_PACKAGE_NAME, PUBLIC_NPM_PACKAGE_VERSION, -} from './build-public-npm-package.mjs'; +} from './public-npm-release-metadata.mjs'; import { npmSmokePnpmWorkspaceYaml } from './package-artifacts.mjs'; const execFileAsync = promisify(execFile); diff --git a/scripts/local-embeddings-runtime-smoke.test.mjs b/scripts/local-embeddings-runtime-smoke.test.mjs index a445ef48..92df9dae 100644 --- a/scripts/local-embeddings-runtime-smoke.test.mjs +++ b/scripts/local-embeddings-runtime-smoke.test.mjs @@ -2,7 +2,7 @@ import assert from 'node:assert/strict'; import { readFile } from 'node:fs/promises'; import { describe, it } from 'node:test'; -import { PUBLIC_NPM_PACKAGE_VERSION } from './build-public-npm-package.mjs'; +import { PUBLIC_NPM_PACKAGE_VERSION } from './public-npm-release-metadata.mjs'; import { buildLocalEmbeddingsSmokeEnv, expectedPublicKtxVersionPattern, @@ -57,11 +57,11 @@ describe('publicKtxTarballName', () => { }); describe('expectedPublicKtxVersionPattern', () => { - it('matches the public package version and rejects the private workspace version', () => { + it('matches the public package version and rejects other versions', () => { const pattern = expectedPublicKtxVersionPattern(); assert.match(`@kaelio/ktx ${PUBLIC_NPM_PACKAGE_VERSION}\n`, pattern); - assert.doesNotMatch('@kaelio/ktx 0.0.0-private\n', pattern); + assert.doesNotMatch('@kaelio/ktx 9.9.9-other\n', pattern); }); }); diff --git a/scripts/normalize-lcov-paths.test.mjs b/scripts/normalize-lcov-paths.test.mjs index 44ab9fe6..2c4a52e7 100644 --- a/scripts/normalize-lcov-paths.test.mjs +++ b/scripts/normalize-lcov-paths.test.mjs @@ -8,11 +8,11 @@ describe('normalizeLcovContent', () => { const input = ['TN:', 'SF:src/index.ts', 'SF:src\\windows.ts', 'DA:1,1', 'end_of_record'].join('\n'); assert.equal( - normalizeLcovContent(input, 'packages/context'), + normalizeLcovContent(input, 'packages/cli'), [ 'TN:', - 'SF:packages/context/src/index.ts', - 'SF:packages/context/src/windows.ts', + 'SF:packages/cli/src/index.ts', + 'SF:packages/cli/src/windows.ts', 'DA:1,1', 'end_of_record', ].join('\n'), diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index 18a098bb..627850b4 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -15,8 +15,8 @@ import { import { PUBLIC_NPM_PACKAGE_NAME, publicNpmPackageTarballName, -} from './build-public-npm-package.mjs'; -import { publicNpmPackageVersion } from './public-npm-release-metadata.mjs'; + publicNpmPackageVersion, +} from './public-npm-release-metadata.mjs'; export { RUNTIME_WHEEL_DISTRIBUTION_NAME, @@ -24,19 +24,6 @@ export { RUNTIME_WHEEL_PACKAGE_VERSION, }; -export const INTERNAL_NPM_WORKSPACE_PACKAGES = [ - { name: '@ktx/context', packageRoot: 'packages/context' }, - { name: '@ktx/llm', packageRoot: 'packages/llm' }, - { name: '@ktx/connector-bigquery', packageRoot: 'packages/connector-bigquery' }, - { name: '@ktx/connector-clickhouse', packageRoot: 'packages/connector-clickhouse' }, - { name: '@ktx/connector-mysql', packageRoot: 'packages/connector-mysql' }, - { name: '@ktx/connector-postgres', packageRoot: 'packages/connector-postgres' }, - { name: '@ktx/connector-snowflake', packageRoot: 'packages/connector-snowflake' }, - { name: '@ktx/connector-sqlite', packageRoot: 'packages/connector-sqlite' }, - { name: '@ktx/connector-sqlserver', packageRoot: 'packages/connector-sqlserver' }, - { name: '@ktx/cli', packageRoot: 'packages/cli' }, -]; - export const NPM_ARTIFACT_PACKAGES = [{ name: PUBLIC_NPM_PACKAGE_NAME, packageRoot: 'packages/cli' }]; export const CLI_PYTHON_ASSET_MANIFEST = 'manifest.json'; @@ -81,33 +68,22 @@ export function packageArtifactLayout(rootDir = scriptRootDir(), version = publi } export function buildArtifactCommands(layout) { - // One recursive pnpm invocation; topology comes from workspace deps in - // each package.json, parallelism from --workspace-concurrency. - const npmBuildCommand = { - command: 'pnpm', - args: [ - '--filter', - './packages/*', - '--workspace-concurrency=10', - 'run', - 'build', - ], - cwd: layout.rootDir, - }; - const publicPackageCommand = { - command: process.execPath, - args: ['scripts/build-public-npm-package.mjs'], - cwd: layout.rootDir, - }; - return [ - npmBuildCommand, + { + command: 'pnpm', + args: ['--filter', PUBLIC_NPM_PACKAGE_NAME, 'run', 'build'], + cwd: layout.rootDir, + }, { command: process.execPath, args: ['scripts/build-python-runtime-wheel.mjs'], cwd: layout.rootDir, }, - publicPackageCommand, + { + command: 'pnpm', + args: ['pack', '--out', layout.cliTarball], + cwd: join(layout.rootDir, 'packages', 'cli'), + }, ]; } @@ -175,19 +151,17 @@ function releaseMetadataEntry({ ecosystem, packageName, packageRoot, packageVers async function readNpmPackageMetadata(rootDir, packageInfo, version) { const packageJson = await readJson(join(rootDir, packageInfo.packageRoot, 'package.json')); - const expectedSourceName = packageInfo.name === PUBLIC_NPM_PACKAGE_NAME ? '@ktx/cli' : packageInfo.name; - if (packageJson.name !== expectedSourceName) { + if (packageJson.name !== packageInfo.name) { throw new Error( - `Unexpected package name in ${packageInfo.packageRoot}/package.json: expected ${expectedSourceName}, got ${packageJson.name}`, + `Unexpected package name in ${packageInfo.packageRoot}/package.json: expected ${packageInfo.name}, got ${packageJson.name}`, ); } - const isPublicKtxPackage = packageInfo.name === PUBLIC_NPM_PACKAGE_NAME; return releaseMetadataEntry({ ecosystem: 'npm', packageName: packageInfo.name, packageRoot: packageInfo.packageRoot, - packageVersion: isPublicKtxPackage ? version : packageJson.version, - privatePackage: isPublicKtxPackage ? false : packageJson.private === true, + packageVersion: version, + privatePackage: false, }); } @@ -934,13 +908,13 @@ async function buildArtifacts(layout) { await mkdir(layout.npmDir, { recursive: true }); await mkdir(layout.pythonDir, { recursive: true }); - const [npmBuildCommand, wheelCommand, publicPackageCommand] = buildArtifactCommands(layout); + const [npmBuildCommand, wheelCommand, packCommand] = buildArtifactCommands(layout); await runCommand(npmBuildCommand.command, npmBuildCommand.args, { cwd: npmBuildCommand.cwd }); await runCommand(wheelCommand.command, wheelCommand.args, { cwd: wheelCommand.cwd }); const pythonArtifacts = await findPythonArtifacts(layout.pythonDir); await copyRuntimeWheelAssets(layout, pythonArtifacts); - await runCommand(publicPackageCommand.command, publicPackageCommand.args, { cwd: publicPackageCommand.cwd }); + await runCommand(packCommand.command, packCommand.args, { cwd: packCommand.cwd }); for (const packageInfo of NPM_ARTIFACT_PACKAGES) { await assertPathExists(layout.npmTarballs[packageInfo.name], `${packageInfo.name} tarball`); diff --git a/scripts/package-artifacts.test.mjs b/scripts/package-artifacts.test.mjs index ea078a23..7ea9339b 100644 --- a/scripts/package-artifacts.test.mjs +++ b/scripts/package-artifacts.test.mjs @@ -5,10 +5,9 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, it } from 'node:test'; -import { PUBLIC_NPM_PACKAGE_VERSION } from './build-public-npm-package.mjs'; +import { PUBLIC_NPM_PACKAGE_VERSION } from './public-npm-release-metadata.mjs'; import { CLI_PYTHON_ASSET_MANIFEST, - INTERNAL_NPM_WORKSPACE_PACKAGES, RUNTIME_WHEEL_DISTRIBUTION_NAME, RUNTIME_WHEEL_NORMALIZED_NAME, RUNTIME_WHEEL_PACKAGE_VERSION, @@ -35,7 +34,6 @@ async function writeJson(path, value) { async function writeReleaseMetadataInputs(root) { await writeJson(join(root, 'release-policy.json'), { schemaVersion: 1, - publicNpmPackageVersion: PUBLIC_NPM_PACKAGE_VERSION, releaseMode: 'ci-artifact-only', npm: { publish: false, @@ -62,12 +60,11 @@ async function writeReleaseMetadataInputs(root) { requiredBeforePublishing: ['Choose public release version.'], }); - for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) { + for (const packageInfo of NPM_ARTIFACT_PACKAGES) { await mkdir(join(root, packageInfo.packageRoot), { recursive: true }); await writeJson(join(root, packageInfo.packageRoot, 'package.json'), { name: packageInfo.name, - version: '0.0.0-private', - private: true, + version: PUBLIC_NPM_PACKAGE_VERSION, }); } } @@ -112,16 +109,20 @@ describe('packageArtifactLayout', () => { }); describe('buildArtifactCommands', () => { - it('builds TypeScript packages in parallel topology, then the runtime wheel, then packs npm artifacts', () => { + it('builds the CLI package, then the runtime wheel, then packs the npm tarball directly', () => { const layout = packageArtifactLayout('/repo/ktx', PUBLIC_NPM_PACKAGE_VERSION); const commands = buildArtifactCommands(layout); assert.deepEqual( - commands.map((command) => [command.command, command.args]), + commands.map((command) => [command.command, command.args, command.cwd]), [ - ['pnpm', ['--filter', './packages/*', '--workspace-concurrency=10', 'run', 'build']], - [process.execPath, ['scripts/build-python-runtime-wheel.mjs']], - [process.execPath, ['scripts/build-public-npm-package.mjs']], + ['pnpm', ['--filter', '@kaelio/ktx', 'run', 'build'], '/repo/ktx'], + [process.execPath, ['scripts/build-python-runtime-wheel.mjs'], '/repo/ktx'], + [ + 'pnpm', + ['pack', '--out', `/repo/ktx/dist/artifacts/npm/kaelio-ktx-${PUBLIC_NPM_PACKAGE_VERSION}.tgz`], + '/repo/ktx/packages/cli', + ], ], ); }); diff --git a/packages/context/scripts/pglite-hybrid-search-spike.mjs b/scripts/pglite-hybrid-search-spike.mjs similarity index 99% rename from packages/context/scripts/pglite-hybrid-search-spike.mjs rename to scripts/pglite-hybrid-search-spike.mjs index 500c34af..040bcf34 100644 --- a/packages/context/scripts/pglite-hybrid-search-spike.mjs +++ b/scripts/pglite-hybrid-search-spike.mjs @@ -7,8 +7,7 @@ import { fileURLToPath } from 'node:url'; const require = createRequire(import.meta.url); const scriptDir = dirname(fileURLToPath(import.meta.url)); -const contextDir = resolve(scriptDir, '..'); -const ktxRoot = resolve(contextDir, '../..'); +const ktxRoot = resolve(scriptDir, '..'); const docsDir = join(ktxRoot, 'docs'); const reportPath = join(docsDir, 'hybrid-search-pglite-spike.md'); diff --git a/packages/context/scripts/pglite-owner-process-prototype.mjs b/scripts/pglite-owner-process-prototype.mjs similarity index 99% rename from packages/context/scripts/pglite-owner-process-prototype.mjs rename to scripts/pglite-owner-process-prototype.mjs index 66a3e51f..dd4a77bd 100644 --- a/packages/context/scripts/pglite-owner-process-prototype.mjs +++ b/scripts/pglite-owner-process-prototype.mjs @@ -11,8 +11,7 @@ import { PGLiteSocketServer } from '@electric-sql/pglite-socket'; import { Client } from 'pg'; const scriptDir = dirname(fileURLToPath(import.meta.url)); -const contextDir = resolve(scriptDir, '..'); -const ktxRoot = resolve(contextDir, '../..'); +const ktxRoot = resolve(scriptDir, '..'); const reportPath = join(ktxRoot, 'docs', 'hybrid-search-pglite-owner-process.md'); async function timed(label, fn) { diff --git a/packages/context/scripts/pglite-sl-search-prototype.mjs b/scripts/pglite-sl-search-prototype.mjs similarity index 98% rename from packages/context/scripts/pglite-sl-search-prototype.mjs rename to scripts/pglite-sl-search-prototype.mjs index 43cb0e6e..8e414a32 100644 --- a/packages/context/scripts/pglite-sl-search-prototype.mjs +++ b/scripts/pglite-sl-search-prototype.mjs @@ -11,8 +11,7 @@ import { PGLiteSocketServer } from '@electric-sql/pglite-socket'; import { Client } from 'pg'; const scriptDir = dirname(fileURLToPath(import.meta.url)); -const contextDir = resolve(scriptDir, '..'); -const ktxRoot = resolve(contextDir, '../..'); +const ktxRoot = resolve(scriptDir, '..'); const reportPath = join(ktxRoot, 'docs', 'hybrid-search-pglite-sl-adapter-prototype.md'); async function timed(label, fn) { diff --git a/scripts/public-npm-release-metadata.mjs b/scripts/public-npm-release-metadata.mjs index 6aeba682..a295c427 100644 --- a/scripts/public-npm-release-metadata.mjs +++ b/scripts/public-npm-release-metadata.mjs @@ -8,6 +8,10 @@ export const PUBLIC_NPM_PACKAGE_NAME = '@kaelio/ktx'; export const PUBLIC_NPM_RELEASE_TAGS = new Set(['latest', 'next']); export const PUBLIC_NPM_BRANCH_RELEASE_TAG_PATTERN = /^branch-[a-z0-9]+(?:-[a-z0-9]+)*$/; +export function publicNpmPackageTarballName(version) { + return `kaelio-ktx-${version}.tgz`; +} + const SEMVER_PATTERN = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/; const SEMVER_PARTS_PATTERN = @@ -21,6 +25,10 @@ export function releasePolicyPath(rootDir = scriptRootDir()) { return join(rootDir, 'release-policy.json'); } +export function cliPackageJsonPath(rootDir = scriptRootDir()) { + return join(rootDir, 'packages', 'cli', 'package.json'); +} + function readJsonSync(path) { return JSON.parse(readFileSync(path, 'utf8')); } @@ -66,22 +74,28 @@ export function assertPublicNpmReleaseTag(tag) { throw new Error(`Invalid public npm release tag: ${tag}`); } +function readCliPackageVersion(rootDir = scriptRootDir()) { + const packageJson = readJsonSync(cliPackageJsonPath(rootDir)); + return assertPublicNpmPackageVersion(packageJson.version); +} + export function readPublicNpmReleaseMetadata(rootDir = scriptRootDir()) { const policy = readJsonSync(releasePolicyPath(rootDir)); - const version = assertPublicNpmPackageVersion(policy.publicNpmPackageVersion); const tag = assertPublicNpmReleaseTag(policy.npm?.tag); return { packageName: PUBLIC_NPM_PACKAGE_NAME, - version, + version: readCliPackageVersion(rootDir), tag, }; } export function publicNpmPackageVersion(rootDir = scriptRootDir()) { - return readPublicNpmReleaseMetadata(rootDir).version; + return readCliPackageVersion(rootDir); } +export const PUBLIC_NPM_PACKAGE_VERSION = publicNpmPackageVersion(); + export function publicPythonRuntimePackageVersion(rootDir = scriptRootDir()) { return publicNpmPackageVersionToPythonVersion(publicNpmPackageVersion(rootDir)); } diff --git a/scripts/published-package-smoke.test.mjs b/scripts/published-package-smoke.test.mjs index 719eed2a..09c54934 100644 --- a/scripts/published-package-smoke.test.mjs +++ b/scripts/published-package-smoke.test.mjs @@ -105,7 +105,7 @@ describe('published package smoke config', () => { /Invalid KTX_PUBLISHED_KTX_PACKAGE/, ); assert.throws( - () => readPublishedPackageSmokeConfig({ KTX_PUBLISHED_KTX_PACKAGE: '@ktx/cli public' }, []), + () => readPublishedPackageSmokeConfig({ KTX_PUBLISHED_KTX_PACKAGE: '@kaelio/ktx public' }, []), /Invalid KTX_PUBLISHED_KTX_PACKAGE/, ); assert.throws( @@ -114,7 +114,7 @@ describe('published package smoke config', () => { {}, [], { - packageName: '@ktx/cli public', + packageName: '@kaelio/ktx public', version: 'latest', registry: null, }, diff --git a/packages/context/scripts/relationship-benchmark-report.mjs b/scripts/relationship-benchmark-report.mjs similarity index 81% rename from packages/context/scripts/relationship-benchmark-report.mjs rename to scripts/relationship-benchmark-report.mjs index 43c86568..7af82d57 100644 --- a/packages/context/scripts/relationship-benchmark-report.mjs +++ b/scripts/relationship-benchmark-report.mjs @@ -1,18 +1,20 @@ import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { - KTX_RELATIONSHIP_BENCHMARK_MODES, buildKtxRelationshipBenchmarkReport, - currentKtxRelationshipBenchmarkDetector, formatKtxRelationshipBenchmarkReportMarkdown, +} from '../packages/cli/dist/context/scan/relationship-benchmark-report.js'; +import { + KTX_RELATIONSHIP_BENCHMARK_MODES, + currentKtxRelationshipBenchmarkDetector, ktxRelationshipBenchmarkDetectorWithLlm, loadKtxRelationshipBenchmarkFixtures, runKtxRelationshipBenchmarkSuite, -} from '../dist/scan/index.js'; +} from '../packages/cli/dist/context/scan/relationship-benchmarks.js'; const scriptDir = dirname(fileURLToPath(import.meta.url)); -const packageRoot = resolve(scriptDir, '..'); -const fixtureRoot = join(packageRoot, 'test/fixtures/relationship-benchmarks'); +const ktxRoot = resolve(scriptDir, '..'); +const fixtureRoot = join(ktxRoot, 'packages/cli/src/test/fixtures/relationship-benchmarks'); async function buildDetector() { const backend = process.env.KTX_BENCHMARK_LLM_BACKEND; @@ -28,7 +30,7 @@ async function buildDetector() { if (!project || !location) { throw new Error('KTX_BENCHMARK_VERTEX_PROJECT and KTX_BENCHMARK_VERTEX_LOCATION are required for vertex backend'); } - const { createKtxLlmProvider } = await import('@ktx/llm'); + const { createKtxLlmProvider } = await import('../packages/cli/dist/llm/index.js'); const provider = createKtxLlmProvider({ backend: 'vertex', vertex: { project, location }, diff --git a/scripts/release-readiness.mjs b/scripts/release-readiness.mjs index bcb66dc8..ae3aa825 100644 --- a/scripts/release-readiness.mjs +++ b/scripts/release-readiness.mjs @@ -5,7 +5,7 @@ import { dirname, join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { packageArtifactLayout, packageReleaseMetadata, verifyArtifactManifest } from './package-artifacts.mjs'; -import { assertPublicNpmPackageVersion, publicNpmPackageVersion } from './public-npm-release-metadata.mjs'; +import { publicNpmPackageVersion } from './public-npm-release-metadata.mjs'; import { readPublishedPackageSmokeConfig } from './published-package-smoke-config.mjs'; function scriptRootDir() { @@ -138,8 +138,6 @@ export function validateReleasePolicy(policy) { throw new Error(`Unsupported release policy schemaVersion: ${policy.schemaVersion}`); } assertSupportedReleaseMode(policy.releaseMode); - assertString(policy.publicNpmPackageVersion, 'Release policy publicNpmPackageVersion'); - assertPublicNpmPackageVersion(policy.publicNpmPackageVersion); assertPlainObject(policy.npm, 'Release policy npm'); assertPlainObject(policy.python, 'Release policy python'); assertPlainObject(policy.publishedPackageSmoke, 'Release policy publishedPackageSmoke'); @@ -228,20 +226,17 @@ function assertNonPublishingArtifactPolicy(policy, metadata, publicPackageVersio if (entry.releaseMode !== CI_ARTIFACT_ONLY_RELEASE_MODE) { throw new Error(`Package ${entry.packageName} releaseMode must remain ci-artifact-only`); } - if (entry.ecosystem === 'npm') { - const isPublicKtxPackage = entry.packageName === '@kaelio/ktx'; - if (isPublicKtxPackage) { - if (entry.private !== false) { - throw new Error(`${policyLabel} npm package @kaelio/ktx must be publishable when npm.publish is false`); - } - if (entry.packageVersion !== publicPackageVersion) { - throw new Error(`${policyLabel} npm package @kaelio/ktx must use public version ${publicPackageVersion}`); - } - } else if (entry.private !== true) { - throw new Error(`${policyLabel} npm package ${entry.packageName} must remain private`); - } else if (!entry.packageVersion.endsWith('-private')) { - throw new Error(`${policyLabel} npm package ${entry.packageName} must use a private version suffix`); - } + if (entry.ecosystem !== 'npm') { + continue; + } + if (entry.packageName !== '@kaelio/ktx') { + throw new Error(`${policyLabel} unexpected npm package ${entry.packageName}`); + } + if (entry.private !== false) { + throw new Error(`${policyLabel} npm package @kaelio/ktx must be publishable when npm.publish is false`); + } + if (entry.packageVersion !== publicPackageVersion) { + throw new Error(`${policyLabel} npm package @kaelio/ktx must use public version ${publicPackageVersion}`); } } } diff --git a/scripts/release-readiness.test.mjs b/scripts/release-readiness.test.mjs index 820d72ea..e01337b9 100644 --- a/scripts/release-readiness.test.mjs +++ b/scripts/release-readiness.test.mjs @@ -5,12 +5,11 @@ import { join } from 'node:path'; import { describe, it } from 'node:test'; import { - INTERNAL_NPM_WORKSPACE_PACKAGES, NPM_ARTIFACT_PACKAGES, packageArtifactLayout, writeArtifactManifest, } from './package-artifacts.mjs'; -import { PUBLIC_NPM_PACKAGE_VERSION } from './build-public-npm-package.mjs'; +import { PUBLIC_NPM_PACKAGE_VERSION } from './public-npm-release-metadata.mjs'; import { RUNTIME_WHEEL_PACKAGE_VERSION } from './build-python-runtime-wheel.mjs'; import { readReleasePolicy, releasePolicyPath, releaseReadinessReport } from './release-readiness.mjs'; @@ -19,12 +18,11 @@ async function writeJson(path, value) { } async function writeReleaseMetadataInputs(root) { - for (const packageInfo of INTERNAL_NPM_WORKSPACE_PACKAGES) { + for (const packageInfo of NPM_ARTIFACT_PACKAGES) { await mkdir(join(root, packageInfo.packageRoot), { recursive: true }); await writeJson(join(root, packageInfo.packageRoot, 'package.json'), { name: packageInfo.name, - version: '0.0.0-private', - private: true, + version: PUBLIC_NPM_PACKAGE_VERSION, }); } } @@ -54,7 +52,6 @@ function releasePolicy(overrides = {}) { return { schemaVersion: 1, - publicNpmPackageVersion: PUBLIC_NPM_PACKAGE_VERSION, releaseMode: 'ci-artifact-only', npm: { publish: false, @@ -528,7 +525,7 @@ describe('release readiness policy', () => { await writeReadyFixture(root, { policy: releasePolicy({ publishedPackageSmoke: { - packageName: '@ktx/cli public', + packageName: '@kaelio/ktx public', version: 'latest', registry: null, }, @@ -550,7 +547,7 @@ describe('release readiness policy', () => { await writeReadyFixture(root, { policy: releasePolicy({ npm: { - packages: ['@kaelio/ktx', '@ktx/context'], + packages: ['@kaelio/ktx', '@ktx/internal-only'], }, }), }); diff --git a/scripts/run-ktx.test.mjs b/scripts/run-ktx.test.mjs index 98035aef..65ca4797 100644 --- a/scripts/run-ktx.test.mjs +++ b/scripts/run-ktx.test.mjs @@ -40,7 +40,7 @@ test('runWorkspaceKtx runs the built CLI when it already exists', async () => { readdir: fs.readdir, execFile: async (command, args, options) => { calls.push({ command, args, cwd: options.cwd }); - return { stdout: '@ktx/cli 0.0.0-private\n', stderr: '' }; + return { stdout: '@kaelio/ktx 0.0.0-private\n', stderr: '' }; }, stdout: { write: (chunk) => logs.push(['stdout', chunk]) }, stderr: { write: (chunk) => logs.push(['stderr', chunk]) }, @@ -54,7 +54,7 @@ test('runWorkspaceKtx runs the built CLI when it already exists', async () => { cwd: '/workspace/ktx', }, ]); - assert.deepEqual(logs, [['stdout', '@ktx/cli 0.0.0-private\n']]); + assert.deepEqual(logs, [['stdout', '@kaelio/ktx 0.0.0-private\n']]); }); test('runWorkspaceKtx forwards a caller-provided environment to buffered commands', async () => { @@ -69,7 +69,7 @@ test('runWorkspaceKtx forwards a caller-provided environment to buffered command env: { PATH: '/bin', GIT_CEILING_DIRECTORIES: '/workspace/ktx/examples' }, execFile: async (command, args, options) => { calls.push({ command, args, cwd: options.cwd, env: options.env }); - return { stdout: '@ktx/cli 0.0.0-private\n', stderr: '' }; + return { stdout: '@kaelio/ktx 0.0.0-private\n', stderr: '' }; }, stdout: { write: () => undefined }, stderr: { write: () => undefined }, diff --git a/scripts/semantic-release-config.cjs b/scripts/semantic-release-config.cjs index 2f7c3426..2dee466a 100644 --- a/scripts/semantic-release-config.cjs +++ b/scripts/semantic-release-config.cjs @@ -166,7 +166,13 @@ function createReleaseConfig(env = process.env) { [ '@semantic-release/git', { - assets: ['package.json', 'release-policy.json', 'packages/cli/package.json'], + assets: [ + 'package.json', + 'release-policy.json', + 'packages/cli/package.json', + 'python/ktx-daemon/pyproject.toml', + 'python/ktx-sl/pyproject.toml', + ], message: 'chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}', }, ], diff --git a/scripts/semantic-release-config.test.mjs b/scripts/semantic-release-config.test.mjs index 56c9e88f..24289896 100644 --- a/scripts/semantic-release-config.test.mjs +++ b/scripts/semantic-release-config.test.mjs @@ -88,7 +88,13 @@ describe('semantic-release config', () => { const config = createReleaseConfig({ KTX_RELEASE_KIND: kind, GITHUB_REF_NAME: 'main' }); const git = gitPluginOptions(config); assert.ok(git, `${kind}: @semantic-release/git plugin must be configured`); - assert.deepEqual(git.assets, ['package.json', 'release-policy.json', 'packages/cli/package.json']); + assert.deepEqual(git.assets, [ + 'package.json', + 'release-policy.json', + 'packages/cli/package.json', + 'python/ktx-daemon/pyproject.toml', + 'python/ktx-sl/pyproject.toml', + ]); assert.match(git.message, /^chore\(release\): \$\{nextRelease\.version\} \[skip ci\]/); } }); diff --git a/scripts/standalone-ci-workflow.test.mjs b/scripts/standalone-ci-workflow.test.mjs index 5aa4cc02..103d61d7 100644 --- a/scripts/standalone-ci-workflow.test.mjs +++ b/scripts/standalone-ci-workflow.test.mjs @@ -24,10 +24,8 @@ describe('standalone KTX CI workflow', () => { 'name: Pre-commit checks', 'typescript-checks:', 'name: TypeScript checks', - 'slow-context-tests:', - 'name: Slow context tests', 'slow-cli-tests:', - 'name: Slow CLI tests', + 'name: Slow TypeScript tests', 'cli-smoke-tests:', 'name: CLI smoke tests', 'python-checks:', @@ -42,8 +40,7 @@ describe('standalone KTX CI workflow', () => { 'pnpm install --frozen-lockfile', 'pnpm run check', 'pnpm run build', - 'pnpm --filter @ktx/context run test:slow', - 'pnpm --filter @ktx/cli run test:slow', + 'pnpm --filter @kaelio/ktx run test:slow', 'pnpm run smoke', 'actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405', 'python-version: "3.13"', diff --git a/scripts/test-tiering.test.mjs b/scripts/test-tiering.test.mjs index 5a984dd1..bd43ce71 100644 --- a/scripts/test-tiering.test.mjs +++ b/scripts/test-tiering.test.mjs @@ -33,41 +33,36 @@ describe('test tiering', () => { ]; const contextSlowTests = [ - 'src/scan/local-scan.test.ts', - 'src/mcp/local-project-ports.test.ts', - 'src/ingest/local-stage-ingest.test.ts', - 'src/sl/pglite-sl-search-prototype.test.ts', - 'src/core/git.service.test.ts', - 'src/ingest/local-adapters.test.ts', - 'src/ingest/local-bundle-ingest.test.ts', - 'src/ingest/local-metabase-ingest.test.ts', - 'src/sl/local-sl.test.ts', - 'src/search/pglite-owner-process.test.ts', - 'src/scan/local-enrichment-artifacts.test.ts', - 'src/search/pglite-spike.test.ts', - 'src/wiki/local-knowledge.test.ts', - 'src/sl/local-query.test.ts', - 'src/scan/relationship-review-decisions.test.ts', - 'src/scan/relationship-profiling.test.ts', + 'src/context/scan/local-scan.test.ts', + 'src/context/mcp/local-project-ports.test.ts', + 'src/context/ingest/local-stage-ingest.test.ts', + 'src/context/sl/pglite-sl-search-prototype.test.ts', + 'src/context/core/git.service.test.ts', + 'src/context/ingest/local-adapters.test.ts', + 'src/context/ingest/local-bundle-ingest.test.ts', + 'src/context/ingest/local-metabase-ingest.test.ts', + 'src/context/sl/local-sl.test.ts', + 'src/context/search/pglite-owner-process.test.ts', + 'src/context/scan/local-enrichment-artifacts.test.ts', + 'src/context/search/pglite-spike.test.ts', + 'src/context/wiki/local-knowledge.test.ts', + 'src/context/sl/local-query.test.ts', + 'src/context/scan/relationship-review-decisions.test.ts', + 'src/context/scan/relationship-profiling.test.ts', ]; it('keeps slow package tests out of default local package test scripts', async () => { const cliPackage = await readJson('../packages/cli/package.json'); - const contextPackage = await readJson('../packages/context/package.json'); - assertScriptContainsAll(cliPackage.scripts.test, cliSlowTests.map((file) => `--exclude ${file}`)); - assertScriptContainsAll(contextPackage.scripts.test, contextSlowTests.map((file) => `--exclude ${file}`)); - assert.match(contextPackage.scripts.test, /--exclude src\/scan\/relationship-benchmarks\.test\.ts/); + assertScriptContainsAll(cliPackage.scripts.test, contextSlowTests.map((file) => `--exclude ${file}`)); }); it('provides explicit slow package test scripts for CI', async () => { const rootPackage = await readJson('../package.json'); const cliPackage = await readJson('../packages/cli/package.json'); - const contextPackage = await readJson('../packages/context/package.json'); - - assert.equal(rootPackage.scripts['test:slow'], 'pnpm --filter @ktx/context run test:slow && pnpm --filter @ktx/cli run test:slow'); + assert.equal(rootPackage.scripts['test:slow'], 'pnpm --filter @kaelio/ktx run test:slow'); assertScriptContainsAll(cliPackage.scripts['test:slow'], cliSlowTests); - assertScriptContainsAll(contextPackage.scripts['test:slow'], contextSlowTests); - assert.doesNotMatch(contextPackage.scripts['test:slow'], /relationship-benchmarks\.test\.ts/); + assertScriptContainsAll(cliPackage.scripts['test:slow'], contextSlowTests); + assert.doesNotMatch(cliPackage.scripts['test:slow'], /relationship-benchmarks\.test\.ts/); }); }); diff --git a/scripts/update-public-release-version.mjs b/scripts/update-public-release-version.mjs index 0df2d234..8ab01849 100644 --- a/scripts/update-public-release-version.mjs +++ b/scripts/update-public-release-version.mjs @@ -8,6 +8,7 @@ import { PUBLIC_NPM_PACKAGE_NAME, assertPublicNpmPackageVersion, assertPublicNpmReleaseTag, + publicNpmPackageVersionToPythonVersion, releasePolicyPath, } from './public-npm-release-metadata.mjs'; @@ -23,6 +24,42 @@ async function writeJson(path, value) { await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); } +function pyprojectWithProjectVersion(source, version) { + const lines = source.split('\n'); + let inProject = false; + let replaced = false; + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index]; + const sectionMatch = /^\s*\[([^\]]+)\]\s*$/.exec(line); + if (sectionMatch) { + inProject = sectionMatch[1] === 'project'; + continue; + } + if (inProject && /^\s*version\s*=\s*"[^"]*"\s*$/.test(line)) { + lines[index] = `version = "${version}"`; + replaced = true; + break; + } + } + if (!replaced) { + throw new Error('No [project].version assignment found in pyproject.toml'); + } + return lines.join('\n'); +} + +async function rewritePyprojectVersion(path, version) { + const source = await readFile(path, 'utf8'); + await writeFile(path, pyprojectWithProjectVersion(source, version)); +} + +function safePythonVersionFor(version) { + try { + return publicNpmPackageVersionToPythonVersion(version); + } catch { + return null; + } +} + export async function updatePublicReleaseVersion(rootDir, version, tag) { const safeVersion = assertPublicNpmPackageVersion(version); const safeTag = assertPublicNpmReleaseTag(tag); @@ -37,9 +74,15 @@ export async function updatePublicReleaseVersion(rootDir, version, tag) { cliPackageJson.version = safeVersion; await writeJson(cliPackageJsonPath, cliPackageJson); + const pythonVersion = safePythonVersionFor(safeVersion); + if (pythonVersion !== null) { + await rewritePyprojectVersion(join(rootDir, 'python', 'ktx-daemon', 'pyproject.toml'), pythonVersion); + await rewritePyprojectVersion(join(rootDir, 'python', 'ktx-sl', 'pyproject.toml'), pythonVersion); + } + const policyPath = releasePolicyPath(rootDir); const policy = await readJson(policyPath); - policy.publicNpmPackageVersion = safeVersion; + delete policy.publicNpmPackageVersion; policy.releaseMode = 'npm-public-release-ready'; policy.requiredBeforePublishing = []; policy.npm = { @@ -60,6 +103,7 @@ export async function updatePublicReleaseVersion(rootDir, version, tag) { return { version: safeVersion, tag: safeTag, + pythonVersion, }; } diff --git a/scripts/update-public-release-version.test.mjs b/scripts/update-public-release-version.test.mjs index 3ddd67f2..63391819 100644 --- a/scripts/update-public-release-version.test.mjs +++ b/scripts/update-public-release-version.test.mjs @@ -11,24 +11,51 @@ async function writeJson(path, value) { await writeFile(path, `${JSON.stringify(value, null, 2)}\n`); } +async function writeText(path, value) { + await mkdir(join(path, '..'), { recursive: true }); + await writeFile(path, value); +} + async function readJson(path) { return JSON.parse(await readFile(path, 'utf8')); } +async function readText(path) { + return readFile(path, 'utf8'); +} + +const DAEMON_PYPROJECT = `[project] +name = "ktx-daemon" +version = "0.4.0" +description = "Portable compute package for KTX semantic-layer operations" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" +`; + +const SL_PYPROJECT = `[project] +name = "ktx-sl" +version = "0.4.0" +description = "Agent-first semantic layer engine with aggregate locality" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" +`; + async function writeReleaseFixture(root) { await writeJson(join(root, 'package.json'), { name: 'ktx-workspace', - version: '0.0.0-private', + version: '0.4.0', private: true, }); await writeJson(join(root, 'packages', 'cli', 'package.json'), { - name: '@ktx/cli', - version: '0.0.0-private', - private: true, + name: '@kaelio/ktx', + version: '0.4.0', }); await writeJson(join(root, 'release-policy.json'), { schemaVersion: 1, - publicNpmPackageVersion: '0.1.0-rc.1', releaseMode: 'ci-artifact-only', npm: { publish: false, @@ -44,7 +71,7 @@ async function writeReleaseFixture(root) { }, publishedPackageSmoke: { packageName: '@kaelio/ktx', - version: '0.1.0-rc.1', + version: 'latest', registry: null, }, runtimeInstaller: { @@ -54,6 +81,8 @@ async function writeReleaseFixture(root) { }, requiredBeforePublishing: ['Choose public release version.'], }); + await writeText(join(root, 'python', 'ktx-daemon', 'pyproject.toml'), DAEMON_PYPROJECT); + await writeText(join(root, 'python', 'ktx-sl', 'pyproject.toml'), SL_PYPROJECT); } describe('updatePublicReleaseVersion', () => { @@ -66,9 +95,10 @@ describe('updatePublicReleaseVersion', () => { assert.equal((await readJson(join(root, 'package.json'))).version, '0.1.0-rc.2'); assert.equal((await readJson(join(root, 'packages', 'cli', 'package.json'))).version, '0.1.0-rc.2'); + assert.match(await readText(join(root, 'python', 'ktx-daemon', 'pyproject.toml')), /^version = "0\.1\.0rc2"$/m); + assert.match(await readText(join(root, 'python', 'ktx-sl', 'pyproject.toml')), /^version = "0\.1\.0rc2"$/m); assert.deepEqual(await readJson(join(root, 'release-policy.json')), { schemaVersion: 1, - publicNpmPackageVersion: '0.1.0-rc.2', releaseMode: 'npm-public-release-ready', npm: { publish: true, @@ -111,8 +141,17 @@ describe('updatePublicReleaseVersion', () => { (await readJson(join(root, 'packages', 'cli', 'package.json'))).version, '0.1.0-feature-foo.0', ); + assert.match( + await readText(join(root, 'python', 'ktx-daemon', 'pyproject.toml')), + /^version = "0\.4\.0"$/m, + ); + assert.match( + await readText(join(root, 'python', 'ktx-sl', 'pyproject.toml')), + /^version = "0\.4\.0"$/m, + ); const policy = await readJson(join(root, 'release-policy.json')); - assert.equal(policy.publicNpmPackageVersion, '0.1.0-feature-foo.0'); + assert.equal(policy.publicNpmPackageVersion, undefined); + assert.equal(policy.publishedPackageSmoke.version, '0.1.0-feature-foo.0'); assert.equal(policy.npm.tag, 'branch-feature-foo'); } finally { await rm(root, { recursive: true, force: true }); diff --git a/uv.lock b/uv.lock index 6c5afa29..29ce5981 100644 --- a/uv.lock +++ b/uv.lock @@ -440,7 +440,7 @@ wheels = [ [[package]] name = "ktx-daemon" -version = "0.0.0+private" +version = "0.4.1" source = { editable = "python/ktx-daemon" } dependencies = [ { name = "fastapi" }, @@ -495,7 +495,7 @@ dev = [ [[package]] name = "ktx-sl" -version = "0.0.0+private" +version = "0.4.1" source = { editable = "python/ktx-sl" } dependencies = [ { name = "pydantic" },