From 34d4a1e9e135598bd37e65b0b6e59aedf5ec3c2c Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Thu, 21 May 2026 12:41:20 +0200 Subject: [PATCH] refactor(cli): delete internal barrel index.ts files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 34 `index.ts` re-export barrels inside `packages/cli/src/` were holdovers from the pre-fold multi-workspace structure. Post-fold-in they served no production purpose: external consumers go through the single package main entry, and in-repo callers mostly imported through them only because the path was short. Internally, knip flagged most barrel re-exports as production-dead (only reached via tests). This change: - Deletes every internal barrel except `packages/cli/src/index.ts` (the published package entry). - Rewrites ~270 source/test files to import each name directly from the file that defines it. - Moves `tools/warehouse-verification/index.ts` to `create-warehouse-verification-tools.ts` (the function it defined locally) and updates its single consumer. - Renames `search/backend-conformance.ts` → `.test-utils.ts` to match the existing test-helper file convention. - Deletes 13 dead test-only chains (dbt-descriptions/*, live-database/extracted-schema, live-database/structural-sync, relationship-* feedback/review chain) plus their tests and a cascading orphan integration test. - Updates test mocks that pointed at deleted barrel paths (notion-client, connector barrels in scan/local-scan-connectors tests) to mock the source files instead. - Points the maintainer benchmark script (`scripts/relationship-benchmark-report.mjs`) at source files instead of `dist/context/scan/index.js`. - Drops the barrel `!` entries from `knip.json`; adds explicit production entries only for the benchmark code reached via dist by the maintainer script. Net: 413 files changed, ~1.2k insertions, ~9.4k deletions. `pnpm run dead-code` (Biome + knip default + knip production) and `pnpm run type-check` are clean; 2277 tests pass. --- knip.json | 51 +- packages/cli/src/admin-reindex.test.ts | 2 +- packages/cli/src/admin-reindex.ts | 8 +- packages/cli/src/admin.ts | 2 +- .../cli/src/claude-code-prompt-caching.ts | 2 +- packages/cli/src/cli-runtime.ts | 2 +- packages/cli/src/connection.test.ts | 9 +- packages/cli/src/connection.ts | 28 +- .../src/connectors/bigquery/connector.test.ts | 13 +- .../cli/src/connectors/bigquery/connector.ts | 26 +- .../cli/src/connectors/bigquery/dialect.ts | 2 +- packages/cli/src/connectors/bigquery/index.ts | 18 - .../bigquery/live-database-introspection.ts | 4 +- .../connectors/clickhouse/connector.test.ts | 9 +- .../src/connectors/clickhouse/connector.ts | 23 +- .../cli/src/connectors/clickhouse/dialect.ts | 2 +- .../cli/src/connectors/clickhouse/index.ts | 16 - .../clickhouse/live-database-introspection.ts | 4 +- .../src/connectors/mysql/connector.test.ts | 9 +- .../cli/src/connectors/mysql/connector.ts | 24 +- packages/cli/src/connectors/mysql/dialect.ts | 2 +- packages/cli/src/connectors/mysql/index.ts | 15 - .../mysql/live-database-introspection.ts | 4 +- .../src/connectors/postgres/connector.test.ts | 9 +- .../cli/src/connectors/postgres/connector.ts | 24 +- .../cli/src/connectors/postgres/dialect.ts | 2 +- .../postgres/historic-sql-query-client.ts | 2 +- packages/cli/src/connectors/postgres/index.ts | 21 - .../postgres/live-database-introspection.ts | 4 +- .../connectors/snowflake/connector.test.ts | 10 +- .../cli/src/connectors/snowflake/connector.ts | 23 +- .../cli/src/connectors/snowflake/dialect.ts | 2 +- .../cli/src/connectors/snowflake/index.ts | 18 - .../snowflake/live-database-introspection.ts | 4 +- .../src/connectors/sqlite/connector.test.ts | 8 +- .../cli/src/connectors/sqlite/connector.ts | 24 +- packages/cli/src/connectors/sqlite/dialect.ts | 2 +- packages/cli/src/connectors/sqlite/index.ts | 16 - .../sqlite/live-database-introspection.ts | 4 +- .../connectors/sqlserver/connector.test.ts | 10 +- .../cli/src/connectors/sqlserver/connector.ts | 25 +- .../cli/src/connectors/sqlserver/dialect.ts | 2 +- .../cli/src/connectors/sqlserver/index.ts | 17 - .../sqlserver/live-database-introspection.ts | 4 +- packages/cli/src/context-build-view.test.ts | 2 +- packages/cli/src/context-build-view.ts | 2 +- packages/cli/src/context/agent/index.ts | 9 - .../cli/src/context/connections/dialects.ts | 2 +- packages/cli/src/context/connections/index.ts | 30 - .../src/context/connections/notion-config.ts | 5 +- .../src/context/connections/query-executor.ts | 2 +- .../connections/sqlite-query-executor.ts | 1 + .../cli/src/context/core/config-reference.ts | 1 + packages/cli/src/context/core/config.ts | 4 +- packages/cli/src/context/core/index.ts | 27 - packages/cli/src/context/core/redaction.ts | 1 + .../context/core/session-worktree.service.ts | 2 +- packages/cli/src/context/daemon/index.ts | 1 - .../context/daemon/semantic-layer-compute.ts | 22 +- packages/cli/src/context/index-sync/index.ts | 2 - .../src/context/index-sync/reindex.test.ts | 4 +- .../cli/src/context/index-sync/reindex.ts | 12 +- packages/cli/src/context/index-sync/types.ts | 2 +- packages/cli/src/context/index.ts | 128 ---- .../cli/src/context/ingest/action-identity.ts | 2 +- .../dbt-descriptions/match-tables.test.ts | 75 -- .../adapters/dbt-descriptions/match-tables.ts | 127 ---- .../merge-semantic-model-tables.test.ts | 62 -- .../merge-semantic-model-tables.ts | 37 - .../adapters/dbt-descriptions/parse-schema.ts | 20 +- .../to-description-updates.test.ts | 102 --- .../to-description-updates.ts | 70 -- .../to-metadata-updates.test.ts | 70 -- .../dbt-descriptions/to-metadata-updates.ts | 74 -- .../to-relationship-updates.test.ts | 62 -- .../to-relationship-updates.ts | 57 -- .../dbt-extraction-golden-parity.test.ts | 410 ----------- .../ingest/adapters/historic-sql/evidence.ts | 4 +- .../historic-sql/historic-sql.adapter.test.ts | 2 +- .../local-ingest-acceptance.test.ts | 11 +- .../adapters/historic-sql/projection.ts | 2 +- .../adapters/historic-sql/skill-schemas.ts | 2 +- .../historic-sql/stage-unified.test.ts | 2 +- .../adapters/historic-sql/stage-unified.ts | 2 +- .../ingest/adapters/historic-sql/types.ts | 5 +- .../live-database/daemon-introspection.ts | 4 +- .../live-database/extracted-schema.test.ts | 136 ---- .../live-database/extracted-schema.ts | 61 -- .../ingest/adapters/live-database/manifest.ts | 4 +- .../live-database/structural-sync.test.ts | 428 ----------- .../adapters/live-database/structural-sync.ts | 525 -------------- .../context/ingest/adapters/looker/factory.ts | 2 + .../context/ingest/adapters/looker/fetch.ts | 2 +- .../adapters/looker/local-looker.adapter.ts | 20 +- .../adapters/looker/local-runtime-store.ts | 4 +- .../context/ingest/adapters/looker/mapping.ts | 19 +- .../context/ingest/adapters/looker/scope.ts | 4 +- .../tools/looker-query-to-sl.tool.test.ts | 2 +- .../looker/tools/looker-query-to-sl.tool.ts | 13 +- .../context/ingest/adapters/looker/types.ts | 5 +- .../ingest/adapters/metabase/client-port.ts | 6 +- .../ingest/adapters/metabase/client.ts | 2 + .../adapters/metabase/fanout-planner.ts | 2 +- .../metabase/local-metabase.adapter.test.ts | 2 +- .../metabase/local-metabase.adapter.ts | 5 +- .../metabase/local-source-state-store.test.ts | 2 +- .../metabase/local-source-state-store.ts | 9 +- .../ingest/adapters/metabase/mapping.ts | 13 +- .../adapters/metabase/source-state-port.ts | 2 +- .../context/ingest/adapters/metabase/types.ts | 3 +- .../ingest/adapters/metricflow/deep-parse.ts | 9 +- .../metricflow/import-semantic-models.ts | 14 +- .../ingest/adapters/metricflow/pull-config.ts | 2 +- .../metricflow/semantic-models.test.ts | 3 +- .../adapters/metricflow/semantic-models.ts | 15 +- .../context/ingest/adapters/notion/chunk.ts | 1 + .../cli/src/context/ingest/artifact-gates.ts | 8 +- .../candidate-dedup.service.ts | 2 +- .../context-candidate-carryforward.service.ts | 2 +- .../curator-pagination.service.ts | 8 +- .../ingest/context-candidates/index.ts | 29 - .../ingest/context-candidates/types.ts | 19 +- .../context-evidence-index.service.ts | 2 +- .../context/ingest/context-evidence/index.ts | 12 - .../sqlite-context-evidence-store.test.ts | 2 +- .../sqlite-context-evidence-store.ts | 16 +- .../context/ingest/dbt-shared/project-vars.ts | 2 + .../context/ingest/dbt-shared/schema-files.ts | 1 + .../src/context/ingest/final-gate-repair.ts | 4 +- .../src/context/ingest/finalization-scope.ts | 4 +- packages/cli/src/context/ingest/index.ts | 670 ------------------ ...ingest-bundle.runner.isolated-diff.test.ts | 5 +- .../ingest/ingest-bundle.runner.test.ts | 2 +- .../context/ingest/ingest-bundle.runner.ts | 17 +- .../ingest/ingest-runtime-assets.test.ts | 4 +- .../cli/src/context/ingest/ingest-trace.ts | 18 +- .../context/ingest/isolated-diff/git-patch.ts | 1 + .../isolated-diff/patch-integrator.test.ts | 2 +- .../ingest/isolated-diff/patch-integrator.ts | 4 +- .../textual-conflict-resolver.ts | 2 +- .../isolated-diff/work-unit-executor.test.ts | 2 +- .../isolated-diff/work-unit-executor.ts | 2 +- .../src/context/ingest/local-adapters.test.ts | 4 +- .../cli/src/context/ingest/local-adapters.ts | 8 +- .../ingest/local-bundle-ingest.test.ts | 4 +- .../ingest/local-bundle-runtime.test.ts | 4 +- .../context/ingest/local-bundle-runtime.ts | 126 ++-- .../cli/src/context/ingest/local-ingest.ts | 37 +- .../ingest/local-mapping-reconcile.test.ts | 3 +- .../context/ingest/local-mapping-reconcile.ts | 9 +- .../ingest/local-metabase-ingest.test.ts | 4 +- .../context/ingest/local-stage-ingest.test.ts | 2 +- .../src/context/ingest/local-stage-ingest.ts | 6 +- .../src/context/ingest/memory-flow/events.ts | 3 +- .../src/context/ingest/memory-flow/index.ts | 17 - .../context/ingest/memory-flow/interaction.ts | 2 + .../src/context/ingest/memory-flow/schema.ts | 13 +- .../src/context/ingest/memory-flow/visuals.ts | 6 +- .../src/context/ingest/page-triage/index.ts | 9 - .../ingest/page-triage/page-triage.service.ts | 12 +- packages/cli/src/context/ingest/ports.ts | 56 +- .../src/context/ingest/raw-sources-paths.ts | 2 + packages/cli/src/context/ingest/repo-fetch.ts | 7 + .../cli/src/context/ingest/report-snapshot.ts | 2 +- packages/cli/src/context/ingest/reports.ts | 18 +- .../ingest/semantic-layer-target-policy.ts | 2 + .../ingest/sqlite-bundle-ingest-store.test.ts | 3 +- .../ingest/stages/build-reconcile-context.ts | 2 +- .../context/ingest/stages/build-wu-context.ts | 3 +- .../ingest/stages/stage-3-work-units.test.ts | 4 +- .../ingest/stages/stage-3-work-units.ts | 8 +- .../ingest/stages/stage-4-reconciliation.ts | 4 +- .../ingest/stages/stage-index.types.ts | 4 +- .../ingest/stages/validate-wu-sources.ts | 5 +- .../context/ingest/tools/tool-call-logger.ts | 2 +- .../ingest/tools/verification-ledger.tool.ts | 2 +- ...=> create-warehouse-verification-tools.ts} | 6 +- .../discover-data.tool.test.ts | 2 +- .../discover-data.tool.ts | 2 +- .../entity-details.tool.test.ts | 4 +- .../entity-details.tool.ts | 2 +- .../sql-execution.tool.test.ts | 4 +- .../sql-execution.tool.ts | 6 +- packages/cli/src/context/ingest/types.ts | 10 +- .../cli/src/context/ingest/wiki-body-refs.ts | 4 +- .../src/context/ingest/wiki-sl-ref-repair.ts | 8 +- .../cli/src/context/llm/ai-sdk-runtime.ts | 7 +- .../cli/src/context/llm/claude-code-env.ts | 1 + .../src/context/llm/claude-code-runtime.ts | 3 +- .../src/context/llm/debug-request-recorder.ts | 5 +- .../cli/src/context/llm/embedding-port.ts | 2 +- packages/cli/src/context/llm/index.ts | 44 -- packages/cli/src/context/llm/local-config.ts | 14 +- packages/cli/src/context/llm/runtime-port.ts | 3 +- packages/cli/src/context/llm/runtime-tools.ts | 5 - packages/cli/src/context/mcp/context-tools.ts | 5 +- packages/cli/src/context/mcp/index.ts | 25 - .../context/mcp/local-project-ports.test.ts | 11 +- .../src/context/mcp/local-project-ports.ts | 20 +- packages/cli/src/context/mcp/server.test.ts | 10 +- packages/cli/src/context/mcp/server.ts | 1 + packages/cli/src/context/mcp/types.ts | 31 +- .../cli/src/context/memory/capture-signals.ts | 1 + packages/cli/src/context/memory/index.ts | 41 -- .../src/context/memory/local-memory.test.ts | 2 +- .../cli/src/context/memory/local-memory.ts | 83 +-- .../memory-agent.service.ingest.test.ts | 2 +- .../memory/memory-agent.service.test.ts | 4 +- .../context/memory/memory-agent.service.ts | 27 +- .../src/context/memory/memory-runs.test.ts | 3 +- .../cli/src/context/memory/memory-runs.ts | 3 +- .../memory/memory-runtime-assets.test.ts | 7 +- packages/cli/src/context/memory/types.ts | 42 +- packages/cli/src/context/project/config.ts | 4 +- .../cli/src/context/project/driver-schemas.ts | 1 - packages/cli/src/context/project/index.ts | 45 -- .../project/local-git-file-store.test.ts | 3 +- .../context/project/local-git-file-store.ts | 11 +- .../context/project/mappings-yaml-schema.ts | 3 + packages/cli/src/context/project/project.ts | 3 +- .../cli/src/context/project/setup-config.ts | 5 +- packages/cli/src/context/prompts/index.ts | 2 - .../cli/src/context/prompts/prompt.service.ts | 2 +- .../cli/src/context/scan/credentials.test.ts | 2 +- packages/cli/src/context/scan/credentials.ts | 7 +- .../cli/src/context/scan/data-dictionary.ts | 5 +- .../context/scan/description-generation.ts | 14 +- .../cli/src/context/scan/embedding-text.ts | 2 +- .../cli/src/context/scan/enrichment-state.ts | 2 +- .../cli/src/context/scan/enrichment-types.ts | 16 +- .../src/context/scan/entity-details.test.ts | 2 +- .../cli/src/context/scan/entity-details.ts | 10 +- packages/cli/src/context/scan/index.ts | 409 ----------- .../scan/local-enrichment-artifacts.test.ts | 2 +- .../scan/local-enrichment-artifacts.ts | 13 +- .../cli/src/context/scan/local-enrichment.ts | 2 +- .../cli/src/context/scan/local-scan.test.ts | 9 +- packages/cli/src/context/scan/local-scan.ts | 23 +- .../scan/local-structural-artifacts.test.ts | 2 +- .../scan/local-structural-artifacts.ts | 2 +- .../scan/relationship-artifacts.test.ts | 308 -------- .../context/scan/relationship-artifacts.ts | 75 -- .../context/scan/relationship-benchmarks.ts | 2 +- .../src/context/scan/relationship-budget.ts | 3 +- .../context/scan/relationship-candidates.ts | 9 +- .../scan/relationship-composite-candidates.ts | 6 +- .../context/scan/relationship-diagnostics.ts | 10 +- .../scan/relationship-discovery.test.ts | 2 +- .../context/scan/relationship-discovery.ts | 2 +- .../relationship-feedback-calibration.test.ts | 211 ------ .../scan/relationship-feedback-calibration.ts | 300 -------- .../scan/relationship-feedback-export.test.ts | 270 ------- .../scan/relationship-feedback-export.ts | 179 ----- .../scan/relationship-graph-resolver.ts | 8 +- .../scan/relationship-llm-proposal.test.ts | 2 +- .../context/scan/relationship-llm-proposal.ts | 4 +- .../scan/relationship-name-similarity.ts | 2 + .../context/scan/relationship-profiling.ts | 1 + .../scan/relationship-review-apply.test.ts | 351 --------- .../context/scan/relationship-review-apply.ts | 231 ------ .../relationship-review-decisions.test.ts | 363 ---------- .../scan/relationship-review-decisions.ts | 182 ----- .../src/context/scan/relationship-scoring.ts | 8 +- .../relationship-threshold-advice.test.ts | 241 ------- .../scan/relationship-threshold-advice.ts | 335 --------- .../context/scan/relationship-validation.ts | 6 +- .../src/context/scan/type-normalization.ts | 2 + packages/cli/src/context/scan/types.ts | 31 +- .../context/scan/warehouse-catalog.test.ts | 2 +- .../cli/src/context/scan/warehouse-catalog.ts | 4 +- ...=> backend-conformance.test-utils.test.ts} | 6 +- ...e.ts => backend-conformance.test-utils.ts} | 0 .../cli/src/context/search/discover.test.ts | 2 +- packages/cli/src/context/search/discover.ts | 18 +- packages/cli/src/context/search/index.ts | 44 -- .../search/pglite-owner-process.test.ts | 2 +- .../search/pglite-runtime-boundary.test.ts | 10 - .../src/context/search/pglite-spike.test.ts | 7 +- packages/cli/src/context/search/types.ts | 7 +- packages/cli/src/context/skills/index.ts | 2 - .../context/skills/skills-registry.service.ts | 2 +- .../src/context/sl/dictionary-search.test.ts | 2 +- .../cli/src/context/sl/dictionary-search.ts | 16 +- packages/cli/src/context/sl/index.ts | 44 -- .../cli/src/context/sl/local-query.test.ts | 4 +- packages/cli/src/context/sl/local-query.ts | 6 +- packages/cli/src/context/sl/local-sl.test.ts | 2 +- packages/cli/src/context/sl/local-sl.ts | 9 +- .../sl/pglite-sl-search-prototype.test.ts | 4 +- .../context/sl/pglite-sl-search-prototype.ts | 7 +- .../src/context/sl/semantic-layer.service.ts | 7 +- .../context/sl/sl-dictionary-profile.test.ts | 2 +- .../src/context/sl/sl-dictionary-profile.ts | 4 +- .../cli/src/context/sl/sl-search.service.ts | 5 +- .../cli/src/context/sl/sl-validator.port.ts | 2 +- .../sl/tools/base-semantic-layer.tool.ts | 7 +- packages/cli/src/context/sl/tools/index.ts | 11 - .../context/sl/tools/sl-discover.tool.test.ts | 5 +- .../src/context/sl/tools/sl-discover.tool.ts | 2 +- .../sl/tools/sl-edit-source.tool.test.ts | 5 +- .../context/sl/tools/sl-edit-source.tool.ts | 11 +- .../tools/sl-read-source.tool.session.test.ts | 5 +- .../context/sl/tools/sl-read-source.tool.ts | 2 +- .../context/sl/tools/sl-rollback.tool.test.ts | 5 +- .../src/context/sl/tools/sl-rollback.tool.ts | 3 +- .../context/sl/tools/sl-validate.tool.test.ts | 5 +- .../src/context/sl/tools/sl-validate.tool.ts | 4 +- .../sl/tools/sl-warehouse-validation.ts | 7 +- .../sl/tools/sl-write-source.tool.test.ts | 5 +- .../context/sl/tools/sl-write-source.tool.ts | 11 +- packages/cli/src/context/sl/types.ts | 6 - .../cli/src/context/sql-analysis/index.ts | 13 - .../cli/src/context/sql-analysis/ports.ts | 2 +- packages/cli/src/context/tools/base-tool.ts | 8 +- .../tools/context-candidate-write.tool.ts | 4 +- .../tools/context-evidence-search.tool.ts | 2 +- .../tools/context-evidence-tool-store.ts | 8 +- .../tools/context-evidence-tools.test.ts | 6 +- packages/cli/src/context/tools/index.ts | 45 -- .../cli/src/context/tools/tool-session.ts | 7 +- .../src/context/tools/touched-sl-sources.ts | 1 + packages/cli/src/context/wiki/index.ts | 37 - packages/cli/src/context/wiki/keys.ts | 2 +- .../context/wiki/knowledge-wiki.service.ts | 8 +- .../src/context/wiki/local-knowledge.test.ts | 2 +- .../cli/src/context/wiki/local-knowledge.ts | 10 +- packages/cli/src/context/wiki/ports.ts | 2 - packages/cli/src/context/wiki/tools/index.ts | 5 - .../wiki/tools/wiki-list-tags.tool.test.ts | 2 +- .../context/wiki/tools/wiki-list-tags.tool.ts | 2 +- .../context/wiki/tools/wiki-read.tool.test.ts | 5 +- .../src/context/wiki/tools/wiki-read.tool.ts | 4 +- .../wiki/tools/wiki-remove.tool.test.ts | 5 +- .../context/wiki/tools/wiki-remove.tool.ts | 5 +- .../context/wiki/tools/wiki-search.tool.ts | 2 +- .../wiki/tools/wiki-write.tool.test.ts | 5 +- .../src/context/wiki/tools/wiki-write.tool.ts | 6 +- packages/cli/src/context/wiki/types.ts | 5 - .../src/context/wiki/wiki-ref-validation.ts | 2 +- packages/cli/src/database-tree-picker.ts | 2 +- packages/cli/src/demo-metrics.test.ts | 2 +- packages/cli/src/demo-metrics.ts | 2 +- packages/cli/src/doctor.ts | 7 +- packages/cli/src/embedding-resolution.test.ts | 3 +- packages/cli/src/embedding-resolution.ts | 11 +- packages/cli/src/index.test.ts | 2 +- packages/cli/src/ingest-depth.ts | 2 +- .../cli/src/ingest-query-executor.test.ts | 4 +- packages/cli/src/ingest-query-executor.ts | 6 +- packages/cli/src/ingest-report-file.ts | 3 +- packages/cli/src/ingest-viz.test.ts | 7 +- packages/cli/src/ingest.test-utils.ts | 38 +- packages/cli/src/ingest.test.ts | 15 +- packages/cli/src/ingest.ts | 31 +- packages/cli/src/knowledge.test.ts | 6 +- packages/cli/src/knowledge.ts | 12 +- packages/cli/src/llm/index.ts | 31 - packages/cli/src/llm/model-health.ts | 2 +- packages/cli/src/llm/model-provider.ts | 2 +- packages/cli/src/llm/types.ts | 10 +- packages/cli/src/local-adapters.test.ts | 2 +- packages/cli/src/local-adapters.ts | 64 +- .../cli/src/local-scan-connectors.test.ts | 4 +- packages/cli/src/local-scan-connectors.ts | 18 +- packages/cli/src/managed-local-embeddings.ts | 2 +- packages/cli/src/managed-python-command.ts | 2 +- packages/cli/src/managed-python-http.ts | 17 +- packages/cli/src/mcp-http-server.ts | 2 +- packages/cli/src/mcp-server-factory.ts | 9 +- packages/cli/src/mcp-stdio-server.ts | 2 +- packages/cli/src/memory-flow-hud.tsx | 2 +- .../cli/src/memory-flow-interactive.test.ts | 2 +- packages/cli/src/memory-flow-interactive.ts | 13 +- packages/cli/src/memory-flow-tui.test.tsx | 2 +- packages/cli/src/memory-flow-tui.tsx | 16 +- packages/cli/src/notion-page-picker.ts | 6 +- packages/cli/src/public-ingest.test.ts | 2 +- packages/cli/src/public-ingest.ts | 5 +- packages/cli/src/runtime-requirements.test.ts | 2 +- packages/cli/src/runtime-requirements.ts | 6 +- packages/cli/src/scan.test.ts | 35 +- packages/cli/src/scan.ts | 11 +- packages/cli/src/setup-agents.test.ts | 2 +- packages/cli/src/setup-agents.ts | 8 +- packages/cli/src/setup-context.test.ts | 10 +- packages/cli/src/setup-context.ts | 10 +- .../cli/src/setup-database-context-depth.ts | 8 +- packages/cli/src/setup-databases.test.ts | 4 +- packages/cli/src/setup-databases.ts | 42 +- packages/cli/src/setup-embeddings.test.ts | 4 +- packages/cli/src/setup-embeddings.ts | 16 +- packages/cli/src/setup-models.test.ts | 4 +- packages/cli/src/setup-models.ts | 18 +- packages/cli/src/setup-project.test.ts | 4 +- packages/cli/src/setup-project.ts | 11 +- packages/cli/src/setup-runtime.test.ts | 3 +- packages/cli/src/setup-runtime.ts | 7 +- packages/cli/src/setup-sources-notion.test.ts | 12 +- packages/cli/src/setup-sources.test.ts | 10 +- packages/cli/src/setup-sources.ts | 40 +- packages/cli/src/setup.test.ts | 2 +- packages/cli/src/setup.ts | 12 +- packages/cli/src/sl.test.ts | 2 +- packages/cli/src/sl.ts | 23 +- packages/cli/src/source-mapping.ts | 37 +- packages/cli/src/sql.test.ts | 7 +- packages/cli/src/sql.ts | 6 +- packages/cli/src/standalone-smoke.test.ts | 2 +- packages/cli/src/status-project.test.ts | 3 +- packages/cli/src/status-project.ts | 23 +- packages/cli/src/text-ingest.test.ts | 4 +- packages/cli/src/text-ingest.ts | 6 +- scripts/relationship-benchmark-report.mjs | 8 +- 413 files changed, 1260 insertions(+), 8739 deletions(-) delete mode 100644 packages/cli/src/connectors/bigquery/index.ts delete mode 100644 packages/cli/src/connectors/clickhouse/index.ts delete mode 100644 packages/cli/src/connectors/mysql/index.ts delete mode 100644 packages/cli/src/connectors/postgres/index.ts delete mode 100644 packages/cli/src/connectors/snowflake/index.ts delete mode 100644 packages/cli/src/connectors/sqlite/index.ts delete mode 100644 packages/cli/src/connectors/sqlserver/index.ts delete mode 100644 packages/cli/src/context/agent/index.ts delete mode 100644 packages/cli/src/context/connections/index.ts delete mode 100644 packages/cli/src/context/core/index.ts delete mode 100644 packages/cli/src/context/daemon/index.ts delete mode 100644 packages/cli/src/context/index-sync/index.ts delete mode 100644 packages/cli/src/context/index.ts delete mode 100644 packages/cli/src/context/ingest/adapters/dbt-descriptions/match-tables.test.ts delete mode 100644 packages/cli/src/context/ingest/adapters/dbt-descriptions/match-tables.ts delete mode 100644 packages/cli/src/context/ingest/adapters/dbt-descriptions/merge-semantic-model-tables.test.ts delete mode 100644 packages/cli/src/context/ingest/adapters/dbt-descriptions/merge-semantic-model-tables.ts delete mode 100644 packages/cli/src/context/ingest/adapters/dbt-descriptions/to-description-updates.test.ts delete mode 100644 packages/cli/src/context/ingest/adapters/dbt-descriptions/to-description-updates.ts delete mode 100644 packages/cli/src/context/ingest/adapters/dbt-descriptions/to-metadata-updates.test.ts delete mode 100644 packages/cli/src/context/ingest/adapters/dbt-descriptions/to-metadata-updates.ts delete mode 100644 packages/cli/src/context/ingest/adapters/dbt-descriptions/to-relationship-updates.test.ts delete mode 100644 packages/cli/src/context/ingest/adapters/dbt-descriptions/to-relationship-updates.ts delete mode 100644 packages/cli/src/context/ingest/adapters/dbt-extraction-golden-parity.test.ts delete mode 100644 packages/cli/src/context/ingest/adapters/live-database/extracted-schema.test.ts delete mode 100644 packages/cli/src/context/ingest/adapters/live-database/extracted-schema.ts delete mode 100644 packages/cli/src/context/ingest/adapters/live-database/structural-sync.test.ts delete mode 100644 packages/cli/src/context/ingest/adapters/live-database/structural-sync.ts delete mode 100644 packages/cli/src/context/ingest/context-candidates/index.ts delete mode 100644 packages/cli/src/context/ingest/context-evidence/index.ts delete mode 100644 packages/cli/src/context/ingest/index.ts delete mode 100644 packages/cli/src/context/ingest/memory-flow/index.ts delete mode 100644 packages/cli/src/context/ingest/page-triage/index.ts rename packages/cli/src/context/ingest/tools/warehouse-verification/{index.ts => create-warehouse-verification-tools.ts} (81%) delete mode 100644 packages/cli/src/context/llm/index.ts delete mode 100644 packages/cli/src/context/mcp/index.ts delete mode 100644 packages/cli/src/context/memory/index.ts delete mode 100644 packages/cli/src/context/project/index.ts delete mode 100644 packages/cli/src/context/prompts/index.ts delete mode 100644 packages/cli/src/context/scan/index.ts delete mode 100644 packages/cli/src/context/scan/relationship-artifacts.test.ts delete mode 100644 packages/cli/src/context/scan/relationship-artifacts.ts delete mode 100644 packages/cli/src/context/scan/relationship-feedback-calibration.test.ts delete mode 100644 packages/cli/src/context/scan/relationship-feedback-calibration.ts delete mode 100644 packages/cli/src/context/scan/relationship-feedback-export.test.ts delete mode 100644 packages/cli/src/context/scan/relationship-feedback-export.ts delete mode 100644 packages/cli/src/context/scan/relationship-review-apply.test.ts delete mode 100644 packages/cli/src/context/scan/relationship-review-apply.ts delete mode 100644 packages/cli/src/context/scan/relationship-review-decisions.test.ts delete mode 100644 packages/cli/src/context/scan/relationship-review-decisions.ts delete mode 100644 packages/cli/src/context/scan/relationship-threshold-advice.test.ts delete mode 100644 packages/cli/src/context/scan/relationship-threshold-advice.ts rename packages/cli/src/context/search/{backend-conformance.test.ts => backend-conformance.test-utils.test.ts} (99%) rename packages/cli/src/context/search/{backend-conformance.ts => backend-conformance.test-utils.ts} (100%) delete mode 100644 packages/cli/src/context/search/index.ts delete mode 100644 packages/cli/src/context/skills/index.ts delete mode 100644 packages/cli/src/context/sl/index.ts delete mode 100644 packages/cli/src/context/sl/tools/index.ts delete mode 100644 packages/cli/src/context/sql-analysis/index.ts delete mode 100644 packages/cli/src/context/tools/index.ts delete mode 100644 packages/cli/src/context/wiki/index.ts delete mode 100644 packages/cli/src/context/wiki/tools/index.ts delete mode 100644 packages/cli/src/llm/index.ts diff --git a/knip.json b/knip.json index 6b692f62..08939c28 100644 --- a/knip.json +++ b/knip.json @@ -2,48 +2,41 @@ "$schema": "https://unpkg.com/knip@6/schema.json", "workspaces": { ".": { - "entry": ["scripts/**/*.mjs"], - "project": ["scripts/**/*.mjs"], - "ignoreDependencies": [ - "@semantic-release/commit-analyzer", - "@semantic-release/github", - "@semantic-release/npm", - "@semantic-release/release-notes-generator", - "conventional-changelog-conventionalcommits" + "entry": [ + "scripts/**/*.mjs", + "scripts/**/*.cjs", + ".releaserc.cjs!" ] }, "packages/cli": { "entry": [ - "src/llm/index.ts!", - "src/context/**/index.ts!", - "src/connectors/*/index.ts!", "src/print-command-tree.ts!", + "scripts/**/*.mjs", "src/**/*.test-utils.ts", - "src/**/acceptance-fixtures.ts" - ], - "project": ["src/**/*.{ts,tsx}!", "scripts/**/*.mjs", "vitest.config.ts"] + "src/**/acceptance-fixtures.ts", + "src/context/scan/relationship-benchmarks.ts!", + "src/context/scan/relationship-benchmark-report.ts!" + ] }, "docs-site": { "entry": [ "components/**/*.{ts,tsx}!", "source.config.ts!" ], - "project": [ - "app/**/*.{ts,tsx}!", - "components/**/*.{ts,tsx}!", - "lib/**/*.{ts,tsx}!", - "*.ts!", - "*.mjs", - "tests/**/*.mjs" - ], - "ignoreDependencies": ["tailwindcss"] + "ignoreDependencies": [ + "tailwindcss" + ] } }, - "ignore": [ - "**/dist/**", - "**/coverage/**", - "**/.next/**", - "**/node_modules/**" + "ignoreDependencies": [ + "@semantic-release/commit-analyzer", + "@semantic-release/github", + "@semantic-release/npm", + "@semantic-release/release-notes-generator", + "conventional-changelog-conventionalcommits" ], - "ignoreBinaries": ["uv", "lsof"] + "ignoreBinaries": [ + "uv", + "lsof" + ] } diff --git a/packages/cli/src/admin-reindex.test.ts b/packages/cli/src/admin-reindex.test.ts index 9bacca87..0802746a 100644 --- a/packages/cli/src/admin-reindex.test.ts +++ b/packages/cli/src/admin-reindex.test.ts @@ -1,6 +1,6 @@ import { createRequire } from 'node:module'; -import type { ReindexSummary } from './context/index-sync/index.js'; +import type { ReindexSummary } from './context/index-sync/types.js'; import { describe, expect, it, vi } from 'vitest'; import { renderReindexJson, renderReindexPlain, reindexHasErrors } from './admin-reindex.js'; import { runKtxCli } from './index.js'; diff --git a/packages/cli/src/admin-reindex.ts b/packages/cli/src/admin-reindex.ts index bbd7b2f6..82e3bbaf 100644 --- a/packages/cli/src/admin-reindex.ts +++ b/packages/cli/src/admin-reindex.ts @@ -1,6 +1,8 @@ -import { KtxIngestEmbeddingPortAdapter, type KtxEmbeddingPort } from './context/index.js'; -import { reindexLocalIndexes, type ReindexScopeResult, type ReindexSummary } from './context/index-sync/index.js'; -import { loadKtxProject } from './context/project/index.js'; +import { KtxIngestEmbeddingPortAdapter } from './context/llm/embedding-port.js'; +import type { KtxEmbeddingPort } from './context/core/embedding.js'; +import { reindexLocalIndexes } from './context/index-sync/reindex.js'; +import type { ReindexScopeResult, ReindexSummary } from './context/index-sync/types.js'; +import { loadKtxProject } from './context/project/project.js'; import { Option, type Command } from '@commander-js/extra-typings'; import { cancel, intro, log, note, outro } from '@clack/prompts'; import type { KtxCliCommandContext } from './cli-program.js'; diff --git a/packages/cli/src/admin.ts b/packages/cli/src/admin.ts index 0d653ff4..6c04f82f 100644 --- a/packages/cli/src/admin.ts +++ b/packages/cli/src/admin.ts @@ -50,7 +50,7 @@ export function registerAdminCommands(program: Command, context: KtxCliCommandCo .description('Print a JSON Schema describing ktx.yaml (for editors and LLM agents)') .option('--output ', 'Write the schema to a file instead of stdout') .action(async (options: { output?: string }) => { - const { generateKtxProjectConfigJsonSchema } = await import('./context/project/index.js'); + const { generateKtxProjectConfigJsonSchema } = await import('./context/project/config.js');; const json = `${JSON.stringify(generateKtxProjectConfigJsonSchema(), null, 2)}\n`; if (options.output) { const { writeFile } = await import('node:fs/promises'); diff --git a/packages/cli/src/claude-code-prompt-caching.ts b/packages/cli/src/claude-code-prompt-caching.ts index b502cfa6..a7c0fa54 100644 --- a/packages/cli/src/claude-code-prompt-caching.ts +++ b/packages/cli/src/claude-code-prompt-caching.ts @@ -1,4 +1,4 @@ -import type { KtxProjectLlmConfig } from './context/project/index.js'; +import type { KtxProjectLlmConfig } from './context/project/config.js'; const CLAUDE_CODE_IGNORED_PROMPT_CACHING_FIELDS = [ 'systemTtl', diff --git a/packages/cli/src/cli-runtime.ts b/packages/cli/src/cli-runtime.ts index c3469c2e..68089720 100644 --- a/packages/cli/src/cli-runtime.ts +++ b/packages/cli/src/cli-runtime.ts @@ -70,7 +70,7 @@ export function packageInfoFromJson(packageJson: unknown): KtxCliPackageInfo { } async function runInit(args: { projectDir: string; force: boolean }, io: KtxCliIo): Promise { - const { initKtxProject } = await import('./context/project/index.js'); + const { initKtxProject } = await import('./context/project/project.js');; const result = await initKtxProject({ projectDir: args.projectDir, force: args.force, diff --git a/packages/cli/src/connection.test.ts b/packages/cli/src/connection.test.ts index 0702587a..b05a6f16 100644 --- a/packages/cli/src/connection.test.ts +++ b/packages/cli/src/connection.test.ts @@ -1,9 +1,12 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import type { LookerClient, MetabaseRuntimeClient, NotionClient } from './context/ingest/index.js'; -import { initKtxProject, parseKtxProjectConfig, serializeKtxProjectConfig } from './context/project/index.js'; -import type { KtxConnectionDriver, KtxScanConnector } from './context/scan/index.js'; +import type { LookerClient } from './context/ingest/adapters/looker/client.js'; +import type { MetabaseRuntimeClient } from './context/ingest/adapters/metabase/client-port.js'; +import type { NotionClient } from './context/ingest/adapters/notion/notion-client.js'; +import { initKtxProject } from './context/project/project.js'; +import { parseKtxProjectConfig, serializeKtxProjectConfig } from './context/project/config.js'; +import type { KtxConnectionDriver, KtxScanConnector } from './context/scan/types.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runKtxConnection } from './connection.js'; diff --git a/packages/cli/src/connection.ts b/packages/cli/src/connection.ts index 6d46d960..174fe5ad 100644 --- a/packages/cli/src/connection.ts +++ b/packages/cli/src/connection.ts @@ -1,19 +1,15 @@ -import { - DEFAULT_METABASE_CLIENT_CONFIG, - DefaultLookerConnectionClientFactory, - DefaultMetabaseConnectionClientFactory, - type LookerClient, - type MetabaseRuntimeClient, - type NotionBotInfo, - NotionClient, - createLocalLookerCredentialResolver, - metabaseRuntimeConfigFromLocalConnection, - testRepoConnection, -} from './context/ingest/index.js'; -import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from './context/connections/index.js'; -import { resolveKtxConfigReference } from './context/core/index.js'; -import { type KtxLocalProject, loadKtxProject } from './context/project/index.js'; -import type { KtxScanConnector } from './context/scan/index.js'; +import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory } from './context/ingest/adapters/metabase/client.js'; +import { DefaultLookerConnectionClientFactory } from './context/ingest/adapters/looker/factory.js'; +import type { LookerClient } from './context/ingest/adapters/looker/client.js'; +import type { MetabaseRuntimeClient } from './context/ingest/adapters/metabase/client-port.js'; +import { type NotionBotInfo, NotionClient } from './context/ingest/adapters/notion/notion-client.js'; +import { createLocalLookerCredentialResolver } from './context/ingest/adapters/looker/local-looker.adapter.js'; +import { metabaseRuntimeConfigFromLocalConnection } from './context/ingest/adapters/metabase/local-metabase.adapter.js'; +import { testRepoConnection } from './context/ingest/repo-fetch.js'; +import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from './context/connections/notion-config.js'; +import { resolveKtxConfigReference } from './context/core/config-reference.js'; +import { type KtxLocalProject, loadKtxProject } from './context/project/project.js'; +import type { KtxScanConnector } from './context/scan/types.js'; import type { KtxCliIo } from './index.js'; import { bold, dim, green, red, SYMBOLS } from './io/symbols.js'; import { createKtxCliScanConnector } from './local-scan-connectors.js'; diff --git a/packages/cli/src/connectors/bigquery/connector.test.ts b/packages/cli/src/connectors/bigquery/connector.test.ts index 0de940df..5a890612 100644 --- a/packages/cli/src/connectors/bigquery/connector.test.ts +++ b/packages/cli/src/connectors/bigquery/connector.test.ts @@ -1,15 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { - bigQueryConnectionConfigFromConfig, - createBigQueryLiveDatabaseIntrospection, - isKtxBigQueryConnectionConfig, - type KtxBigQueryClient, - KtxBigQueryScanConnector, - type KtxBigQueryClientFactory, - type KtxBigQueryDataset, - type KtxBigQueryQueryJob, - type KtxBigQueryTableRef, -} from './index.js'; +import { bigQueryConnectionConfigFromConfig, isKtxBigQueryConnectionConfig, type KtxBigQueryClient, KtxBigQueryScanConnector, type KtxBigQueryClientFactory, type KtxBigQueryDataset, type KtxBigQueryQueryJob, type KtxBigQueryTableRef } from '../../connectors/bigquery/connector.js'; +import { createBigQueryLiveDatabaseIntrospection } from '../../connectors/bigquery/live-database-introspection.js'; function fakeClientFactory(): KtxBigQueryClientFactory { const queryResults = vi.fn(async (): ReturnType => [ diff --git a/packages/cli/src/connectors/bigquery/connector.ts b/packages/cli/src/connectors/bigquery/connector.ts index 42d3da38..1c6c964b 100644 --- a/packages/cli/src/connectors/bigquery/connector.ts +++ b/packages/cli/src/connectors/bigquery/connector.ts @@ -1,24 +1,6 @@ import { BigQuery, type TableField } from '@google-cloud/bigquery'; -import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/index.js'; -import { - createKtxConnectorCapabilities, - type KtxColumnSampleInput, - type KtxColumnSampleResult, - type KtxColumnStatsInput, - type KtxColumnStatsResult, - type KtxQueryResult, - type KtxReadOnlyQueryInput, - type KtxScanConnector, - type KtxScanContext, - type KtxScanInput, - type KtxSchemaColumn, - type KtxSchemaSnapshot, - type KtxSchemaTable, - type KtxTableListEntry, - type KtxTableRef, - type KtxTableSampleInput, - type KtxTableSampleResult, -} from '../../context/scan/index.js'; +import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; +import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js'; import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; @@ -57,12 +39,14 @@ export interface KtxBigQueryColumnDistinctValuesResult { cardinality: number; } +/** @internal */ export interface KtxBigQueryQueryJob { getQueryResults(): Promise< [Array>, unknown, { schema?: { fields?: TableField[] } }?, ...unknown[]] >; } +/** @internal */ export interface KtxBigQueryTableRef { id?: string; metadata?: { type?: string }; @@ -81,6 +65,7 @@ export interface KtxBigQueryTableRef { >; } +/** @internal */ export interface KtxBigQueryDataset { get(): Promise; getTables(): Promise<[KtxBigQueryTableRef[], ...unknown[]]>; @@ -223,6 +208,7 @@ export function isKtxBigQueryConnectionConfig( return String(connection?.driver ?? '').toLowerCase() === 'bigquery'; } +/** @internal */ export function bigQueryConnectionConfigFromConfig(input: { connectionId: string; connection: KtxBigQueryConnectionConfig | undefined; diff --git a/packages/cli/src/connectors/bigquery/dialect.ts b/packages/cli/src/connectors/bigquery/dialect.ts index 21a4848d..02d904ed 100644 --- a/packages/cli/src/connectors/bigquery/dialect.ts +++ b/packages/cli/src/connectors/bigquery/dialect.ts @@ -1,4 +1,4 @@ -import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/index.js'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type BigQueryTableNameRef = Pick & Partial>; diff --git a/packages/cli/src/connectors/bigquery/index.ts b/packages/cli/src/connectors/bigquery/index.ts deleted file mode 100644 index e4003530..00000000 --- a/packages/cli/src/connectors/bigquery/index.ts +++ /dev/null @@ -1,18 +0,0 @@ -export { KtxBigQueryDialect } from './dialect.js'; -export { - bigQueryConnectionConfigFromConfig, - isKtxBigQueryConnectionConfig, - KtxBigQueryScanConnector, - type KtxBigQueryClient, - type KtxBigQueryClientFactory, - type KtxBigQueryColumnDistinctValuesOptions, - type KtxBigQueryColumnDistinctValuesResult, - type KtxBigQueryConnectionConfig, - type KtxBigQueryDataset, - type KtxBigQueryQueryJob, - type KtxBigQueryReadOnlyQueryInput, - type KtxBigQueryResolvedConnectionConfig, - type KtxBigQueryScanConnectorOptions, - type KtxBigQueryTableRef, -} from './connector.js'; -export { createBigQueryLiveDatabaseIntrospection } from './live-database-introspection.js'; diff --git a/packages/cli/src/connectors/bigquery/live-database-introspection.ts b/packages/cli/src/connectors/bigquery/live-database-introspection.ts index dc17194c..5e854b9e 100644 --- a/packages/cli/src/connectors/bigquery/live-database-introspection.ts +++ b/packages/cli/src/connectors/bigquery/live-database-introspection.ts @@ -1,5 +1,5 @@ -import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/index.js'; -import type { KtxProjectConnectionConfig } from '../../context/project/index.js'; +import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; import { KtxBigQueryScanConnector, type KtxBigQueryClientFactory, diff --git a/packages/cli/src/connectors/clickhouse/connector.test.ts b/packages/cli/src/connectors/clickhouse/connector.test.ts index 4f8b7f52..6ff60299 100644 --- a/packages/cli/src/connectors/clickhouse/connector.test.ts +++ b/packages/cli/src/connectors/clickhouse/connector.test.ts @@ -1,11 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { - clickHouseClientConfigFromConfig, - createClickHouseLiveDatabaseIntrospection, - isKtxClickHouseConnectionConfig, - KtxClickHouseScanConnector, - type KtxClickHouseClientFactory, -} from './index.js'; +import { clickHouseClientConfigFromConfig, isKtxClickHouseConnectionConfig, KtxClickHouseScanConnector, type KtxClickHouseClientFactory } from '../../connectors/clickhouse/connector.js'; +import { createClickHouseLiveDatabaseIntrospection } from '../../connectors/clickhouse/live-database-introspection.js'; function result(payload: T) { return { diff --git a/packages/cli/src/connectors/clickhouse/connector.ts b/packages/cli/src/connectors/clickhouse/connector.ts index f8aa7eb2..714ccfb1 100644 --- a/packages/cli/src/connectors/clickhouse/connector.ts +++ b/packages/cli/src/connectors/clickhouse/connector.ts @@ -1,24 +1,6 @@ import { createClient } from '@clickhouse/client'; -import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/index.js'; -import { - createKtxConnectorCapabilities, - type KtxColumnSampleInput, - type KtxColumnSampleResult, - type KtxColumnStatsInput, - type KtxColumnStatsResult, - type KtxQueryResult, - type KtxReadOnlyQueryInput, - type KtxScanConnector, - type KtxScanContext, - type KtxScanInput, - type KtxSchemaColumn, - type KtxSchemaSnapshot, - type KtxSchemaTable, - type KtxTableRef, - type KtxTableSampleInput, - type KtxTableListEntry, - type KtxTableSampleResult, -} from '../../context/scan/index.js'; +import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; +import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js'; import { readFileSync } from 'node:fs'; import { Agent as HttpsAgent } from 'node:https'; import { homedir } from 'node:os'; @@ -198,6 +180,7 @@ export function isKtxClickHouseConnectionConfig( return String(connection?.driver ?? '').toLowerCase() === 'clickhouse'; } +/** @internal */ export function clickHouseClientConfigFromConfig(input: { connectionId: string; connection: KtxClickHouseConnectionConfig | undefined; diff --git a/packages/cli/src/connectors/clickhouse/dialect.ts b/packages/cli/src/connectors/clickhouse/dialect.ts index 48388ae0..48452ea6 100644 --- a/packages/cli/src/connectors/clickhouse/dialect.ts +++ b/packages/cli/src/connectors/clickhouse/dialect.ts @@ -1,4 +1,4 @@ -import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/index.js'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type ClickHouseTableNameRef = Pick & Partial>; diff --git a/packages/cli/src/connectors/clickhouse/index.ts b/packages/cli/src/connectors/clickhouse/index.ts deleted file mode 100644 index 106df237..00000000 --- a/packages/cli/src/connectors/clickhouse/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -export { KtxClickHouseDialect } from './dialect.js'; -export { - clickHouseClientConfigFromConfig, - isKtxClickHouseConnectionConfig, - KtxClickHouseScanConnector, - type KtxClickHouseClient, - type KtxClickHouseClientFactory, - type KtxClickHouseColumnDistinctValuesOptions, - type KtxClickHouseColumnDistinctValuesResult, - type KtxClickHouseConnectionConfig, - type KtxClickHouseEndpointResolver, - type KtxClickHouseReadOnlyQueryInput, - type KtxClickHouseResolvedClientConfig, - type KtxClickHouseScanConnectorOptions, -} from './connector.js'; -export { createClickHouseLiveDatabaseIntrospection } from './live-database-introspection.js'; diff --git a/packages/cli/src/connectors/clickhouse/live-database-introspection.ts b/packages/cli/src/connectors/clickhouse/live-database-introspection.ts index 36f9769c..1e0ec918 100644 --- a/packages/cli/src/connectors/clickhouse/live-database-introspection.ts +++ b/packages/cli/src/connectors/clickhouse/live-database-introspection.ts @@ -1,5 +1,5 @@ -import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/index.js'; -import type { KtxProjectConnectionConfig } from '../../context/project/index.js'; +import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; import { KtxClickHouseScanConnector, type KtxClickHouseClientFactory, diff --git a/packages/cli/src/connectors/mysql/connector.test.ts b/packages/cli/src/connectors/mysql/connector.test.ts index c5c5a3fa..64b576e2 100644 --- a/packages/cli/src/connectors/mysql/connector.test.ts +++ b/packages/cli/src/connectors/mysql/connector.test.ts @@ -1,12 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; import type { FieldPacket, RowDataPacket } from 'mysql2/promise'; -import { - createMysqlLiveDatabaseIntrospection, - isKtxMysqlConnectionConfig, - KtxMysqlScanConnector, - mysqlConnectionPoolConfigFromConfig, - type KtxMysqlPoolFactory, -} from './index.js'; +import { createMysqlLiveDatabaseIntrospection } from '../../connectors/mysql/live-database-introspection.js'; +import { isKtxMysqlConnectionConfig, KtxMysqlScanConnector, mysqlConnectionPoolConfigFromConfig, type KtxMysqlPoolFactory } from '../../connectors/mysql/connector.js'; function mysqlResult(rows: Record[], fields: Array<{ name: string; type?: number }>): [RowDataPacket[], FieldPacket[]] { return [rows as RowDataPacket[], fields as FieldPacket[]]; diff --git a/packages/cli/src/connectors/mysql/connector.ts b/packages/cli/src/connectors/mysql/connector.ts index 1cba7661..9cf242f5 100644 --- a/packages/cli/src/connectors/mysql/connector.ts +++ b/packages/cli/src/connectors/mysql/connector.ts @@ -2,27 +2,8 @@ import mysql, { type FieldPacket, type Pool, type RowDataPacket } from 'mysql2/p import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; -import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/index.js'; -import { - createKtxConnectorCapabilities, - type KtxColumnSampleInput, - type KtxColumnSampleResult, - type KtxColumnStatsInput, - type KtxColumnStatsResult, - type KtxQueryResult, - type KtxReadOnlyQueryInput, - type KtxScanConnector, - type KtxScanContext, - type KtxScanInput, - type KtxSchemaColumn, - type KtxTableListEntry, - type KtxSchemaForeignKey, - type KtxSchemaSnapshot, - type KtxSchemaTable, - type KtxTableRef, - type KtxTableSampleInput, - type KtxTableSampleResult, -} from '../../context/scan/index.js'; +import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; +import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxTableListEntry, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js'; import { KtxMysqlDialect } from './dialect.js'; export interface KtxMysqlConnectionConfig { @@ -237,6 +218,7 @@ export function isKtxMysqlConnectionConfig( return String(connection?.driver ?? '').toLowerCase() === 'mysql'; } +/** @internal */ export function mysqlConnectionPoolConfigFromConfig(input: { connectionId: string; connection: KtxMysqlConnectionConfig | undefined; diff --git a/packages/cli/src/connectors/mysql/dialect.ts b/packages/cli/src/connectors/mysql/dialect.ts index 1111f940..d61db36c 100644 --- a/packages/cli/src/connectors/mysql/dialect.ts +++ b/packages/cli/src/connectors/mysql/dialect.ts @@ -1,4 +1,4 @@ -import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/index.js'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type MysqlTableNameRef = Pick & Partial>; diff --git a/packages/cli/src/connectors/mysql/index.ts b/packages/cli/src/connectors/mysql/index.ts deleted file mode 100644 index ea5da962..00000000 --- a/packages/cli/src/connectors/mysql/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -export { KtxMysqlDialect } from './dialect.js'; -export { - isKtxMysqlConnectionConfig, - KtxMysqlScanConnector, - mysqlConnectionPoolConfigFromConfig, - type KtxMysqlColumnDistinctValuesOptions, - type KtxMysqlColumnDistinctValuesResult, - type KtxMysqlConnectionConfig, - type KtxMysqlEndpointResolver, - type KtxMysqlPoolConfig, - type KtxMysqlPoolFactory, - type KtxMysqlReadOnlyQueryInput, - type KtxMysqlScanConnectorOptions, -} from './connector.js'; -export { createMysqlLiveDatabaseIntrospection } from './live-database-introspection.js'; diff --git a/packages/cli/src/connectors/mysql/live-database-introspection.ts b/packages/cli/src/connectors/mysql/live-database-introspection.ts index e644a503..ea649761 100644 --- a/packages/cli/src/connectors/mysql/live-database-introspection.ts +++ b/packages/cli/src/connectors/mysql/live-database-introspection.ts @@ -1,5 +1,5 @@ -import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/index.js'; -import type { KtxProjectConnectionConfig } from '../../context/project/index.js'; +import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; import { KtxMysqlScanConnector, type KtxMysqlConnectionConfig, diff --git a/packages/cli/src/connectors/postgres/connector.test.ts b/packages/cli/src/connectors/postgres/connector.test.ts index 8093acda..cf595f5c 100644 --- a/packages/cli/src/connectors/postgres/connector.test.ts +++ b/packages/cli/src/connectors/postgres/connector.test.ts @@ -1,11 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { - createPostgresLiveDatabaseIntrospection, - isKtxPostgresConnectionConfig, - KtxPostgresScanConnector, - postgresPoolConfigFromConfig, - type KtxPostgresPoolFactory, -} from './index.js'; +import { createPostgresLiveDatabaseIntrospection } from '../../connectors/postgres/live-database-introspection.js'; +import { isKtxPostgresConnectionConfig, KtxPostgresScanConnector, postgresPoolConfigFromConfig, type KtxPostgresPoolFactory } from '../../connectors/postgres/connector.js'; interface FakeQueryResult { rows: Record[]; diff --git a/packages/cli/src/connectors/postgres/connector.ts b/packages/cli/src/connectors/postgres/connector.ts index 9b085076..36a2bda6 100644 --- a/packages/cli/src/connectors/postgres/connector.ts +++ b/packages/cli/src/connectors/postgres/connector.ts @@ -1,27 +1,8 @@ import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; -import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/index.js'; -import { - createKtxConnectorCapabilities, - type KtxColumnSampleInput, - type KtxColumnSampleResult, - type KtxColumnStatsInput, - type KtxColumnStatsResult, - type KtxQueryResult, - type KtxReadOnlyQueryInput, - type KtxScanConnector, - type KtxScanContext, - type KtxScanInput, - type KtxSchemaColumn, - type KtxSchemaForeignKey, - type KtxSchemaSnapshot, - type KtxSchemaTable, - type KtxTableListEntry, - type KtxTableRef, - type KtxTableSampleInput, - type KtxTableSampleResult, -} from '../../context/scan/index.js'; +import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; +import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js'; import { Pool } from 'pg'; import { KtxPostgresDialect } from './dialect.js'; @@ -297,6 +278,7 @@ export function isKtxPostgresConnectionConfig( return driver === 'postgres' || driver === 'postgresql'; } +/** @internal */ export function postgresPoolConfigFromConfig(input: { connectionId: string; connection: KtxPostgresConnectionConfig | undefined; diff --git a/packages/cli/src/connectors/postgres/dialect.ts b/packages/cli/src/connectors/postgres/dialect.ts index aca75bc6..ea0590b8 100644 --- a/packages/cli/src/connectors/postgres/dialect.ts +++ b/packages/cli/src/connectors/postgres/dialect.ts @@ -1,4 +1,4 @@ -import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/index.js'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type PostgresTableNameRef = Pick & Partial>; diff --git a/packages/cli/src/connectors/postgres/historic-sql-query-client.ts b/packages/cli/src/connectors/postgres/historic-sql-query-client.ts index 61e61a2c..58f0f8eb 100644 --- a/packages/cli/src/connectors/postgres/historic-sql-query-client.ts +++ b/packages/cli/src/connectors/postgres/historic-sql-query-client.ts @@ -1,4 +1,4 @@ -import type { KtxPostgresQueryClient } from '../../context/ingest/index.js'; +import type { KtxPostgresQueryClient } from '../../context/ingest/adapters/historic-sql/types.js'; import { KtxPostgresScanConnector, type KtxPostgresScanConnectorOptions } from './connector.js'; export type KtxPostgresHistoricSqlQueryClientOptions = KtxPostgresScanConnectorOptions; diff --git a/packages/cli/src/connectors/postgres/index.ts b/packages/cli/src/connectors/postgres/index.ts deleted file mode 100644 index b32e085d..00000000 --- a/packages/cli/src/connectors/postgres/index.ts +++ /dev/null @@ -1,21 +0,0 @@ -export { KtxPostgresDialect } from './dialect.js'; -export { - isKtxPostgresConnectionConfig, - KtxPostgresScanConnector, - postgresPoolConfigFromConfig, - type KtxPostgresColumnDistinctValuesOptions, - type KtxPostgresColumnDistinctValuesResult, - type KtxPostgresColumnStatisticsResult, - type KtxPostgresConnectionConfig, - type KtxPostgresEndpointResolver, - type KtxPostgresPoolConfig, - type KtxPostgresPoolFactory, - type KtxPostgresReadOnlyQueryInput, - type KtxPostgresScanConnectorOptions, - type KtxPostgresTableSampleResult, -} from './connector.js'; -export { - KtxPostgresHistoricSqlQueryClient, - type KtxPostgresHistoricSqlQueryClientOptions, -} from './historic-sql-query-client.js'; -export { createPostgresLiveDatabaseIntrospection } from './live-database-introspection.js'; diff --git a/packages/cli/src/connectors/postgres/live-database-introspection.ts b/packages/cli/src/connectors/postgres/live-database-introspection.ts index 894b2104..83e29489 100644 --- a/packages/cli/src/connectors/postgres/live-database-introspection.ts +++ b/packages/cli/src/connectors/postgres/live-database-introspection.ts @@ -1,5 +1,5 @@ -import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/index.js'; -import type { KtxProjectConnectionConfig } from '../../context/project/index.js'; +import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; import { KtxPostgresScanConnector, type KtxPostgresConnectionConfig, diff --git a/packages/cli/src/connectors/snowflake/connector.test.ts b/packages/cli/src/connectors/snowflake/connector.test.ts index a49be885..7b2e600f 100644 --- a/packages/cli/src/connectors/snowflake/connector.test.ts +++ b/packages/cli/src/connectors/snowflake/connector.test.ts @@ -1,12 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { - createSnowflakeLiveDatabaseIntrospection, - isKtxSnowflakeConnectionConfig, - KtxSnowflakeScanConnector, - snowflakeConnectionConfigFromConfig, - type KtxSnowflakeDriver, - type KtxSnowflakeDriverFactory, -} from './index.js'; +import { createSnowflakeLiveDatabaseIntrospection } from '../../connectors/snowflake/live-database-introspection.js'; +import { isKtxSnowflakeConnectionConfig, KtxSnowflakeScanConnector, snowflakeConnectionConfigFromConfig, type KtxSnowflakeDriver, type KtxSnowflakeDriverFactory } from '../../connectors/snowflake/connector.js'; function fakeDriverFactory(): KtxSnowflakeDriverFactory { const driver: KtxSnowflakeDriver = { diff --git a/packages/cli/src/connectors/snowflake/connector.ts b/packages/cli/src/connectors/snowflake/connector.ts index 08c029ce..5b0dfcca 100644 --- a/packages/cli/src/connectors/snowflake/connector.ts +++ b/packages/cli/src/connectors/snowflake/connector.ts @@ -2,26 +2,8 @@ import { createPrivateKey } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; -import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/index.js'; -import { - createKtxConnectorCapabilities, - type KtxColumnSampleInput, - type KtxColumnSampleResult, - type KtxColumnStatsInput, - type KtxColumnStatsResult, - type KtxQueryResult, - type KtxReadOnlyQueryInput, - type KtxScanConnector, - type KtxScanContext, - type KtxScanInput, - type KtxSchemaColumn, - type KtxSchemaSnapshot, - type KtxSchemaTable, - type KtxTableRef, - type KtxTableSampleInput, - type KtxTableListEntry, - type KtxTableSampleResult, -} from '../../context/scan/index.js'; +import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; +import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js'; import * as snowflake from 'snowflake-sdk'; import { KtxSnowflakeDialect } from './dialect.js'; @@ -196,6 +178,7 @@ export function isKtxSnowflakeConnectionConfig( return String(connection?.driver ?? '').toLowerCase() === 'snowflake'; } +/** @internal */ export function snowflakeConnectionConfigFromConfig(input: { connectionId: string; connection: KtxSnowflakeConnectionConfig | undefined; diff --git a/packages/cli/src/connectors/snowflake/dialect.ts b/packages/cli/src/connectors/snowflake/dialect.ts index c8c6cfbb..db508134 100644 --- a/packages/cli/src/connectors/snowflake/dialect.ts +++ b/packages/cli/src/connectors/snowflake/dialect.ts @@ -1,4 +1,4 @@ -import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/index.js'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type SnowflakeTableNameRef = Pick & Partial>; diff --git a/packages/cli/src/connectors/snowflake/index.ts b/packages/cli/src/connectors/snowflake/index.ts deleted file mode 100644 index ea96b0cb..00000000 --- a/packages/cli/src/connectors/snowflake/index.ts +++ /dev/null @@ -1,18 +0,0 @@ -export { KtxSnowflakeDialect } from './dialect.js'; -export { - isKtxSnowflakeConnectionConfig, - KtxSnowflakeScanConnector, - snowflakeConnectionConfigFromConfig, - type KtxSnowflakeColumnDistinctValuesOptions, - type KtxSnowflakeColumnDistinctValuesResult, - type KtxSnowflakeConnectionConfig, - type KtxSnowflakeDriver, - type KtxSnowflakeDriverFactory, - type KtxSnowflakeRawColumnMetadata, - type KtxSnowflakeRawTableMetadata, - type KtxSnowflakeReadOnlyQueryInput, - type KtxSnowflakeResolvedConnectionConfig, - type KtxSnowflakeScanConnectorOptions, - type KtxSnowflakeSdkOptionsProvider, -} from './connector.js'; -export { createSnowflakeLiveDatabaseIntrospection } from './live-database-introspection.js'; diff --git a/packages/cli/src/connectors/snowflake/live-database-introspection.ts b/packages/cli/src/connectors/snowflake/live-database-introspection.ts index 36ead436..58812c1a 100644 --- a/packages/cli/src/connectors/snowflake/live-database-introspection.ts +++ b/packages/cli/src/connectors/snowflake/live-database-introspection.ts @@ -1,5 +1,5 @@ -import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/index.js'; -import type { KtxProjectConnectionConfig } from '../../context/project/index.js'; +import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; import { KtxSnowflakeScanConnector, type KtxSnowflakeConnectionConfig, diff --git a/packages/cli/src/connectors/sqlite/connector.test.ts b/packages/cli/src/connectors/sqlite/connector.test.ts index c3c11d64..77ec4b3c 100644 --- a/packages/cli/src/connectors/sqlite/connector.test.ts +++ b/packages/cli/src/connectors/sqlite/connector.test.ts @@ -4,12 +4,8 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { - createSqliteLiveDatabaseIntrospection, - isKtxSqliteConnectionConfig, - KtxSqliteScanConnector, - sqliteDatabasePathFromConfig, -} from './index.js'; +import { createSqliteLiveDatabaseIntrospection } from '../../connectors/sqlite/live-database-introspection.js'; +import { isKtxSqliteConnectionConfig, KtxSqliteScanConnector, sqliteDatabasePathFromConfig } from '../../connectors/sqlite/connector.js'; describe('KtxSqliteScanConnector', () => { let tempDir: string; diff --git a/packages/cli/src/connectors/sqlite/connector.ts b/packages/cli/src/connectors/sqlite/connector.ts index 86e33f0f..e915c776 100644 --- a/packages/cli/src/connectors/sqlite/connector.ts +++ b/packages/cli/src/connectors/sqlite/connector.ts @@ -3,25 +3,9 @@ import { existsSync, readFileSync, statSync } from 'node:fs'; import { homedir } from 'node:os'; import { isAbsolute, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { assertReadOnlySql, limitSqlForExecution, normalizeQueryRows } from '../../context/connections/index.js'; -import { - createKtxConnectorCapabilities, - type KtxColumnSampleInput, - type KtxColumnSampleResult, - type KtxColumnStatsInput, - type KtxColumnStatsResult, - type KtxQueryResult, - type KtxReadOnlyQueryInput, - type KtxScanConnector, - type KtxScanContext, - type KtxScanInput, - type KtxSchemaForeignKey, - type KtxSchemaSnapshot, - type KtxSchemaTable, - type KtxTableRef, - type KtxTableSampleInput, - type KtxTableSampleResult, -} from '../../context/scan/index.js'; +import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; +import { normalizeQueryRows } from '../../context/connections/query-executor.js'; +import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js'; import { KtxSqliteDialect } from './dialect.js'; export interface KtxSqliteConnectionConfig { @@ -31,6 +15,7 @@ export interface KtxSqliteConnectionConfig { [key: string]: unknown; } +/** @internal */ export interface SqliteDatabasePathInput { connectionId: string; projectDir?: string; @@ -142,6 +127,7 @@ export function isKtxSqliteConnectionConfig( return driver === 'sqlite' || driver === 'sqlite3'; } +/** @internal */ export function sqliteDatabasePathFromConfig(input: SqliteDatabasePathInput): string { const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxSqliteConnectionConfig(input.connection)) { diff --git a/packages/cli/src/connectors/sqlite/dialect.ts b/packages/cli/src/connectors/sqlite/dialect.ts index ba4f6116..b5771b62 100644 --- a/packages/cli/src/connectors/sqlite/dialect.ts +++ b/packages/cli/src/connectors/sqlite/dialect.ts @@ -1,4 +1,4 @@ -import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/index.js'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type SqliteTableNameRef = Pick & Partial>; diff --git a/packages/cli/src/connectors/sqlite/index.ts b/packages/cli/src/connectors/sqlite/index.ts deleted file mode 100644 index 653a9e81..00000000 --- a/packages/cli/src/connectors/sqlite/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -export { KtxSqliteDialect } from './dialect.js'; -export { - isKtxSqliteConnectionConfig, - KtxSqliteScanConnector, - sqliteDatabasePathFromConfig, - type KtxSqliteColumnDistinctValuesOptions, - type KtxSqliteColumnDistinctValuesResult, - type KtxSqliteConnectionConfig, - type KtxSqliteReadOnlyQueryInput, - type KtxSqliteScanConnectorOptions, - type SqliteDatabasePathInput, -} from './connector.js'; -export { - createSqliteLiveDatabaseIntrospection, - type CreateSqliteLiveDatabaseIntrospectionOptions, -} from './live-database-introspection.js'; diff --git a/packages/cli/src/connectors/sqlite/live-database-introspection.ts b/packages/cli/src/connectors/sqlite/live-database-introspection.ts index d6e52d60..62a1f8c5 100644 --- a/packages/cli/src/connectors/sqlite/live-database-introspection.ts +++ b/packages/cli/src/connectors/sqlite/live-database-introspection.ts @@ -1,5 +1,5 @@ -import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/index.js'; -import type { KtxProjectConnectionConfig } from '../../context/project/index.js'; +import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; import { KtxSqliteScanConnector, type KtxSqliteConnectionConfig } from './connector.js'; export interface CreateSqliteLiveDatabaseIntrospectionOptions { diff --git a/packages/cli/src/connectors/sqlserver/connector.test.ts b/packages/cli/src/connectors/sqlserver/connector.test.ts index b7915fa8..bd9a8af1 100644 --- a/packages/cli/src/connectors/sqlserver/connector.test.ts +++ b/packages/cli/src/connectors/sqlserver/connector.test.ts @@ -1,12 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { - createSqlServerLiveDatabaseIntrospection, - isKtxSqlServerConnectionConfig, - KtxSqlServerScanConnector, - sqlServerConnectionPoolConfigFromConfig, - type KtxSqlServerPoolFactory, - type KtxSqlServerQueryResult, -} from './index.js'; +import { createSqlServerLiveDatabaseIntrospection } from '../../connectors/sqlserver/live-database-introspection.js'; +import { isKtxSqlServerConnectionConfig, KtxSqlServerScanConnector, sqlServerConnectionPoolConfigFromConfig, type KtxSqlServerPoolFactory, type KtxSqlServerQueryResult } from '../../connectors/sqlserver/connector.js'; function recordset>( rows: T[], diff --git a/packages/cli/src/connectors/sqlserver/connector.ts b/packages/cli/src/connectors/sqlserver/connector.ts index 5f754b21..d9c227d7 100644 --- a/packages/cli/src/connectors/sqlserver/connector.ts +++ b/packages/cli/src/connectors/sqlserver/connector.ts @@ -1,24 +1,5 @@ -import { assertReadOnlySql } from '../../context/connections/index.js'; -import { - createKtxConnectorCapabilities, - type KtxColumnSampleInput, - type KtxColumnSampleResult, - type KtxColumnStatsInput, - type KtxColumnStatsResult, - type KtxQueryResult, - type KtxReadOnlyQueryInput, - type KtxScanConnector, - type KtxScanContext, - type KtxScanInput, - type KtxSchemaColumn, - type KtxSchemaForeignKey, - type KtxSchemaSnapshot, - type KtxSchemaTable, - type KtxTableListEntry, - type KtxTableRef, - type KtxTableSampleInput, - type KtxTableSampleResult, -} from '../../context/scan/index.js'; +import { assertReadOnlySql } from '../../context/connections/read-only-sql.js'; +import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js'; import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; @@ -50,6 +31,7 @@ export interface KtxSqlServerPoolConfig { pool: { max: number; min: number; idleTimeoutMillis: number }; } +/** @internal */ export interface KtxSqlServerQueryResult { recordset?: Array> & { columns?: Record }; } @@ -239,6 +221,7 @@ export function isKtxSqlServerConnectionConfig( return String(connection?.driver ?? '').toLowerCase() === 'sqlserver'; } +/** @internal */ export function sqlServerConnectionPoolConfigFromConfig(input: { connectionId: string; connection: KtxSqlServerConnectionConfig | undefined; diff --git a/packages/cli/src/connectors/sqlserver/dialect.ts b/packages/cli/src/connectors/sqlserver/dialect.ts index a1db5adc..8444317d 100644 --- a/packages/cli/src/connectors/sqlserver/dialect.ts +++ b/packages/cli/src/connectors/sqlserver/dialect.ts @@ -1,4 +1,4 @@ -import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/index.js'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; type SqlServerTableNameRef = Pick & Partial>; diff --git a/packages/cli/src/connectors/sqlserver/index.ts b/packages/cli/src/connectors/sqlserver/index.ts deleted file mode 100644 index cbea4d95..00000000 --- a/packages/cli/src/connectors/sqlserver/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -export { KtxSqlServerDialect } from './dialect.js'; -export { - isKtxSqlServerConnectionConfig, - KtxSqlServerScanConnector, - sqlServerConnectionPoolConfigFromConfig, - type KtxSqlServerColumnDistinctValuesOptions, - type KtxSqlServerColumnDistinctValuesResult, - type KtxSqlServerConnectionConfig, - type KtxSqlServerEndpointResolver, - type KtxSqlServerPool, - type KtxSqlServerPoolConfig, - type KtxSqlServerPoolFactory, - type KtxSqlServerQueryResult, - type KtxSqlServerReadOnlyQueryInput, - type KtxSqlServerScanConnectorOptions, -} from './connector.js'; -export { createSqlServerLiveDatabaseIntrospection } from './live-database-introspection.js'; diff --git a/packages/cli/src/connectors/sqlserver/live-database-introspection.ts b/packages/cli/src/connectors/sqlserver/live-database-introspection.ts index bc4f229b..6bd54ba1 100644 --- a/packages/cli/src/connectors/sqlserver/live-database-introspection.ts +++ b/packages/cli/src/connectors/sqlserver/live-database-introspection.ts @@ -1,5 +1,5 @@ -import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/index.js'; -import type { KtxProjectConnectionConfig } from '../../context/project/index.js'; +import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; import { KtxSqlServerScanConnector, type KtxSqlServerConnectionConfig, diff --git a/packages/cli/src/context-build-view.test.ts b/packages/cli/src/context-build-view.test.ts index d0c665fb..c1550219 100644 --- a/packages/cli/src/context-build-view.test.ts +++ b/packages/cli/src/context-build-view.test.ts @@ -1,4 +1,4 @@ -import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/index.js'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; import { describe, expect, it, vi } from 'vitest'; import type { KtxPublicIngestProject, KtxPublicIngestTargetResult } from './public-ingest.js'; import { diff --git a/packages/cli/src/context-build-view.ts b/packages/cli/src/context-build-view.ts index adb5d459..c734c6b9 100644 --- a/packages/cli/src/context-build-view.ts +++ b/packages/cli/src/context-build-view.ts @@ -1,4 +1,4 @@ -import type { KtxProgressPort, KtxProgressUpdateOptions } from './context/scan/index.js'; +import type { KtxProgressPort, KtxProgressUpdateOptions } from './context/scan/types.js'; import type { KtxCliIo } from './index.js'; import type { KtxIngestProgressUpdate } from './ingest.js'; import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js'; diff --git a/packages/cli/src/context/agent/index.ts b/packages/cli/src/context/agent/index.ts deleted file mode 100644 index 8cc0a17c..00000000 --- a/packages/cli/src/context/agent/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export type { - AgentRunnerPort, - RunLoopParams, - RunLoopResult, - RunLoopStepInfo, - RunLoopStopReason, -} from '../llm/runtime-port.js'; -export { RuntimeAgentRunner } from '../llm/runtime-port.js'; -export type { AgentTelemetryPort } from '../llm/ai-sdk-runtime.js'; diff --git a/packages/cli/src/context/connections/dialects.ts b/packages/cli/src/context/connections/dialects.ts index afac4bd2..75a8ae4c 100644 --- a/packages/cli/src/context/connections/dialects.ts +++ b/packages/cli/src/context/connections/dialects.ts @@ -1,6 +1,6 @@ import type { KtxSchemaDimensionType, KtxTableRef } from '../scan/types.js'; -export type SupportedDriver = +type SupportedDriver = | 'postgres' | 'postgresql' | 'mysql' diff --git a/packages/cli/src/context/connections/index.ts b/packages/cli/src/context/connections/index.ts deleted file mode 100644 index 0917a7ca..00000000 --- a/packages/cli/src/context/connections/index.ts +++ /dev/null @@ -1,30 +0,0 @@ -export type { - KtxSqlQueryExecutionInput, - KtxSqlQueryExecutionResult, - KtxSqlQueryExecutorPort, -} from './query-executor.js'; -export type { KtxDialect, SupportedDriver } from './dialects.js'; -export { createDefaultLocalQueryExecutor, type DefaultLocalQueryExecutorOptions } from './local-query-executor.js'; -export { getDialectForDriver } from './dialects.js'; -export { normalizeQueryRows } from './query-executor.js'; -export { createPostgresQueryExecutor } from './postgres-query-executor.js'; -export { assertReadOnlySql, limitSqlForExecution } from './read-only-sql.js'; -export { createSqliteQueryExecutor, sqliteDatabasePathFromConnection } from './sqlite-query-executor.js'; -export { connectionTypeSchema, type ConnectionType } from './connection-type.js'; -export { - localConnectionInfoFromConfig, - localConnectionToWarehouseDescriptor, - localConnectionTypeForConfig, - type LocalConnectionInfo, - type LocalWarehouseDescriptor, -} from './local-warehouse-descriptor.js'; -export { - KTX_NOTION_ORG_KNOWLEDGE_WARNING, - notionConnectionToPullConfig, - parseNotionConnectionConfig, - redactNotionConnectionConfig, - resolveNotionConnectionAuthToken, - resolveNotionAuthToken, - type KtxNotionConnectionConfig, - type RedactedKtxNotionConnectionConfig, -} from './notion-config.js'; diff --git a/packages/cli/src/context/connections/notion-config.ts b/packages/cli/src/context/connections/notion-config.ts index 24dd5d4b..79ebe23d 100644 --- a/packages/cli/src/context/connections/notion-config.ts +++ b/packages/cli/src/context/connections/notion-config.ts @@ -8,7 +8,7 @@ import { } from '../ingest/adapters/notion/types.js'; import type { KtxProjectConnectionConfig } from '../project/config.js'; -export const KTX_NOTION_ORG_KNOWLEDGE_WARNING = +const KTX_NOTION_ORG_KNOWLEDGE_WARNING = 'Anything accessible to this Notion integration can become organization knowledge.'; type KtxNotionCrawlMode = 'all_accessible' | 'selected_roots'; @@ -39,6 +39,7 @@ export type KtxNotionConnectionConfig = Omit< max_knowledge_updates_per_run: number; }; +/** @internal */ export interface RedactedKtxNotionConnectionConfig { driver: 'notion'; hasAuthToken: boolean; @@ -152,6 +153,7 @@ export function parseNotionConnectionConfig(raw: unknown): KtxNotionConnectionCo }; } +/** @internal */ export function redactNotionConnectionConfig(config: KtxNotionConnectionConfig): RedactedKtxNotionConnectionConfig { return { driver: 'notion', @@ -171,6 +173,7 @@ function expandHome(path: string): string { return path === '~' || path.startsWith('~/') ? resolve(homedir(), path.slice(2)) : path; } +/** @internal */ export async function resolveNotionAuthToken( authTokenRef: string, options: ResolveNotionTokenOptions = {}, diff --git a/packages/cli/src/context/connections/query-executor.ts b/packages/cli/src/context/connections/query-executor.ts index d3ca68bc..e169d164 100644 --- a/packages/cli/src/context/connections/query-executor.ts +++ b/packages/cli/src/context/connections/query-executor.ts @@ -1,4 +1,4 @@ -import type { KtxProjectConnectionConfig } from '../project/index.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; export interface KtxSqlQueryExecutionInput { connectionId: string; diff --git a/packages/cli/src/context/connections/sqlite-query-executor.ts b/packages/cli/src/context/connections/sqlite-query-executor.ts index 2a87ef7d..22c69005 100644 --- a/packages/cli/src/context/connections/sqlite-query-executor.ts +++ b/packages/cli/src/context/connections/sqlite-query-executor.ts @@ -49,6 +49,7 @@ function sqlitePathFromUrl(url: string): string { return url; } +/** @internal */ export function sqliteDatabasePathFromConnection(input: KtxSqlQueryExecutionInput): string { const driver = connectionDriver(input); if (driver !== 'sqlite' && driver !== 'sqlite3') { diff --git a/packages/cli/src/context/core/config-reference.ts b/packages/cli/src/context/core/config-reference.ts index 7aa69e69..f1f235ca 100644 --- a/packages/cli/src/context/core/config-reference.ts +++ b/packages/cli/src/context/core/config-reference.ts @@ -2,6 +2,7 @@ import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; +/** @internal */ export function resolveKtxHomePath(path: string): string { if (path === '~') { return homedir(); diff --git a/packages/cli/src/context/core/config.ts b/packages/cli/src/context/core/config.ts index 21e5a277..d61d98d2 100644 --- a/packages/cli/src/context/core/config.ts +++ b/packages/cli/src/context/core/config.ts @@ -1,10 +1,10 @@ -export interface KtxStorageConfig { +interface KtxStorageConfig { configDir?: string; homeDir?: string; worktreesDir?: string; } -export interface KtxGitConfig { +interface KtxGitConfig { userName: string; userEmail: string; bootstrapMessage?: string; diff --git a/packages/cli/src/context/core/index.ts b/packages/cli/src/context/core/index.ts deleted file mode 100644 index 892ef98e..00000000 --- a/packages/cli/src/context/core/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -export type { KtxCoreConfig, KtxGitConfig, KtxLogger, KtxStorageConfig } from './config.js'; -export { noopLogger, resolveConfigDir, resolveWorktreesDir } from './config.js'; -export { resolveKtxConfigReference, resolveKtxHomePath } from './config-reference.js'; -export type { KtxEmbeddingPort } from './embedding.js'; -export { - REDACTED_KTX_CREDENTIAL_VALUE, - redactKtxSensitiveMetadata, - redactKtxSensitiveText, - redactKtxSensitiveValue, -} from './redaction.js'; -export type { - KtxFileHistoryEntry, - KtxFileListResult, - KtxFileReadResult, - KtxFileStorePort, - KtxFileWriteResult, -} from './file-store.js'; -export type { GitCommitInfo, SquashMergeResult, WorktreeEntry } from './git.service.js'; -export { GitService } from './git.service.js'; -export type { - SentinelPayload, - SessionOutcome, - SessionWorktree, - SessionWorktreeServiceDeps, - WorktreeConfigPort, -} from './session-worktree.service.js'; -export { SessionWorktreeService } from './session-worktree.service.js'; diff --git a/packages/cli/src/context/core/redaction.ts b/packages/cli/src/context/core/redaction.ts index 87241742..0ba5bab5 100644 --- a/packages/cli/src/context/core/redaction.ts +++ b/packages/cli/src/context/core/redaction.ts @@ -1,3 +1,4 @@ +/** @internal */ export const REDACTED_KTX_CREDENTIAL_VALUE = ''; const SENSITIVE_FIELD_NAME = /(password|secret|token|api[_-]?key|private[_-]?key|passphrase|credential|authorization|url)/i; diff --git a/packages/cli/src/context/core/session-worktree.service.ts b/packages/cli/src/context/core/session-worktree.service.ts index f001d58b..4667478b 100644 --- a/packages/cli/src/context/core/session-worktree.service.ts +++ b/packages/cli/src/context/core/session-worktree.service.ts @@ -5,7 +5,7 @@ import { GitService } from './git.service.js'; export type SessionOutcome = 'success' | 'empty' | 'conflict' | 'crash'; -export interface SentinelPayload { +interface SentinelPayload { outcome: SessionOutcome; at: string; chatId: string; diff --git a/packages/cli/src/context/daemon/index.ts b/packages/cli/src/context/daemon/index.ts deleted file mode 100644 index 0a218181..00000000 --- a/packages/cli/src/context/daemon/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './semantic-layer-compute.js'; diff --git a/packages/cli/src/context/daemon/semantic-layer-compute.ts b/packages/cli/src/context/daemon/semantic-layer-compute.ts index a48239eb..f416b169 100644 --- a/packages/cli/src/context/daemon/semantic-layer-compute.ts +++ b/packages/cli/src/context/daemon/semantic-layer-compute.ts @@ -4,21 +4,21 @@ import { URL } from 'node:url'; import { spawn } from 'node:child_process'; import type { ResolvedSemanticLayerSource, SemanticLayerQueryInput } from '../sl/types.js'; -export interface KtxSemanticLayerComputeQueryResult { +interface KtxSemanticLayerComputeQueryResult { sql: string; dialect: string; columns: Array>; plan: Record; } -export interface KtxSemanticLayerComputeValidationResult { +interface KtxSemanticLayerComputeValidationResult { valid: boolean; errors: string[]; warnings: string[]; perSourceWarnings: Record; } -export interface KtxSemanticLayerSourceGenerationColumnInput { +interface KtxSemanticLayerSourceGenerationColumnInput { name: string; type: string; primaryKey?: boolean; @@ -26,7 +26,7 @@ export interface KtxSemanticLayerSourceGenerationColumnInput { comment?: string | null; } -export interface KtxSemanticLayerSourceGenerationTableInput { +interface KtxSemanticLayerSourceGenerationTableInput { name: string; catalog?: string | null; db?: string | null; @@ -34,7 +34,7 @@ export interface KtxSemanticLayerSourceGenerationTableInput { columns: KtxSemanticLayerSourceGenerationColumnInput[]; } -export interface KtxSemanticLayerSourceGenerationLinkInput { +interface KtxSemanticLayerSourceGenerationLinkInput { fromTable: string; fromColumn: string; toTable: string; @@ -42,13 +42,13 @@ export interface KtxSemanticLayerSourceGenerationLinkInput { relationshipType: string; } -export interface KtxSemanticLayerSourceGenerationInput { +interface KtxSemanticLayerSourceGenerationInput { tables: KtxSemanticLayerSourceGenerationTableInput[]; links: KtxSemanticLayerSourceGenerationLinkInput[]; dialect?: string; } -export interface KtxSemanticLayerSourceGenerationResult { +interface KtxSemanticLayerSourceGenerationResult { sources: Array>; sourceCount: number; } @@ -75,14 +75,14 @@ export interface KtxSemanticLayerComputePort { generateSources(input: KtxSemanticLayerSourceGenerationInput): Promise; } -export type KtxDaemonCommand = 'semantic-query' | 'semantic-validate' | 'semantic-generate-sources'; +type KtxDaemonCommand = 'semantic-query' | 'semantic-validate' | 'semantic-generate-sources'; -export type KtxDaemonJsonRunner = ( +type KtxDaemonJsonRunner = ( subcommand: KtxDaemonCommand, payload: Record, ) => Promise>; -export type KtxDaemonHttpJsonRunner = (path: string, payload: Record) => Promise>; +type KtxDaemonHttpJsonRunner = (path: string, payload: Record) => Promise>; export interface PythonSemanticLayerComputeOptions { command?: string; @@ -92,6 +92,7 @@ export interface PythonSemanticLayerComputeOptions { runJson?: KtxDaemonJsonRunner; } +/** @internal */ export interface HttpSemanticLayerComputeOptions { baseUrl: string; requestJson?: KtxDaemonHttpJsonRunner; @@ -272,6 +273,7 @@ export function createPythonSemanticLayerComputePort( }; } +/** @internal */ export function createHttpSemanticLayerComputePort( options: HttpSemanticLayerComputeOptions, ): KtxSemanticLayerComputePort { diff --git a/packages/cli/src/context/index-sync/index.ts b/packages/cli/src/context/index-sync/index.ts deleted file mode 100644 index 5863688e..00000000 --- a/packages/cli/src/context/index-sync/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export type { ReindexOptions, ReindexScopeResult, ReindexSummary, ReindexWorkResult } from './types.js'; -export { discoverReindexScopes, reindexLocalIndexes } from './reindex.js'; diff --git a/packages/cli/src/context/index-sync/reindex.test.ts b/packages/cli/src/context/index-sync/reindex.test.ts index beb62342..90c6a178 100644 --- a/packages/cli/src/context/index-sync/reindex.test.ts +++ b/packages/cli/src/context/index-sync/reindex.test.ts @@ -2,8 +2,8 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import type { KtxEmbeddingPort } from '../core/index.js'; -import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../project/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { SqliteKnowledgeIndex } from '../wiki/sqlite-knowledge-index.js'; import { reindexLocalIndexes } from './reindex.js'; diff --git a/packages/cli/src/context/index-sync/reindex.ts b/packages/cli/src/context/index-sync/reindex.ts index d0cbe29a..14252a5e 100644 --- a/packages/cli/src/context/index-sync/reindex.ts +++ b/packages/cli/src/context/index-sync/reindex.ts @@ -1,8 +1,12 @@ import { readdir, stat } from 'node:fs/promises'; import { join, relative } from 'node:path'; -import { ktxLocalStateDbPath, type KtxLocalProject } from '../project/index.js'; -import { loadLocalSlSourceRecords, SlSearchService, SqliteSlSourcesIndex } from '../sl/index.js'; -import { KnowledgeWikiService, SqliteKnowledgeIndex } from '../wiki/index.js'; +import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { loadLocalSlSourceRecords } from '../../context/sl/local-sl.js'; +import { SlSearchService } from '../../context/sl/sl-search.service.js'; +import { SqliteSlSourcesIndex } from '../../context/sl/sqlite-sl-sources-index.js'; +import { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; +import { SqliteKnowledgeIndex } from '../../context/wiki/sqlite-knowledge-index.js'; import type { ReindexOptions, ReindexScopeResult, ReindexSummary, ReindexWorkResult } from './types.js'; type DiscoveredScope = @@ -41,7 +45,7 @@ async function childDirectories(path: string): Promise { } } -export async function discoverReindexScopes(project: KtxLocalProject): Promise { +async function discoverReindexScopes(project: KtxLocalProject): Promise { const scopes: DiscoveredScope[] = []; if (await directoryExists(join(project.projectDir, 'wiki/global'))) { scopes.push({ kind: 'wiki', scope: 'GLOBAL', scopeId: null, label: 'global' }); diff --git a/packages/cli/src/context/index-sync/types.ts b/packages/cli/src/context/index-sync/types.ts index 39970b57..5a038182 100644 --- a/packages/cli/src/context/index-sync/types.ts +++ b/packages/cli/src/context/index-sync/types.ts @@ -1,4 +1,4 @@ -import type { KtxEmbeddingPort } from '../core/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; export interface ReindexOptions { force: boolean; diff --git a/packages/cli/src/context/index.ts b/packages/cli/src/context/index.ts deleted file mode 100644 index 053711a9..00000000 --- a/packages/cli/src/context/index.ts +++ /dev/null @@ -1,128 +0,0 @@ -export * from './agent/index.js'; -export * from './core/index.js'; -export * from './daemon/index.js'; -export * from './ingest/index.js'; -export * from './index-sync/index.js'; -export * from './llm/index.js'; -export type { - CaptureSession, - CaptureSignals, - MemoryAgentInput, - MemoryAgentResult, - MemoryAgentServiceDeps, - MemoryAgentSettings, - MemoryAgentSourceType, - MemoryCommitMessagePort, - MemoryConnectionPort, - MemoryFileStorePort, - MemoryKnowledgeSlRefsPort, - MemoryLockPort, - MemorySlSourceReconcilerPort, - MemoryTelemetryPort, - MemoryToolSetLike, - MemoryToolsetFactoryPort, -} from './memory/index.js'; -export * from './project/index.js'; -export * from './prompts/index.js'; -export * from './search/index.js'; -export * from './sql-analysis/index.js'; -export type { - KtxColumnAnalysisResult, - KtxColumnDescriptionPromptInput, - KtxColumnEmbeddingForeignKeys, - KtxColumnEmbeddingTextInput, - KtxColumnSampleInput, - KtxColumnSampleResult, - KtxColumnSampleUpdate, - KtxColumnStatsInput, - KtxColumnStatsResult, - KtxConnectionDriver, - KtxConnectorCapabilities, - KtxCredentialEnvelope, - KtxCredentialEnvReference, - KtxCredentialFileReference, - KtxDataDictionaryColumnState, - KtxDataDictionarySampleDecision, - KtxDataDictionarySettings, - KtxDataDictionarySkipReason, - KtxDataSourceDescriptionPromptInput, - KtxDescriptionCachePort, - KtxDescriptionColumn, - KtxDescriptionColumnTable, - KtxDescriptionGenerationSettings, - KtxDescriptionGeneratorOptions, - KtxDescriptionSource, - KtxDescriptionTableInput, - KtxDescriptionUpdate, - KtxEmbeddingPort as KtxScanEmbeddingPort, - KtxEmbeddingUpdate, - KtxEnrichedColumn, - KtxEnrichedRelationship, - KtxEnrichedSchema, - KtxEnrichedTable, - KtxGenerateColumnDescriptionsInput, - KtxGenerateDataSourceDescriptionInput, - KtxGenerateTableDescriptionInput, - KtxOptionalConnectorCapabilities, - KtxProgressPort, - KtxQueryResult as KtxScanQueryResult, - KtxReadOnlyQueryInput, - KtxRelationshipEndpoint, - KtxRelationshipSource, - KtxRelationshipType, - KtxRelationshipUpdate, - KtxResolvedCredentialEnvelope, - KtxScanArtifactPaths, - KtxScanConnector, - KtxScanContext, - KtxScanDiffSummary, - KtxScanEnrichmentSummary, - KtxScanInput, - KtxScanLoggerPort, - KtxScanMetadataStore, - KtxScanMode, - KtxScanRelationshipSummary, - KtxScanReport, - KtxScanTrigger, - KtxScanWarning, - KtxScanWarningCode, - KtxSchemaColumn, - KtxSchemaDimensionType, - KtxSchemaForeignKey, - KtxSchemaScope, - KtxSchemaSnapshot, - KtxSchemaTable, - KtxSchemaTableKind, - KtxSkippedRelationship, - KtxStructuralSyncPlan, - KtxStructuralSyncStats, - KtxTableDescriptionPromptInput, - KtxTableRef, - KtxTableSampleInput, - KtxTableSampleResult, - KtxColumnTypeMapping, -} from './scan/index.js'; -export { - buildKtxColumnDescriptionPrompt, - buildKtxColumnEmbeddingText, - buildKtxDataSourceDescriptionPrompt, - buildKtxTableDescriptionPrompt, - createKtxConnectorCapabilities, - defaultKtxDataDictionarySettings, - inferKtxDimensionType, - isKtxDataDictionaryCandidate, - ktxColumnTypeMappingFromNative, - KtxDescriptionGenerator, - normalizeKtxNativeType, - REDACTED_KTX_CREDENTIAL_VALUE, - redactKtxCredentialEnvelope, - redactKtxCredentialValue, - redactKtxScanMetadata, - redactKtxScanReport, - redactKtxScanWarning, - shouldKtxSampleColumnForDictionary, -} from './scan/index.js'; -export * from './skills/index.js'; -export * from './sl/index.js'; -export * from './tools/index.js'; -export * from './wiki/index.js'; diff --git a/packages/cli/src/context/ingest/action-identity.ts b/packages/cli/src/context/ingest/action-identity.ts index d0111d09..18c25e27 100644 --- a/packages/cli/src/context/ingest/action-identity.ts +++ b/packages/cli/src/context/ingest/action-identity.ts @@ -1,4 +1,4 @@ -import type { MemoryAction } from '../memory/index.js'; +import type { MemoryAction } from '../../context/memory/types.js'; export function actionTargetConnectionId(action: MemoryAction, runConnectionId: string): string { return action.target === 'sl' ? (action.targetConnectionId ?? runConnectionId) : runConnectionId; diff --git a/packages/cli/src/context/ingest/adapters/dbt-descriptions/match-tables.test.ts b/packages/cli/src/context/ingest/adapters/dbt-descriptions/match-tables.test.ts deleted file mode 100644 index ce83e974..00000000 --- a/packages/cli/src/context/ingest/adapters/dbt-descriptions/match-tables.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { DbtParsedTable } from './parse-schema.js'; -import { findMatchingKtxTable, matchDbtTables, type DbtHostTableLite } from './match-tables.js'; - -const hostTables: DbtHostTableLite[] = [ - { id: '1', name: 'orders', catalog: 'warehouse', db: 'analytics', columns: [{ id: 'c1', name: 'id' }] }, - { id: '2', name: 'orders', catalog: 'warehouse', db: 'staging', columns: [{ id: 'c2', name: 'id' }] }, - { id: '3', name: 'customers', catalog: null, db: null, columns: [{ id: 'c3', name: 'id' }] }, -]; - -function table(input: Partial): DbtParsedTable { - return { - name: 'orders', - description: null, - database: null, - schema: null, - columns: [], - resourceType: 'model', - ...input, - }; -} - -describe('dbt descriptions table matching', () => { - it('uses schema plus name first and checks catalog when dbt database is present', () => { - expect( - findMatchingKtxTable(table({ database: 'warehouse', schema: 'analytics' }), hostTables, null)?.id, - ).toBe('1'); - }); - - it('does not fall back to name-only for source tables', () => { - expect(findMatchingKtxTable(table({ resourceType: 'source' }), hostTables, null)).toBeUndefined(); - }); - - it('uses targetSchema for models and name-only only when unique', () => { - expect(findMatchingKtxTable(table({ resourceType: 'model' }), hostTables, 'staging')?.id).toBe('2'); - expect(findMatchingKtxTable(table({ name: 'customers', resourceType: 'model' }), hostTables, null)?.id).toBe( - '3', - ); - expect(findMatchingKtxTable(table({ resourceType: 'model' }), hostTables, null)).toBeUndefined(); - }); - - it('summarizes matched columns and descriptions', () => { - const matches = matchDbtTables( - [ - table({ - name: 'customers', - description: 'Customers', - columns: [ - { name: 'id', description: 'Primary key', dataType: null }, - { name: 'missing', description: 'Missing', dataType: null }, - ], - }), - ], - hostTables, - null, - ); - - expect(matches).toEqual([ - { - dbtTable: 'customers', - dbtSchema: null, - dbtDatabase: null, - hostTableId: '3', - hostTableName: 'customers', - matched: true, - tableDescriptionAction: 'import', - tableDescriptionFound: true, - columnsToImport: 1, - columnsMatched: 1, - columnsTotal: 2, - columnDescriptionsFound: 1, - }, - ]); - }); -}); diff --git a/packages/cli/src/context/ingest/adapters/dbt-descriptions/match-tables.ts b/packages/cli/src/context/ingest/adapters/dbt-descriptions/match-tables.ts deleted file mode 100644 index ca8414ae..00000000 --- a/packages/cli/src/context/ingest/adapters/dbt-descriptions/match-tables.ts +++ /dev/null @@ -1,127 +0,0 @@ -import type { DbtParsedTable } from './parse-schema.js'; - -export interface DbtHostTableLite { - id: string; - name: string; - catalog: string | null; - db: string | null; - columns: Array<{ id: string; name: string }>; -} - -export interface DbtTableMatch { - dbtTable: string; - dbtSchema: string | null; - dbtDatabase: string | null; - hostTableId: string | null; - hostTableName: string | null; - matched: boolean; - tableDescriptionAction: 'skip' | 'import'; - tableDescriptionFound: boolean; - columnsToImport: number; - columnsMatched: number; - columnsTotal: number; - columnDescriptionsFound: number; -} - -export function matchDbtTables( - dbtTables: DbtParsedTable[], - hostTables: DbtHostTableLite[], - targetSchema?: string | null, -): DbtTableMatch[] { - return dbtTables.map((dbtTable) => { - const hostTable = findMatchingKtxTable(dbtTable, hostTables, targetSchema); - - if (!hostTable) { - return { - dbtTable: dbtTable.name, - dbtSchema: dbtTable.schema, - dbtDatabase: dbtTable.database, - hostTableId: null, - hostTableName: null, - matched: false, - tableDescriptionAction: 'skip', - tableDescriptionFound: Boolean(dbtTable.description), - columnsToImport: 0, - columnsMatched: 0, - columnsTotal: dbtTable.columns.length, - columnDescriptionsFound: dbtTable.columns.filter((column) => Boolean(column.description)).length, - }; - } - - const analysis = analyzeColumns(dbtTable, hostTable); - return { - dbtTable: dbtTable.name, - dbtSchema: dbtTable.schema, - dbtDatabase: dbtTable.database, - hostTableId: hostTable.id, - hostTableName: hostTable.name, - matched: true, - tableDescriptionAction: dbtTable.description ? 'import' : 'skip', - tableDescriptionFound: Boolean(dbtTable.description), - ...analysis, - }; - }); -} - -export function findMatchingKtxTable( - dbtTable: DbtParsedTable, - hostTables: DbtHostTableLite[], - targetSchema?: string | null, -): DbtHostTableLite | undefined { - const dbtName = dbtTable.name.toLowerCase(); - const effectiveSchema = dbtTable.schema ?? targetSchema ?? null; - - if (effectiveSchema) { - const strictMatch = hostTables.find((table) => { - const nameMatches = table.name.toLowerCase() === dbtName; - const schemaMatches = table.db?.toLowerCase() === effectiveSchema.toLowerCase(); - if (!nameMatches || !schemaMatches) { - return false; - } - if (dbtTable.database && table.catalog) { - return table.catalog.toLowerCase() === dbtTable.database.toLowerCase(); - } - return true; - }); - if (strictMatch) { - return strictMatch; - } - } - - if (dbtTable.resourceType === 'source') { - return undefined; - } - - const nameMatches = hostTables.filter((table) => table.name.toLowerCase() === dbtName); - return nameMatches.length === 1 ? nameMatches[0] : undefined; -} - -function analyzeColumns( - dbtTable: DbtParsedTable, - hostTable: DbtHostTableLite, -): Pick { - let columnsToImport = 0; - let columnsMatched = 0; - let columnDescriptionsFound = 0; - - for (const dbtColumn of dbtTable.columns) { - const hostColumn = hostTable.columns.find( - (column) => column.name.toLowerCase() === dbtColumn.name.toLowerCase(), - ); - if (!hostColumn) { - continue; - } - columnsMatched++; - if (dbtColumn.description) { - columnDescriptionsFound++; - columnsToImport++; - } - } - - return { - columnsToImport, - columnsMatched, - columnsTotal: dbtTable.columns.length, - columnDescriptionsFound, - }; -} diff --git a/packages/cli/src/context/ingest/adapters/dbt-descriptions/merge-semantic-model-tables.test.ts b/packages/cli/src/context/ingest/adapters/dbt-descriptions/merge-semantic-model-tables.test.ts deleted file mode 100644 index 9d1ec735..00000000 --- a/packages/cli/src/context/ingest/adapters/dbt-descriptions/merge-semantic-model-tables.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { ParsedSemanticModel } from '../metricflow/deep-parse.js'; -import { mergeSemanticModelTables } from './merge-semantic-model-tables.js'; -import type { DbtSchemaParseResult } from './parse-schema.js'; - -const semanticModel: ParsedSemanticModel = { - name: 'orders_semantic', - description: 'Order facts', - modelRef: 'fct_orders', - dimensions: [ - { name: 'status', column: 'status', type: 'categorical', description: 'Order status' }, - { name: 'ordered_at', column: 'ordered_at', type: 'time' }, - ], - measures: [], - entities: [], - defaultTimeDimension: null, -}; - -describe('mergeSemanticModelTables', () => { - it('adds missing MetricFlow model refs as dbt model tables', () => { - const input: DbtSchemaParseResult = { projectName: null, dbtVersion: null, tables: [], relationships: [] }; - - expect(mergeSemanticModelTables(input, [semanticModel])).toEqual({ - projectName: null, - dbtVersion: null, - relationships: [], - tables: [ - { - name: 'fct_orders', - description: 'Order facts', - database: null, - schema: null, - resourceType: 'model', - columns: [ - { name: 'status', description: 'Order status', dataType: null }, - { name: 'ordered_at', description: null, dataType: 'TIMESTAMP' }, - ], - }, - ], - }); - }); - - it('does not add a duplicate table when schema parsing already found the model ref', () => { - const input: DbtSchemaParseResult = { - projectName: null, - dbtVersion: null, - relationships: [], - tables: [ - { - name: 'FCT_ORDERS', - description: 'Existing', - database: null, - schema: null, - resourceType: 'model', - columns: [], - }, - ], - }; - - expect(mergeSemanticModelTables(input, [semanticModel]).tables).toHaveLength(1); - }); -}); diff --git a/packages/cli/src/context/ingest/adapters/dbt-descriptions/merge-semantic-model-tables.ts b/packages/cli/src/context/ingest/adapters/dbt-descriptions/merge-semantic-model-tables.ts deleted file mode 100644 index 2991153f..00000000 --- a/packages/cli/src/context/ingest/adapters/dbt-descriptions/merge-semantic-model-tables.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { ParsedSemanticModel } from '../metricflow/deep-parse.js'; -import type { DbtSchemaParseResult } from './parse-schema.js'; - -export function mergeSemanticModelTables( - parseResult: DbtSchemaParseResult, - semanticModels: ParsedSemanticModel[], -): DbtSchemaParseResult { - const merged: DbtSchemaParseResult = { - ...parseResult, - tables: [...parseResult.tables], - relationships: [...parseResult.relationships], - }; - const existingTableNames = new Set(merged.tables.map((table) => table.name.toLowerCase())); - - for (const model of semanticModels) { - const tableName = model.modelRef; - if (existingTableNames.has(tableName.toLowerCase())) { - continue; - } - - merged.tables.push({ - name: tableName, - description: model.description, - database: null, - schema: null, - columns: model.dimensions.map((dimension) => ({ - name: dimension.column, - description: dimension.description ?? null, - dataType: dimension.type === 'time' ? 'TIMESTAMP' : null, - })), - resourceType: 'model', - }); - existingTableNames.add(tableName.toLowerCase()); - } - - return merged; -} diff --git a/packages/cli/src/context/ingest/adapters/dbt-descriptions/parse-schema.ts b/packages/cli/src/context/ingest/adapters/dbt-descriptions/parse-schema.ts index a8f07a72..2fb7e7e3 100644 --- a/packages/cli/src/context/ingest/adapters/dbt-descriptions/parse-schema.ts +++ b/packages/cli/src/context/ingest/adapters/dbt-descriptions/parse-schema.ts @@ -1,9 +1,9 @@ import { createHash } from 'node:crypto'; import { parse as parseYaml } from 'yaml'; -import { type KtxLogger, noopLogger } from '../../../core/index.js'; +import { type KtxLogger, noopLogger } from '../../../../context/core/config.js'; import { resolveJinjaVariables } from '../../dbt-shared/project-vars.js'; -export interface DbtParsedColumn { +interface DbtParsedColumn { name: string; description: string | null; dataType: string | null; @@ -12,20 +12,20 @@ export interface DbtParsedColumn { enumValuesDbt?: string[]; } -export interface DbtDataTestRef { +interface DbtDataTestRef { name: string; package: string; kwargs?: Record; } -export interface DbtColumnConstraints { +interface DbtColumnConstraints { dbt: { not_null?: boolean; unique?: boolean; }; } -export interface DbtParsedRelationship { +interface DbtParsedRelationship { fromTable: string; fromColumn: string; toTable: string; @@ -35,7 +35,7 @@ export interface DbtParsedRelationship { description?: string; } -export interface DbtParsedTable { +interface DbtParsedTable { name: string; description: string | null; database: string | null; @@ -126,6 +126,7 @@ type DbtSchemaDataTest = [key: string]: unknown; }; +/** @internal */ export function parseDbtSchemaFile(content: string, options: ParseDbtSchemaOptions = {}): DbtSchemaParseResult { return new DbtSchemaParser(options.logger ?? noopLogger).parseFile(content, options); } @@ -138,13 +139,6 @@ export function parseDbtSchemaFiles( return new DbtSchemaParser(options.logger ?? noopLogger).parseFiles(files, variables, options.projectName ?? null); } -export function computeDbtSchemaHash(files: DbtSchemaFile[]): string { - const combined = [...files] - .sort((a, b) => a.path.localeCompare(b.path)) - .map((file) => `${file.path}:${file.content}`) - .join('\n'); - return createHash('sha256').update(combined).digest('hex').substring(0, 16); -} class DbtSchemaParser { constructor(private readonly logger: KtxLogger) {} diff --git a/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-description-updates.test.ts b/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-description-updates.test.ts deleted file mode 100644 index 0b083213..00000000 --- a/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-description-updates.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { DbtSchemaParseResult } from './parse-schema.js'; -import { toDescriptionUpdates } from './to-description-updates.js'; -import type { DbtHostTableLite } from './match-tables.js'; - -const hostTables: DbtHostTableLite[] = [ - { - id: '1', - name: 'orders', - catalog: 'warehouse', - db: 'analytics', - columns: [ - { id: 'c1', name: 'id' }, - { id: 'c2', name: 'amount' }, - ], - }, -]; - -function parseResult(description: string | null, columnDescription: string | null): DbtSchemaParseResult { - return { - projectName: null, - dbtVersion: null, - relationships: [], - tables: [ - { - name: 'orders', - description, - database: 'warehouse', - schema: 'analytics', - resourceType: 'model', - columns: [ - { name: 'id', description: columnDescription, dataType: null }, - { name: 'missing', description: 'not imported', dataType: null }, - ], - }, - ], - }; -} - -describe('dbt descriptions update payloads', () => { - it('emits dbt writes and matching ai invalidations when descriptions exist', () => { - expect( - toDescriptionUpdates({ - connectionId: 'conn-1', - parseResult: parseResult('Orders table', 'Primary key'), - hostTables, - targetSchema: null, - }), - ).toEqual({ - dbt: [ - { - connectionId: 'conn-1', - table: { catalog: 'warehouse', db: 'analytics', name: 'orders' }, - source: 'dbt', - tableDescription: 'Orders table', - columnDescriptions: { id: 'Primary key' }, - }, - ], - aiInvalidations: [ - { - connectionId: 'conn-1', - table: { catalog: 'warehouse', db: 'analytics', name: 'orders' }, - source: 'ai', - }, - ], - }); - }); - - it('does not emit spurious dbt writes or ai invalidations when no descriptions exist', () => { - expect( - toDescriptionUpdates({ - connectionId: 'conn-1', - parseResult: parseResult(null, null), - hostTables, - targetSchema: null, - }), - ).toEqual({ dbt: [], aiInvalidations: [] }); - }); - - it('emits ai invalidation without a dbt description write when only structural metadata exists', () => { - const result = parseResult(null, null); - result.tables[0]!.tagsDbt = ['finance']; - - expect( - toDescriptionUpdates({ - connectionId: 'conn-1', - parseResult: result, - hostTables, - targetSchema: null, - }), - ).toEqual({ - dbt: [], - aiInvalidations: [ - { - connectionId: 'conn-1', - table: { catalog: 'warehouse', db: 'analytics', name: 'orders' }, - source: 'ai', - }, - ], - }); - }); -}); diff --git a/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-description-updates.ts b/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-description-updates.ts deleted file mode 100644 index 70c895be..00000000 --- a/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-description-updates.ts +++ /dev/null @@ -1,70 +0,0 @@ -import type { KtxDescriptionUpdate } from '../../../scan/enrichment-types.js'; -import { findMatchingKtxTable, type DbtHostTableLite } from './match-tables.js'; -import type { DbtSchemaParseResult } from './parse-schema.js'; - -export interface DbtDescriptionUpdates { - dbt: KtxDescriptionUpdate[]; - aiInvalidations: KtxDescriptionUpdate[]; -} - -export function toDescriptionUpdates(input: { - connectionId: string; - parseResult: DbtSchemaParseResult; - hostTables: DbtHostTableLite[]; - targetSchema: string | null; -}): DbtDescriptionUpdates { - const dbt: KtxDescriptionUpdate[] = []; - const aiInvalidations: KtxDescriptionUpdate[] = []; - - for (const dbtTable of input.parseResult.tables) { - const hostTable = findMatchingKtxTable(dbtTable, input.hostTables, input.targetSchema); - if (!hostTable) { - continue; - } - - const tableDescription = dbtTable.description ?? undefined; - const columnDescriptions: Record = {}; - - for (const dbtColumn of dbtTable.columns) { - if (!dbtColumn.description) { - continue; - } - const hostColumn = hostTable.columns.find( - (column) => column.name.toLowerCase() === dbtColumn.name.toLowerCase(), - ); - if (hostColumn) { - columnDescriptions[hostColumn.name] = dbtColumn.description; - } - } - - const hasColumnDescriptions = Object.keys(columnDescriptions).length > 0; - const hasDescriptionChange = tableDescription !== undefined || hasColumnDescriptions; - const hasMetadataChange = - !!dbtTable.tagsDbt?.length || - dbtTable.freshnessDbt !== undefined || - dbtTable.columns.some( - (column) => column.constraints !== undefined || !!column.enumValuesDbt?.length || !!column.dataTests?.length, - ); - if (!hasDescriptionChange && !hasMetadataChange) { - continue; - } - - const tableRef = { catalog: hostTable.catalog, db: hostTable.db, name: hostTable.name }; - if (hasDescriptionChange) { - dbt.push({ - connectionId: input.connectionId, - table: tableRef, - source: 'dbt', - ...(tableDescription !== undefined ? { tableDescription } : {}), - ...(hasColumnDescriptions ? { columnDescriptions } : {}), - }); - } - aiInvalidations.push({ - connectionId: input.connectionId, - table: tableRef, - source: 'ai', - }); - } - - return { dbt, aiInvalidations }; -} diff --git a/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-metadata-updates.test.ts b/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-metadata-updates.test.ts deleted file mode 100644 index ee33b369..00000000 --- a/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-metadata-updates.test.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { toMetadataUpdates } from './to-metadata-updates.js'; - -describe('toMetadataUpdates', () => { - it('emits source-keyed dbt metadata updates for matched tables and columns', () => { - const updates = toMetadataUpdates({ - connectionId: 'conn_1', - targetSchema: 'analytics', - hostTables: [ - { - id: 'orders-id', - name: 'orders', - catalog: 'warehouse', - db: 'analytics', - columns: [ - { id: 'status-id', name: 'status' }, - { id: 'created-id', name: 'created_at' }, - ], - }, - ], - parseResult: { - projectName: null, - dbtVersion: null, - relationships: [], - tables: [ - { - name: 'orders', - description: null, - database: 'warehouse', - schema: 'analytics', - resourceType: 'model', - tagsDbt: ['finance'], - freshnessDbt: { loadedAtField: 'created_at' }, - columns: [ - { - name: 'status', - description: null, - dataType: null, - enumValuesDbt: ['placed', 'shipped'], - constraints: { dbt: { not_null: true } }, - dataTests: [{ name: 'accepted_values', package: 'dbt', kwargs: { values: ['placed', 'shipped'] } }], - }, - ], - }, - ], - }, - }); - - expect(updates).toEqual([ - { - connectionId: 'conn_1', - table: { catalog: 'warehouse', db: 'analytics', name: 'orders' }, - source: 'dbt', - tableFields: { - tags: ['finance'], - freshness: { loaded_at_field: 'created_at' }, - }, - columnFields: { - status: { - constraints: { not_null: true }, - enum_values: ['placed', 'shipped'], - tests: [ - { name: 'accepted_values', package: 'dbt', kwargs: { values: ['placed', 'shipped'] } }, - ], - }, - }, - }, - ]); - }); -}); diff --git a/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-metadata-updates.ts b/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-metadata-updates.ts deleted file mode 100644 index 6ebe4a8f..00000000 --- a/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-metadata-updates.ts +++ /dev/null @@ -1,74 +0,0 @@ -import type { KtxMetadataUpdate } from '../../../scan/enrichment-types.js'; -import { findMatchingKtxTable, type DbtHostTableLite } from './match-tables.js'; -import type { DbtSchemaParseResult } from './parse-schema.js'; - -export function toMetadataUpdates(input: { - connectionId: string; - parseResult: DbtSchemaParseResult; - hostTables: DbtHostTableLite[]; - targetSchema: string | null; -}): KtxMetadataUpdate[] { - const updates: KtxMetadataUpdate[] = []; - - for (const dbtTable of input.parseResult.tables) { - const hostTable = findMatchingKtxTable(dbtTable, input.hostTables, input.targetSchema); - if (!hostTable) { - continue; - } - - const tableFields: Record = {}; - if (dbtTable.tagsDbt?.length) { - tableFields.tags = dbtTable.tagsDbt; - } - if (dbtTable.freshnessDbt) { - tableFields.freshness = { - ...(dbtTable.freshnessDbt.raw !== undefined ? { raw: dbtTable.freshnessDbt.raw } : {}), - ...(dbtTable.freshnessDbt.loadedAtField !== undefined - ? { loaded_at_field: dbtTable.freshnessDbt.loadedAtField } - : {}), - }; - } - - const columnFields: Record> = {}; - for (const dbtColumn of dbtTable.columns) { - const hostColumn = hostTable.columns.find( - (column) => column.name.toLowerCase() === dbtColumn.name.toLowerCase(), - ); - if (!hostColumn) { - continue; - } - - const fields: Record = {}; - if (dbtColumn.constraints) { - fields.constraints = dbtColumn.constraints.dbt; - } - if (dbtColumn.enumValuesDbt?.length) { - fields.enum_values = dbtColumn.enumValuesDbt; - } - if (dbtColumn.dataTests?.length) { - fields.tests = dbtColumn.dataTests.map((test) => ({ - name: test.name, - package: test.package, - ...(test.kwargs ? { kwargs: test.kwargs } : {}), - })); - } - if (Object.keys(fields).length > 0) { - columnFields[hostColumn.name] = fields; - } - } - - if (Object.keys(tableFields).length === 0 && Object.keys(columnFields).length === 0) { - continue; - } - - updates.push({ - connectionId: input.connectionId, - table: { catalog: hostTable.catalog, db: hostTable.db, name: hostTable.name }, - source: 'dbt', - ...(Object.keys(tableFields).length > 0 ? { tableFields } : {}), - ...(Object.keys(columnFields).length > 0 ? { columnFields } : {}), - }); - } - - return updates; -} diff --git a/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-relationship-updates.test.ts b/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-relationship-updates.test.ts deleted file mode 100644 index 02fe5f63..00000000 --- a/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-relationship-updates.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { DbtHostTableLite } from './match-tables.js'; -import type { DbtSchemaParseResult } from './parse-schema.js'; -import { toRelationshipUpdates } from './to-relationship-updates.js'; - -const DBT_SYSTEM_EMAIL = ['system@kae', 'lio.dev'].join(''); - -const hostTables: DbtHostTableLite[] = [ - { - id: '1', - name: 'orders', - catalog: 'warehouse', - db: 'analytics', - columns: [{ id: 'c1', name: 'customer_id' }], - }, - { - id: '2', - name: 'customers', - catalog: 'warehouse', - db: 'staging', - columns: [{ id: 'c2', name: 'id' }], - }, -]; - -const parseResult: DbtSchemaParseResult = { - projectName: null, - dbtVersion: null, - tables: [], - relationships: [ - { - fromTable: 'orders', - fromColumn: 'customer_id', - toTable: 'customers', - toColumn: 'id', - fromSchema: 'analytics', - toSchema: 'analytics', - description: 'schema intentionally differs from the host customers table', - }, - { fromTable: 'orders', fromColumn: 'missing', toTable: 'customers', toColumn: 'id' }, - { fromTable: 'orders', fromColumn: 'customer_id', toTable: 'missing_table', toColumn: 'id' }, - ], -}; - -describe('dbt relationship update payloads', () => { - it('validates relationships using the current name-only matching behavior and dbt provenance', () => { - expect(toRelationshipUpdates({ connectionId: 'conn-1', parseResult, hostTables })).toEqual({ - joins: [ - { - connectionId: 'conn-1', - fromTable: 'orders', - fromColumns: ['customer_id'], - toTable: 'customers', - toColumns: ['id'], - relationship: 'many_to_one', - author: 'dbt', - authorEmail: DBT_SYSTEM_EMAIL, - }, - ], - skippedNoMatch: 2, - }); - }); -}); diff --git a/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-relationship-updates.ts b/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-relationship-updates.ts deleted file mode 100644 index 9f649434..00000000 --- a/packages/cli/src/context/ingest/adapters/dbt-descriptions/to-relationship-updates.ts +++ /dev/null @@ -1,57 +0,0 @@ -import type { KtxJoinUpdate } from '../../../scan/enrichment-types.js'; -import type { DbtHostTableLite } from './match-tables.js'; -import type { DbtSchemaParseResult } from './parse-schema.js'; - -export interface DbtRelationshipUpdates { - joins: KtxJoinUpdate[]; - skippedNoMatch: number; -} - -const DBT_SYSTEM_EMAIL = ['system@kae', 'lio.dev'].join(''); - -export function toRelationshipUpdates(input: { - connectionId: string; - parseResult: DbtSchemaParseResult; - hostTables: DbtHostTableLite[]; -}): DbtRelationshipUpdates { - const tablesByName = new Map(); - for (const table of input.hostTables) { - tablesByName.set(table.name.toLowerCase(), table); - } - - const joins: KtxJoinUpdate[] = []; - let skippedNoMatch = 0; - - for (const relationship of input.parseResult.relationships) { - const fromTable = tablesByName.get(relationship.fromTable.toLowerCase()); - const toTable = tablesByName.get(relationship.toTable.toLowerCase()); - if (!fromTable || !toTable) { - skippedNoMatch++; - continue; - } - - const fromColumn = fromTable.columns.find( - (column) => column.name.toLowerCase() === relationship.fromColumn.toLowerCase(), - ); - const toColumn = toTable.columns.find( - (column) => column.name.toLowerCase() === relationship.toColumn.toLowerCase(), - ); - if (!fromColumn || !toColumn) { - skippedNoMatch++; - continue; - } - - joins.push({ - connectionId: input.connectionId, - fromTable: fromTable.name, - fromColumns: [fromColumn.name], - toTable: toTable.name, - toColumns: [toColumn.name], - relationship: 'many_to_one', - author: 'dbt', - authorEmail: DBT_SYSTEM_EMAIL, - }); - } - - return { joins, skippedNoMatch }; -} diff --git a/packages/cli/src/context/ingest/adapters/dbt-extraction-golden-parity.test.ts b/packages/cli/src/context/ingest/adapters/dbt-extraction-golden-parity.test.ts deleted file mode 100644 index ee96377c..00000000 --- a/packages/cli/src/context/ingest/adapters/dbt-extraction-golden-parity.test.ts +++ /dev/null @@ -1,410 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { type DbtHostTableLite, matchDbtTables } from './dbt-descriptions/match-tables.js'; -import { mergeSemanticModelTables } from './dbt-descriptions/merge-semantic-model-tables.js'; -import { parseDbtSchemaFiles } from './dbt-descriptions/parse-schema.js'; -import { toDescriptionUpdates } from './dbt-descriptions/to-description-updates.js'; -import { toRelationshipUpdates } from './dbt-descriptions/to-relationship-updates.js'; -import { parseMetricflowFiles } from './metricflow/deep-parse.js'; -import { mapCrossModelMetricToSource, mapSemanticModelToSource } from './metricflow/semantic-models.js'; - -const DBT_SYSTEM_EMAIL = ['system@kae', 'lio.dev'].join(''); - -const metricflowYaml = ` -semantic_models: - - name: orders_semantic - description: MetricFlow order facts - model: ref('fct_orders') - defaults: - agg_time_dimension: ordered_at - entities: - - name: customer - type: foreign - expr: customer_id - description: Customer relationship - dimensions: - - name: status - type: categorical - expr: status - description: Order status - - name: ordered_at - type: time - expr: ordered_at - measures: - - name: total_revenue - agg: sum - expr: amount - description: Revenue - - name: customers_semantic - description: Customer dimension - model: ref('dim_customers') - entities: - - name: customer - type: primary - expr: id - dimensions: - - name: country - type: categorical - expr: country - description: Customer country - measures: - - name: customer_count - agg: count - expr: id - description: Customer count -metrics: - - name: total_revenue - type: simple - type_params: - measure: total_revenue - - name: customer_count - type: simple - type_params: - measure: customer_count - - name: revenue_per_customer - description: Revenue per customer - type: derived - type_params: - expr: total_revenue / NULLIF(customer_count, 0) - metrics: - - name: total_revenue - alias: total_revenue - - name: customer_count - alias: customer_count -`; - -const schemaYaml = ` -version: 2 -sources: - - name: raw - database: warehouse - schema: landing - tables: - - name: customers - identifier: dim_customers - description: Raw customer dimension - columns: - - name: id - description: Customer primary key - - name: country - description: Country name -models: - - name: "{{ var('orders_model', 'fct_orders') }}" - schema: "{{ var('mart_schema', 'analytics') }}" - description: Modeled orders - columns: - - name: customer_id - description: Linked customer id - tests: - - relationships: - to: ref('dim_customers') - field: id - - name: status - description: Order status - - name: amount - description: Gross amount -`; - -const hostTables: DbtHostTableLite[] = [ - { - id: 'orders-table', - name: 'fct_orders', - catalog: 'warehouse', - db: 'analytics', - columns: [ - { id: 'orders-customer-id', name: 'customer_id' }, - { id: 'orders-status', name: 'status' }, - { id: 'orders-amount', name: 'amount' }, - { id: 'orders-ordered-at', name: 'ordered_at' }, - ], - }, - { - id: 'customers-table', - name: 'dim_customers', - catalog: 'warehouse', - db: 'landing', - columns: [ - { id: 'customers-id', name: 'id' }, - { id: 'customers-country', name: 'country' }, - ], - }, -]; - -describe('dbt extraction golden parity fixture', () => { - it('freezes the relocated MetricFlow and dbt-description contract together', () => { - const metricflow = parseMetricflowFiles([{ path: 'semantic_models/orders.yml', content: metricflowYaml }]); - - expect(metricflow).toEqual({ - semanticModels: [ - { - name: 'orders_semantic', - description: 'MetricFlow order facts', - modelRef: 'fct_orders', - dimensions: [ - { - name: 'status', - column: 'status', - type: 'string', - label: 'Status', - description: 'Order status', - }, - { - name: 'ordered_at', - column: 'ordered_at', - type: 'time', - label: 'Ordered At', - description: undefined, - }, - ], - measures: [ - { - type: 'simple', - name: 'total_revenue', - column: 'amount', - aggregation: 'sum', - label: 'Total Revenue', - description: 'Revenue', - }, - ], - entities: [{ name: 'customer', type: 'foreign', expr: 'customer_id', description: 'Customer relationship' }], - defaultTimeDimension: 'ordered_at', - }, - { - name: 'customers_semantic', - description: 'Customer dimension', - modelRef: 'dim_customers', - dimensions: [ - { - name: 'country', - column: 'country', - type: 'string', - label: 'Country', - description: 'Customer country', - }, - ], - measures: [ - { - type: 'simple', - name: 'customer_count', - column: 'id', - aggregation: 'count', - label: 'Customer Count', - description: 'Customer count', - }, - ], - entities: [{ name: 'customer', type: 'primary', expr: 'id' }], - defaultTimeDimension: null, - }, - ], - crossModelMetrics: [ - { - name: 'revenue_per_customer', - label: null, - description: 'Revenue per customer', - type: 'derived', - expr: 'total_revenue / NULLIF(customer_count, 0)', - dependsOn: [ - { metricName: 'orders_semantic', alias: 'total_revenue' }, - { metricName: 'customers_semantic', alias: 'customer_count' }, - ], - filter: null, - }, - ], - relationships: [ - { - fromTable: 'fct_orders', - fromColumn: 'customer_id', - toTable: 'dim_customers', - toColumn: 'id', - description: 'Customer relationship', - }, - ], - warnings: [], - }); - - expect(mapSemanticModelToSource(metricflow.semanticModels[0], 'analytics.fct_orders')).toEqual({ - name: 'fct-orders', - table: 'analytics.fct_orders', - grain: ['status', 'ordered_at'], - columns: [ - { name: 'status', type: 'string', description: 'Order status' }, - { name: 'ordered_at', type: 'time' }, - ], - measures: [ - { - name: 'total_revenue', - expr: 'sum(amount)', - description: 'Revenue', - }, - ], - joins: [], - descriptions: { dbt: 'MetricFlow order facts' }, - }); - - expect(mapCrossModelMetricToSource(metricflow.crossModelMetrics[0])).toEqual({ - name: 'revenue-per-customer', - sql: 'total_revenue / NULLIF(customer_count, 0)', - descriptions: { dbt: 'Revenue per customer' }, - grain: [], - columns: [], - measures: [ - { - name: 'revenue_per_customer', - expr: 'total_revenue / NULLIF(customer_count, 0)', - description: 'Revenue per customer', - }, - ], - joins: [], - }); - - const schema = parseDbtSchemaFiles( - [{ path: 'models/schema.yml', content: schemaYaml }], - new Map([ - ['orders_model', 'fct_orders'], - ['mart_schema', 'analytics'], - ]), - ); - const merged = mergeSemanticModelTables(schema, metricflow.semanticModels); - - expect(merged).toEqual({ - projectName: null, - dbtVersion: null, - tables: [ - { - name: 'dim_customers', - description: 'Raw customer dimension', - database: 'warehouse', - schema: 'landing', - columns: [ - { name: 'id', description: 'Customer primary key', dataType: null }, - { name: 'country', description: 'Country name', dataType: null }, - ], - resourceType: 'source', - }, - { - name: 'fct_orders', - description: 'Modeled orders', - database: null, - schema: 'analytics', - columns: [ - { - name: 'customer_id', - description: 'Linked customer id', - dataType: null, - dataTests: [ - { - name: 'relationships', - package: 'dbt', - kwargs: { to: "ref('dim_customers')", field: 'id' }, - }, - ], - }, - { name: 'status', description: 'Order status', dataType: null }, - { name: 'amount', description: 'Gross amount', dataType: null }, - ], - resourceType: 'model', - }, - ], - relationships: [ - { - fromTable: 'fct_orders', - fromColumn: 'customer_id', - toTable: 'dim_customers', - toColumn: 'id', - fromSchema: 'analytics', - }, - ], - }); - - expect(matchDbtTables(merged.tables, hostTables, 'analytics')).toEqual([ - { - dbtTable: 'dim_customers', - dbtSchema: 'landing', - dbtDatabase: 'warehouse', - hostTableId: 'customers-table', - hostTableName: 'dim_customers', - matched: true, - tableDescriptionAction: 'import', - tableDescriptionFound: true, - columnsToImport: 2, - columnsMatched: 2, - columnsTotal: 2, - columnDescriptionsFound: 2, - }, - { - dbtTable: 'fct_orders', - dbtSchema: 'analytics', - dbtDatabase: null, - hostTableId: 'orders-table', - hostTableName: 'fct_orders', - matched: true, - tableDescriptionAction: 'import', - tableDescriptionFound: true, - columnsToImport: 3, - columnsMatched: 3, - columnsTotal: 3, - columnDescriptionsFound: 3, - }, - ]); - - expect( - toDescriptionUpdates({ - connectionId: 'warehouse-1', - parseResult: merged, - hostTables, - targetSchema: 'analytics', - }), - ).toEqual({ - dbt: [ - { - connectionId: 'warehouse-1', - table: { catalog: 'warehouse', db: 'landing', name: 'dim_customers' }, - source: 'dbt', - tableDescription: 'Raw customer dimension', - columnDescriptions: { - id: 'Customer primary key', - country: 'Country name', - }, - }, - { - connectionId: 'warehouse-1', - table: { catalog: 'warehouse', db: 'analytics', name: 'fct_orders' }, - source: 'dbt', - tableDescription: 'Modeled orders', - columnDescriptions: { - customer_id: 'Linked customer id', - status: 'Order status', - amount: 'Gross amount', - }, - }, - ], - aiInvalidations: [ - { - connectionId: 'warehouse-1', - table: { catalog: 'warehouse', db: 'landing', name: 'dim_customers' }, - source: 'ai', - }, - { - connectionId: 'warehouse-1', - table: { catalog: 'warehouse', db: 'analytics', name: 'fct_orders' }, - source: 'ai', - }, - ], - }); - - expect(toRelationshipUpdates({ connectionId: 'warehouse-1', parseResult: merged, hostTables })).toEqual({ - joins: [ - { - connectionId: 'warehouse-1', - fromTable: 'fct_orders', - fromColumns: ['customer_id'], - toTable: 'dim_customers', - toColumns: ['id'], - relationship: 'many_to_one', - author: 'dbt', - authorEmail: DBT_SYSTEM_EMAIL, - }, - ], - skippedNoMatch: 0, - }); - }); -}); diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/evidence.ts b/packages/cli/src/context/ingest/adapters/historic-sql/evidence.ts index 18c01a85..ddf26aed 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/evidence.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/evidence.ts @@ -9,6 +9,7 @@ function safeEvidenceSegment(value: string): string { return segment; } +/** @internal */ export const historicSqlTableUsageEvidenceSchema = z.object({ kind: z.literal('table_usage'), connectionId: z.string().min(1), @@ -16,15 +17,14 @@ export const historicSqlTableUsageEvidenceSchema = z.object({ rawPath: z.string().min(1), usage: tableUsageOutputSchema, }); -export type HistoricSqlTableUsageEvidence = z.infer; +/** @internal */ export const historicSqlPatternEvidenceSchema = z.object({ kind: z.literal('pattern'), connectionId: z.string().min(1), rawPath: z.string().min(1), pattern: patternOutputSchema, }); -export type HistoricSqlPatternEvidence = z.infer; export const historicSqlEvidenceEnvelopeSchema = z.discriminatedUnion('kind', [ historicSqlTableUsageEvidenceSchema, diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/historic-sql.adapter.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/historic-sql.adapter.test.ts index 3b5697d0..80df5c26 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/historic-sql.adapter.test.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/historic-sql.adapter.test.ts @@ -2,7 +2,7 @@ import { mkdtemp } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import type { SqlAnalysisPort } from '../../../sql-analysis/index.js'; +import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js'; import type { SourceAdapter } from '../../types.js'; import { HistoricSqlSourceAdapter } from './historic-sql.adapter.js'; import type { HistoricSqlReader } from './types.js'; diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts index 971a05ec..62d5ba42 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts @@ -2,14 +2,9 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import YAML from 'yaml'; -import type { AgentRunnerPort, RunLoopParams } from '../../../llm/index.js'; -import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../../project/index.js'; -import { - type SqlAnalysisBatchItem, - type SqlAnalysisBatchResult, - type SqlAnalysisDialect, - type SqlAnalysisPort, -} from '../../../sql-analysis/index.js'; +import type { AgentRunnerPort, RunLoopParams } from '../../../../context/llm/runtime-port.js'; +import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../../../context/project/project.js'; +import type { SqlAnalysisBatchItem, SqlAnalysisBatchResult, SqlAnalysisDialect, SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js'; import { searchLocalSlSources } from '../../../sl/local-sl.js'; import { searchLocalKnowledgePages } from '../../../wiki/local-knowledge.js'; import { runLocalIngest } from '../../local-ingest.js'; diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/projection.ts b/packages/cli/src/context/ingest/adapters/historic-sql/projection.ts index 56b1e360..272a96dc 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/projection.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/projection.ts @@ -1,7 +1,7 @@ import { access, mkdir, readdir, readFile, rename, writeFile } from 'node:fs/promises'; import { dirname, join, relative } from 'node:path'; import YAML from 'yaml'; -import type { MemoryAction } from '../../../memory/index.js'; +import type { MemoryAction } from '../../../../context/memory/types.js'; import { rawSourcesDirForSync } from '../../raw-sources-paths.js'; import type { FinalizationOverrideReplay } from '../../types.js'; import { mergeUsagePreservingExternal } from '../live-database/manifest.js'; diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/skill-schemas.ts b/packages/cli/src/context/ingest/adapters/historic-sql/skill-schemas.ts index 340cd5b1..a8ba985e 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/skill-schemas.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/skill-schemas.ts @@ -26,6 +26,6 @@ export const patternOutputSchema = z.object({ slRefs: z.array(z.string()), constituentTemplateIds: z.array(z.string()), }); -export type PatternOutput = z.infer; +/** @internal */ export const patternsArraySchema = z.array(patternOutputSchema); diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.test.ts b/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.test.ts index d09a4d40..d49c3a1d 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.test.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, readFile, readdir } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it, vi } from 'vitest'; -import type { SqlAnalysisPort } from '../../../sql-analysis/index.js'; +import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js'; import { stageHistoricSqlAggregatedSnapshot } from './stage-unified.js'; import type { AggregatedTemplate, HistoricSqlReader } from './types.js'; diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.ts b/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.ts index 18675b11..70997648 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.ts @@ -1,6 +1,6 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; -import type { SqlAnalysisPort } from '../../../sql-analysis/index.js'; +import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js'; import { bucketDistinctUsers, bucketErrorRate, diff --git a/packages/cli/src/context/ingest/adapters/historic-sql/types.ts b/packages/cli/src/context/ingest/adapters/historic-sql/types.ts index ddb1846a..1d256b13 100644 --- a/packages/cli/src/context/ingest/adapters/historic-sql/types.ts +++ b/packages/cli/src/context/ingest/adapters/historic-sql/types.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import type { SqlAnalysisPort } from '../../../sql-analysis/index.js'; +import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js'; export const HISTORIC_SQL_SOURCE_KEY = 'historic-sql' as const; @@ -115,9 +115,8 @@ export const stagedManifestSchema = z.object({ probeWarnings: z.array(z.string()), staleArchiveAfterDays: z.number().int().positive().default(90), }); -export type StagedManifest = z.infer; -export interface HistoricSqlProbeResult { +interface HistoricSqlProbeResult { warnings: string[]; info?: string[]; } diff --git a/packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.ts b/packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.ts index d20b0adc..ff01fda9 100644 --- a/packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.ts +++ b/packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.ts @@ -8,9 +8,9 @@ import type { KtxSchemaColumn, KtxSchemaForeignKey, KtxSchemaSnapshot, KtxSchema import { inferKtxDimensionType, normalizeKtxNativeType } from '../../../scan/type-normalization.js'; import type { LiveDatabaseIntrospectionPort } from './types.js'; -export type KtxDaemonDatabaseIntrospectionCommand = 'database-introspect'; +type KtxDaemonDatabaseIntrospectionCommand = 'database-introspect'; -export type KtxDaemonDatabaseJsonRunner = ( +type KtxDaemonDatabaseJsonRunner = ( subcommand: KtxDaemonDatabaseIntrospectionCommand, payload: Record, ) => Promise>; diff --git a/packages/cli/src/context/ingest/adapters/live-database/extracted-schema.test.ts b/packages/cli/src/context/ingest/adapters/live-database/extracted-schema.test.ts deleted file mode 100644 index 69bd062d..00000000 --- a/packages/cli/src/context/ingest/adapters/live-database/extracted-schema.test.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { KtxSchemaSnapshot } from '../../../scan/types.js'; -import { buildLiveDatabaseTableNaturalKey, ktxSchemaSnapshotToExtractedSchema } from './extracted-schema.js'; - -function snapshot(): KtxSchemaSnapshot { - return { - connectionId: 'conn-1', - driver: 'postgres', - extractedAt: '2026-04-27T00:00:00.000Z', - scope: { schemas: ['public'] }, - metadata: { driver: 'postgres' }, - tables: [ - { - name: 'orders', - catalog: null, - db: 'public', - kind: 'table', - comment: 'Orders placed by customers', - estimatedRows: null, - columns: [ - { - name: 'id', - nativeType: 'integer', - normalizedType: 'integer', - dimensionType: 'number', - nullable: false, - primaryKey: true, - comment: 'Primary key', - }, - { - name: 'customer_id', - nativeType: 'integer', - normalizedType: 'integer', - dimensionType: 'number', - nullable: false, - primaryKey: false, - comment: null, - }, - ], - foreignKeys: [ - { - fromColumn: 'customer_id', - toCatalog: null, - toDb: 'public', - toTable: 'customers', - toColumn: 'id', - constraintName: 'orders_customer_id_fkey', - }, - ], - }, - { - name: 'customers', - catalog: null, - db: 'public', - kind: 'table', - comment: null, - estimatedRows: null, - columns: [ - { - name: 'id', - nativeType: 'integer', - normalizedType: 'integer', - dimensionType: 'number', - nullable: false, - primaryKey: true, - comment: null, - }, - ], - foreignKeys: [], - }, - ], - }; -} - -describe('ktxSchemaSnapshotToExtractedSchema', () => { - it('preserves structural table, column, comment, and key metadata', () => { - const extracted = ktxSchemaSnapshotToExtractedSchema(snapshot()); - - expect(extracted.tables).toEqual([ - { - name: 'orders', - catalog: null, - db: 'public', - dbComment: 'Orders placed by customers', - columns: [ - { - name: 'id', - type: 'integer', - nullable: false, - primaryKey: true, - dbComment: 'Primary key', - }, - { - name: 'customer_id', - type: 'integer', - nullable: false, - primaryKey: false, - dbComment: null, - }, - ], - foreignKeys: [ - { - fromTable: 'orders', - fromColumn: 'customer_id', - toTable: 'customers', - toColumn: 'id', - constraintName: 'orders_customer_id_fkey', - }, - ], - }, - { - name: 'customers', - catalog: null, - db: 'public', - dbComment: null, - columns: [ - { - name: 'id', - type: 'integer', - nullable: false, - primaryKey: true, - dbComment: null, - }, - ], - foreignKeys: [], - }, - ]); - }); - - it('builds the same natural key shape used by schema sync', () => { - expect(buildLiveDatabaseTableNaturalKey({ catalog: null, db: 'public', name: 'orders' })).toBe('|public|orders'); - expect(buildLiveDatabaseTableNaturalKey({ catalog: 'warehouse', db: 'analytics', name: 'events' })).toBe( - 'warehouse|analytics|events', - ); - }); -}); diff --git a/packages/cli/src/context/ingest/adapters/live-database/extracted-schema.ts b/packages/cli/src/context/ingest/adapters/live-database/extracted-schema.ts deleted file mode 100644 index 39950c69..00000000 --- a/packages/cli/src/context/ingest/adapters/live-database/extracted-schema.ts +++ /dev/null @@ -1,61 +0,0 @@ -import type { KtxSchemaSnapshot, KtxSchemaTable } from '../../../scan/types.js'; - -export interface LiveDatabaseExtractedForeignKey { - fromTable: string; - fromColumn: string; - toTable: string; - toColumn: string; - constraintName?: string; -} - -export interface LiveDatabaseExtractedColumn { - name: string; - type: string; - nullable: boolean; - primaryKey: boolean; - dbComment: string | null; -} - -export interface LiveDatabaseExtractedTable { - name: string; - catalog: string | null; - db: string | null; - dbComment: string | null; - columns: LiveDatabaseExtractedColumn[]; - foreignKeys: LiveDatabaseExtractedForeignKey[]; -} - -export interface LiveDatabaseExtractedSchema { - connectionId?: string; - tables: LiveDatabaseExtractedTable[]; -} - -export function buildLiveDatabaseTableNaturalKey(table: Pick): string { - return `${table.catalog ?? ''}|${table.db ?? ''}|${table.name}`; -} - -export function ktxSchemaSnapshotToExtractedSchema(snapshot: KtxSchemaSnapshot): LiveDatabaseExtractedSchema { - return { - connectionId: snapshot.connectionId, - tables: snapshot.tables.map((table) => ({ - name: table.name, - catalog: table.catalog ?? null, - db: table.db ?? null, - dbComment: table.comment ?? null, - columns: table.columns.map((column) => ({ - name: column.name, - type: column.nativeType, - nullable: column.nullable, - primaryKey: column.primaryKey, - dbComment: column.comment ?? null, - })), - foreignKeys: table.foreignKeys.map((foreignKey) => ({ - fromTable: table.name, - fromColumn: foreignKey.fromColumn, - toTable: foreignKey.toTable, - toColumn: foreignKey.toColumn, - ...(foreignKey.constraintName ? { constraintName: foreignKey.constraintName } : {}), - })), - })), - }; -} diff --git a/packages/cli/src/context/ingest/adapters/live-database/manifest.ts b/packages/cli/src/context/ingest/adapters/live-database/manifest.ts index c6a6e2d5..3c35b463 100644 --- a/packages/cli/src/context/ingest/adapters/live-database/manifest.ts +++ b/packages/cli/src/context/ingest/adapters/live-database/manifest.ts @@ -22,7 +22,7 @@ const HISTORIC_SQL_MANAGED_USAGE_KEYS = new Set([ 'staleSince', ]); -export interface LiveDatabaseManifestColumn { +interface LiveDatabaseManifestColumn { name: string; type: string; pk?: boolean; @@ -37,7 +37,7 @@ export interface LiveDatabaseManifestJoinEntry { source: string; } -export interface LiveDatabaseManifestTableEntry { +interface LiveDatabaseManifestTableEntry { table: string; descriptions?: Record; usage?: TableUsageOutput; diff --git a/packages/cli/src/context/ingest/adapters/live-database/structural-sync.test.ts b/packages/cli/src/context/ingest/adapters/live-database/structural-sync.test.ts deleted file mode 100644 index 1df5faf8..00000000 --- a/packages/cli/src/context/ingest/adapters/live-database/structural-sync.test.ts +++ /dev/null @@ -1,428 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { type LiveDatabaseSyncedSchema, planLiveDatabaseStructuralSync } from './structural-sync.js'; - -function idFactory(): () => string { - let next = 1; - return () => `id-${next++}`; -} - -describe('planLiveDatabaseStructuralSync', () => { - it('plans table and column creates, updates, deletes, and metadata invalidation', () => { - const current: LiveDatabaseSyncedSchema = { - connectionId: 'conn-1', - tables: [ - { - id: 'tbl-orders', - name: 'orders', - catalog: null, - db: 'public', - enabled: true, - descriptions: { ai: 'Old AI order text', db: 'Old DB order text' }, - columns: [ - { - id: 'col-order-id', - name: 'id', - type: 'number', - nullable: false, - primaryKey: true, - parentColumnId: null, - descriptions: { db: 'Order id' }, - embedding: [1, 2, 3], - sampleValues: null, - cardinality: null, - }, - { - id: 'col-order-total', - name: 'total', - type: 'number', - nullable: true, - primaryKey: false, - parentColumnId: null, - descriptions: { ai: 'Old AI total text', db: 'Old total text' }, - embedding: [4, 5, 6], - sampleValues: ['10'], - cardinality: 12, - }, - { - id: 'col-order-removed', - name: 'removed', - type: 'string', - nullable: true, - primaryKey: false, - parentColumnId: null, - descriptions: {}, - embedding: null, - sampleValues: null, - cardinality: null, - }, - ], - }, - { - id: 'tbl-removed', - name: 'removed_table', - catalog: null, - db: 'public', - enabled: true, - descriptions: {}, - columns: [ - { - id: 'col-removed-id', - name: 'id', - type: 'number', - nullable: false, - primaryKey: true, - parentColumnId: null, - descriptions: {}, - embedding: null, - sampleValues: null, - cardinality: null, - }, - ], - }, - ], - links: [ - { - id: 'inferred-total-link', - fromTableId: 'tbl-orders', - fromColumnId: 'col-order-total', - toTableId: 'tbl-orders', - toColumnId: 'col-order-id', - source: 'inferred', - confidence: 0.7, - relationshipType: 'MANY_TO_ONE', - isPrimaryKeyReference: true, - }, - ], - }; - - const plan = planLiveDatabaseStructuralSync({ - connectionId: 'conn-1', - current, - extracted: { - connectionId: 'conn-1', - tables: [ - { - name: 'orders', - catalog: null, - db: 'public', - dbComment: 'Fresh DB order text', - columns: [ - { - name: 'id', - type: 'number', - nullable: false, - primaryKey: true, - dbComment: 'Order id', - }, - { - name: 'total', - type: 'string', - nullable: false, - primaryKey: false, - dbComment: 'Fresh total text', - }, - { - name: 'created_at', - type: 'time', - nullable: false, - primaryKey: false, - dbComment: 'Creation timestamp', - }, - ], - foreignKeys: [], - }, - { - name: 'customers', - catalog: null, - db: 'public', - dbComment: 'Customer table', - columns: [ - { - name: 'id', - type: 'number', - nullable: false, - primaryKey: true, - dbComment: null, - }, - ], - foreignKeys: [], - }, - ], - }, - idFactory: idFactory(), - }); - - expect(plan.stats).toEqual({ - tablesCreated: 1, - tablesDeleted: 1, - columnsCreated: 2, - columnsDeleted: 2, - columnsModified: 1, - formalLinksCreated: 0, - formalLinksDeleted: 0, - }); - expect(plan.operations.deleteTableIds).toEqual(['tbl-removed']); - expect(plan.operations.deleteColumnIds).toEqual(['col-order-removed']); - expect(plan.operations.insertTables).toEqual([ - { - id: 'id-2', - connectionId: 'conn-1', - name: 'customers', - catalog: null, - db: 'public', - enabled: true, - }, - ]); - expect(plan.operations.insertColumns).toEqual([ - { - id: 'id-1', - tableId: 'tbl-orders', - name: 'created_at', - parentColumnId: null, - }, - { - id: 'id-3', - tableId: 'id-2', - name: 'id', - parentColumnId: null, - }, - ]); - expect(plan.operations.touchColumnIds).toEqual(['col-order-total']); - expect(plan.operations.invalidateColumnEmbeddingIds).toEqual(['col-order-total']); - expect(plan.inferredLinksToValidate).toEqual(['inferred-total-link']); - expect(plan.changes).toEqual({ - newTableIds: ['id-2'], - newColumnIds: ['id-1', 'id-3'], - tablesWithStructuralChanges: ['tbl-orders', 'id-2'], - columnsWithTypeChange: ['col-order-total'], - columnsWithDescriptionChange: ['col-order-total'], - tablesWithDescriptionChange: ['tbl-orders'], - }); - - const orders = plan.schema.tables.find((table) => table.name === 'orders'); - expect(orders?.descriptions).toEqual({ db: 'Fresh DB order text' }); - expect(orders?.columns.map((column) => column.name)).toEqual(['id', 'total', 'created_at']); - expect(orders?.columns.find((column) => column.name === 'total')).toMatchObject({ - id: 'col-order-total', - type: 'string', - nullable: false, - primaryKey: false, - descriptions: { db: 'Fresh total text' }, - embedding: null, - sampleValues: ['10'], - cardinality: 12, - }); - }); - - it('builds formal links from extracted foreign keys and preserves valid inferred links', () => { - const current: LiveDatabaseSyncedSchema = { - connectionId: 'conn-1', - tables: [ - { - id: 'tbl-orders', - name: 'orders', - catalog: null, - db: 'public', - enabled: true, - descriptions: {}, - columns: [ - { - id: 'col-orders-id', - name: 'id', - type: 'number', - nullable: false, - primaryKey: true, - parentColumnId: null, - descriptions: {}, - embedding: null, - sampleValues: null, - cardinality: null, - }, - { - id: 'col-orders-customer', - name: 'customer_id', - type: 'number', - nullable: false, - primaryKey: false, - parentColumnId: null, - descriptions: {}, - embedding: null, - sampleValues: null, - cardinality: null, - }, - ], - }, - { - id: 'tbl-customers', - name: 'customers', - catalog: null, - db: 'public', - enabled: true, - descriptions: {}, - columns: [ - { - id: 'col-customers-id', - name: 'id', - type: 'number', - nullable: false, - primaryKey: true, - parentColumnId: null, - descriptions: {}, - embedding: null, - sampleValues: null, - cardinality: null, - }, - ], - }, - ], - links: [ - { - id: 'formal-existing', - fromTableId: 'tbl-orders', - fromColumnId: 'col-orders-customer', - toTableId: 'tbl-customers', - toColumnId: 'col-customers-id', - source: 'formal', - confidence: 1, - relationshipType: 'MANY_TO_ONE', - isPrimaryKeyReference: true, - }, - { - id: 'inferred-existing', - fromTableId: 'tbl-orders', - fromColumnId: 'col-orders-id', - toTableId: 'tbl-customers', - toColumnId: 'col-customers-id', - source: 'inferred', - confidence: 0.6, - relationshipType: 'MANY_TO_ONE', - isPrimaryKeyReference: true, - }, - ], - }; - - const plan = planLiveDatabaseStructuralSync({ - connectionId: 'conn-1', - current, - extracted: { - connectionId: 'conn-1', - tables: [ - { - name: 'orders', - catalog: null, - db: 'public', - dbComment: null, - columns: [ - { name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }, - { name: 'customer_id', type: 'number', nullable: false, primaryKey: false, dbComment: null }, - ], - foreignKeys: [ - { - fromTable: 'orders', - fromColumn: 'customer_id', - toTable: 'customers', - toColumn: 'id', - }, - ], - }, - { - name: 'customers', - catalog: null, - db: 'public', - dbComment: null, - columns: [{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }], - foreignKeys: [], - }, - ], - }, - idFactory: idFactory(), - }); - - expect(plan.stats.formalLinksCreated).toBe(0); - expect(plan.stats.formalLinksDeleted).toBe(0); - expect(plan.schema.links.map((link) => link.id)).toEqual(['formal-existing', 'inferred-existing']); - - const planAfterForeignKeyRemoval = planLiveDatabaseStructuralSync({ - connectionId: 'conn-1', - current, - extracted: { - connectionId: 'conn-1', - tables: [ - { - name: 'orders', - catalog: null, - db: 'public', - dbComment: null, - columns: [ - { name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }, - { name: 'customer_id', type: 'number', nullable: false, primaryKey: false, dbComment: null }, - ], - foreignKeys: [], - }, - { - name: 'customers', - catalog: null, - db: 'public', - dbComment: null, - columns: [{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }], - foreignKeys: [], - }, - ], - }, - idFactory: idFactory(), - }); - - expect(planAfterForeignKeyRemoval.stats.formalLinksDeleted).toBe(1); - expect(planAfterForeignKeyRemoval.schema.links.map((link) => link.id)).toEqual(['inferred-existing']); - - const planAfterForeignKeyCreation = planLiveDatabaseStructuralSync({ - connectionId: 'conn-1', - current: { ...current, links: [current.links[1]] }, - extracted: { - connectionId: 'conn-1', - tables: [ - { - name: 'orders', - catalog: null, - db: 'public', - dbComment: null, - columns: [ - { name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }, - { name: 'customer_id', type: 'number', nullable: false, primaryKey: false, dbComment: null }, - ], - foreignKeys: [ - { - fromTable: 'orders', - fromColumn: 'customer_id', - toTable: 'customers', - toColumn: 'id', - }, - ], - }, - { - name: 'customers', - catalog: null, - db: 'public', - dbComment: null, - columns: [{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }], - foreignKeys: [], - }, - ], - }, - idFactory: idFactory(), - }); - - expect(planAfterForeignKeyCreation.stats.formalLinksCreated).toBe(1); - expect(planAfterForeignKeyCreation.schema.links[0]).toMatchObject({ - id: 'id-1', - fromTableId: 'tbl-orders', - fromColumnId: 'col-orders-customer', - toTableId: 'tbl-customers', - toColumnId: 'col-customers-id', - source: 'formal', - confidence: 1, - relationshipType: 'MANY_TO_ONE', - isPrimaryKeyReference: true, - }); - }); -}); diff --git a/packages/cli/src/context/ingest/adapters/live-database/structural-sync.ts b/packages/cli/src/context/ingest/adapters/live-database/structural-sync.ts deleted file mode 100644 index d98a692b..00000000 --- a/packages/cli/src/context/ingest/adapters/live-database/structural-sync.ts +++ /dev/null @@ -1,525 +0,0 @@ -import type { LiveDatabaseExtractedSchema, LiveDatabaseExtractedTable } from './extracted-schema.js'; -import { buildLiveDatabaseTableNaturalKey } from './extracted-schema.js'; - -export interface LiveDatabaseSyncedColumn { - id: string; - name: string; - type: string; - nullable: boolean; - primaryKey: boolean; - parentColumnId: string | null; - descriptions: Record; - embedding: number[] | null; - sampleValues: string[] | null; - cardinality: number | null; -} - -export interface LiveDatabaseSyncedTable { - id: string; - name: string; - catalog: string | null; - db: string | null; - enabled: boolean; - descriptions: Record; - columns: LiveDatabaseSyncedColumn[]; -} - -export interface LiveDatabaseSyncedLink { - id: string; - fromTableId: string; - fromColumnId: string; - toTableId: string; - toColumnId: string; - source: 'formal' | 'inferred' | 'manual'; - confidence: number; - relationshipType: string; - isPrimaryKeyReference: boolean; -} - -export interface LiveDatabaseSyncedSchema { - connectionId: string; - tables: LiveDatabaseSyncedTable[]; - links: LiveDatabaseSyncedLink[]; -} - -export interface LiveDatabaseStructuralChanges { - newTableIds: string[]; - newColumnIds: string[]; - tablesWithStructuralChanges: string[]; - columnsWithTypeChange: string[]; - columnsWithDescriptionChange: string[]; - tablesWithDescriptionChange: string[]; -} - -export interface LiveDatabaseStructuralSyncStats { - tablesCreated: number; - tablesDeleted: number; - columnsCreated: number; - columnsDeleted: number; - columnsModified: number; - formalLinksCreated: number; - formalLinksDeleted: number; -} - -export interface LiveDatabaseStructuralSyncOperations { - deleteTableIds: string[]; - deleteColumnIds: string[]; - insertTables: Array<{ - id: string; - connectionId: string; - name: string; - catalog: string | null; - db: string | null; - enabled: boolean; - }>; - insertColumns: Array<{ - id: string; - tableId: string; - name: string; - parentColumnId: string | null; - }>; - touchColumnIds: string[]; - invalidateColumnEmbeddingIds: string[]; -} - -export interface LiveDatabaseStructuralSyncPlan { - schema: LiveDatabaseSyncedSchema; - inferredLinksToValidate: string[]; - stats: LiveDatabaseStructuralSyncStats; - changes: LiveDatabaseStructuralChanges; - operations: LiveDatabaseStructuralSyncOperations; -} - -export interface PlanLiveDatabaseStructuralSyncInput { - connectionId: string; - current: LiveDatabaseSyncedSchema | null; - extracted: LiveDatabaseExtractedSchema; - idFactory: () => string; -} - -interface UpdatedTableResult { - table: LiveDatabaseSyncedTable; - columnsCreated: number; - columnsDeleted: number; - columnsModified: number; - newColumnIds: string[]; - columnsWithTypeChange: string[]; - columnsWithDescriptionChange: string[]; - tableDescriptionChanged: boolean; -} - -function updateDescription( - descriptions: Record, - dbComment: string | null | undefined, - changed: boolean, -): Record { - const updated = { ...descriptions }; - if (dbComment) { - updated.db = dbComment; - } else { - delete updated.db; - } - if (changed) { - delete updated.ai; - } - return updated; -} - -function descriptionFromDbComment(dbComment: string | null | undefined): Record { - return dbComment ? { db: dbComment } : {}; -} - -function planUpdatedTable(args: { - currentTable: LiveDatabaseSyncedTable; - extractedTable: LiveDatabaseExtractedTable; - currentLinks: LiveDatabaseSyncedLink[]; - inferredLinksToValidate: string[]; - operations: LiveDatabaseStructuralSyncOperations; - idFactory: () => string; -}): UpdatedTableResult { - const { currentTable, extractedTable, currentLinks, inferredLinksToValidate, operations, idFactory } = args; - - let columnsCreated = 0; - let columnsDeleted = 0; - let columnsModified = 0; - const newColumnIds: string[] = []; - const columnsWithTypeChange: string[] = []; - const columnsWithDescriptionChange: string[] = []; - const updatedColumns: LiveDatabaseSyncedColumn[] = []; - - const tableDescriptionChanged = (currentTable.descriptions.db ?? null) !== (extractedTable.dbComment ?? null); - const currentColumnsByName = new Map(currentTable.columns.map((column) => [column.name, column])); - const extractedColumnsByName = new Map(extractedTable.columns.map((column) => [column.name, column])); - - for (const [name, currentColumn] of currentColumnsByName) { - if (!extractedColumnsByName.has(name)) { - operations.deleteColumnIds.push(currentColumn.id); - columnsDeleted++; - } - } - - for (const [name, extractedColumn] of extractedColumnsByName) { - const currentColumn = currentColumnsByName.get(name); - if (!currentColumn) { - const columnId = idFactory(); - operations.insertColumns.push({ - id: columnId, - tableId: currentTable.id, - name: extractedColumn.name, - parentColumnId: null, - }); - columnsCreated++; - newColumnIds.push(columnId); - updatedColumns.push({ - id: columnId, - name: extractedColumn.name, - type: extractedColumn.type, - nullable: extractedColumn.nullable, - primaryKey: extractedColumn.primaryKey, - descriptions: descriptionFromDbComment(extractedColumn.dbComment), - parentColumnId: null, - embedding: null, - sampleValues: null, - cardinality: null, - }); - continue; - } - - const typeChanged = currentColumn.type !== extractedColumn.type; - const nullableChanged = currentColumn.nullable !== extractedColumn.nullable; - const primaryKeyChanged = currentColumn.primaryKey !== extractedColumn.primaryKey; - const dbDescriptionChanged = (currentColumn.descriptions.db ?? null) !== (extractedColumn.dbComment ?? null); - - if (typeChanged || nullableChanged || primaryKeyChanged || dbDescriptionChanged) { - operations.touchColumnIds.push(currentColumn.id); - columnsModified++; - - if (typeChanged || dbDescriptionChanged) { - operations.invalidateColumnEmbeddingIds.push(currentColumn.id); - } - - if (typeChanged) { - columnsWithTypeChange.push(currentColumn.id); - const affectedLinks = currentLinks.filter( - (link) => - link.source === 'inferred' && - (link.fromColumnId === currentColumn.id || link.toColumnId === currentColumn.id), - ); - for (const link of affectedLinks) { - if (!inferredLinksToValidate.includes(link.id)) { - inferredLinksToValidate.push(link.id); - } - } - } - - if (dbDescriptionChanged) { - columnsWithDescriptionChange.push(currentColumn.id); - } - } - - updatedColumns.push({ - ...currentColumn, - type: extractedColumn.type, - nullable: extractedColumn.nullable, - primaryKey: extractedColumn.primaryKey, - descriptions: updateDescription(currentColumn.descriptions, extractedColumn.dbComment, dbDescriptionChanged), - embedding: typeChanged ? null : currentColumn.embedding, - }); - } - - return { - table: { - ...currentTable, - descriptions: updateDescription(currentTable.descriptions, extractedTable.dbComment, tableDescriptionChanged), - columns: updatedColumns, - }, - columnsCreated, - columnsDeleted, - columnsModified, - newColumnIds, - columnsWithTypeChange, - columnsWithDescriptionChange, - tableDescriptionChanged, - }; -} - -function planCreatedTable(args: { - connectionId: string; - extractedTable: LiveDatabaseExtractedTable; - operations: LiveDatabaseStructuralSyncOperations; - idFactory: () => string; -}): LiveDatabaseSyncedTable { - const { connectionId, extractedTable, operations, idFactory } = args; - const tableId = idFactory(); - operations.insertTables.push({ - id: tableId, - connectionId, - name: extractedTable.name, - catalog: extractedTable.catalog, - db: extractedTable.db, - enabled: true, - }); - - const columns: LiveDatabaseSyncedColumn[] = extractedTable.columns.map((extractedColumn) => { - const columnId = idFactory(); - operations.insertColumns.push({ - id: columnId, - tableId, - name: extractedColumn.name, - parentColumnId: null, - }); - return { - id: columnId, - name: extractedColumn.name, - type: extractedColumn.type, - nullable: extractedColumn.nullable, - primaryKey: extractedColumn.primaryKey, - descriptions: descriptionFromDbComment(extractedColumn.dbComment), - parentColumnId: null, - embedding: null, - sampleValues: null, - cardinality: null, - }; - }); - - return { - id: tableId, - name: extractedTable.name, - catalog: extractedTable.catalog, - db: extractedTable.db, - enabled: true, - descriptions: descriptionFromDbComment(extractedTable.dbComment), - columns, - }; -} - -function syncFormalLinks(args: { - extracted: LiveDatabaseExtractedSchema; - tables: LiveDatabaseSyncedTable[]; - tableNaturalKeyToId: Map; - currentLinks: LiveDatabaseSyncedLink[]; - idFactory: () => string; -}): { links: LiveDatabaseSyncedLink[]; created: number; deleted: number } { - const { extracted, tables, tableNaturalKeyToId, currentLinks, idFactory } = args; - const columnKeyToId = new Map(); - - for (const table of tables) { - const tableKey = buildLiveDatabaseTableNaturalKey(table); - for (const column of table.columns) { - columnKeyToId.set(`${tableKey}.${column.name}`, column.id); - } - } - - const extractedFormalLinks: Array<{ - fromTableId: string; - fromColumnId: string; - toTableId: string; - toColumnId: string; - }> = []; - - for (const table of extracted.tables) { - const fromTableKey = buildLiveDatabaseTableNaturalKey(table); - const fromTableId = tableNaturalKeyToId.get(fromTableKey); - if (!fromTableId) { - continue; - } - - for (const foreignKey of table.foreignKeys) { - const toTableKey = buildLiveDatabaseTableNaturalKey({ - catalog: table.catalog, - db: table.db, - name: foreignKey.toTable, - }); - const toTableId = tableNaturalKeyToId.get(toTableKey); - if (!toTableId) { - continue; - } - - const fromColumnId = columnKeyToId.get(`${fromTableKey}.${foreignKey.fromColumn}`); - const toColumnId = columnKeyToId.get(`${toTableKey}.${foreignKey.toColumn}`); - if (!fromColumnId || !toColumnId) { - continue; - } - - extractedFormalLinks.push({ fromTableId, fromColumnId, toTableId, toColumnId }); - } - } - - const currentFormalLinks = currentLinks.filter((link) => link.source === 'formal'); - const extractedLinkKeys = new Set(extractedFormalLinks.map((link) => `${link.fromColumnId}->${link.toColumnId}`)); - const linksToDelete = currentFormalLinks.filter( - (link) => !extractedLinkKeys.has(`${link.fromColumnId}->${link.toColumnId}`), - ); - - const currentLinkKeys = new Set(currentFormalLinks.map((link) => `${link.fromColumnId}->${link.toColumnId}`)); - const linksToCreate = extractedFormalLinks.filter( - (link) => !currentLinkKeys.has(`${link.fromColumnId}->${link.toColumnId}`), - ); - - const newLinks = linksToCreate.map((linkData) => ({ - id: idFactory(), - fromTableId: linkData.fromTableId, - fromColumnId: linkData.fromColumnId, - toTableId: linkData.toTableId, - toColumnId: linkData.toColumnId, - source: 'formal' as const, - confidence: 1, - relationshipType: 'MANY_TO_ONE', - isPrimaryKeyReference: true, - })); - - const deletedLinkIds = new Set(linksToDelete.map((link) => link.id)); - const preservedFormalLinks = currentFormalLinks.filter((link) => !deletedLinkIds.has(link.id)); - - return { - links: [...preservedFormalLinks, ...newLinks], - created: linksToCreate.length, - deleted: linksToDelete.length, - }; -} - -export function planLiveDatabaseStructuralSync( - input: PlanLiveDatabaseStructuralSyncInput, -): LiveDatabaseStructuralSyncPlan { - const operations: LiveDatabaseStructuralSyncOperations = { - deleteTableIds: [], - deleteColumnIds: [], - insertTables: [], - insertColumns: [], - touchColumnIds: [], - invalidateColumnEmbeddingIds: [], - }; - const stats: LiveDatabaseStructuralSyncStats = { - tablesCreated: 0, - tablesDeleted: 0, - columnsCreated: 0, - columnsDeleted: 0, - columnsModified: 0, - formalLinksCreated: 0, - formalLinksDeleted: 0, - }; - const changes: LiveDatabaseStructuralChanges = { - newTableIds: [], - newColumnIds: [], - tablesWithStructuralChanges: [], - columnsWithTypeChange: [], - columnsWithDescriptionChange: [], - tablesWithDescriptionChange: [], - }; - const inferredLinksToValidate: string[] = []; - - const currentTablesByKey = new Map(); - const extractedTablesByKey = new Map(); - - if (input.current) { - for (const table of input.current.tables) { - currentTablesByKey.set(buildLiveDatabaseTableNaturalKey(table), table); - } - } - for (const table of input.extracted.tables) { - extractedTablesByKey.set(buildLiveDatabaseTableNaturalKey(table), table); - } - - const tablesToDelete: LiveDatabaseSyncedTable[] = []; - const tablesToUpdate: Array<{ - current: LiveDatabaseSyncedTable; - extracted: LiveDatabaseExtractedTable; - }> = []; - const tablesToCreate: LiveDatabaseExtractedTable[] = []; - - for (const [key, table] of currentTablesByKey) { - const extractedTable = extractedTablesByKey.get(key); - if (!extractedTable) { - tablesToDelete.push(table); - } else { - tablesToUpdate.push({ current: table, extracted: extractedTable }); - } - } - - for (const [key, table] of extractedTablesByKey) { - if (!currentTablesByKey.has(key)) { - tablesToCreate.push(table); - } - } - - for (const table of tablesToDelete) { - operations.deleteTableIds.push(table.id); - stats.tablesDeleted++; - stats.columnsDeleted += table.columns.length; - } - - const updatedTables: LiveDatabaseSyncedTable[] = []; - for (const { current, extracted } of tablesToUpdate) { - const result = planUpdatedTable({ - currentTable: current, - extractedTable: extracted, - currentLinks: input.current?.links ?? [], - inferredLinksToValidate, - operations, - idFactory: input.idFactory, - }); - updatedTables.push(result.table); - stats.columnsCreated += result.columnsCreated; - stats.columnsDeleted += result.columnsDeleted; - stats.columnsModified += result.columnsModified; - changes.newColumnIds.push(...result.newColumnIds); - changes.columnsWithTypeChange.push(...result.columnsWithTypeChange); - changes.columnsWithDescriptionChange.push(...result.columnsWithDescriptionChange); - if (result.tableDescriptionChanged) { - changes.tablesWithDescriptionChange.push(current.id); - } - if (result.columnsCreated > 0 || result.columnsDeleted > 0 || result.columnsWithTypeChange.length > 0) { - changes.tablesWithStructuralChanges.push(current.id); - } - } - - const createdTables: LiveDatabaseSyncedTable[] = []; - for (const extractedTable of tablesToCreate) { - const table = planCreatedTable({ - connectionId: input.connectionId, - extractedTable, - operations, - idFactory: input.idFactory, - }); - createdTables.push(table); - stats.tablesCreated++; - stats.columnsCreated += table.columns.length; - changes.newTableIds.push(table.id); - changes.newColumnIds.push(...table.columns.map((column) => column.id)); - changes.tablesWithStructuralChanges.push(table.id); - } - - const allTables = [...updatedTables, ...createdTables]; - const tableNaturalKeyToId = new Map(); - for (const table of allTables) { - tableNaturalKeyToId.set(buildLiveDatabaseTableNaturalKey(table), table.id); - } - - const formalLinkResult = syncFormalLinks({ - extracted: input.extracted, - tables: allTables, - tableNaturalKeyToId, - currentLinks: input.current?.links ?? [], - idFactory: input.idFactory, - }); - stats.formalLinksCreated = formalLinkResult.created; - stats.formalLinksDeleted = formalLinkResult.deleted; - - const deletedTableIds = new Set(tablesToDelete.map((table) => table.id)); - const preservedInferredLinks = (input.current?.links ?? []).filter( - (link) => - link.source === 'inferred' && !deletedTableIds.has(link.fromTableId) && !deletedTableIds.has(link.toTableId), - ); - - return { - schema: { - connectionId: input.connectionId, - tables: allTables, - links: [...formalLinkResult.links, ...preservedInferredLinks], - }, - inferredLinksToValidate, - stats, - changes, - operations, - }; -} diff --git a/packages/cli/src/context/ingest/adapters/looker/factory.ts b/packages/cli/src/context/ingest/adapters/looker/factory.ts index e80d781c..64fdfa42 100644 --- a/packages/cli/src/context/ingest/adapters/looker/factory.ts +++ b/packages/cli/src/context/ingest/adapters/looker/factory.ts @@ -7,6 +7,7 @@ export interface LookerCredentialResolver { resolve(lookerConnectionId: string): Promise; } +/** @internal */ export interface LookerConnectionClientFactory { createClient(lookerConnectionId: string): Promise; } @@ -23,6 +24,7 @@ export class DefaultLookerConnectionClientFactory implements LookerConnectionCli } } +/** @internal */ export class DefaultLookerClientFactory implements LookerClientFactory { constructor(private readonly inner: LookerConnectionClientFactory) {} diff --git a/packages/cli/src/context/ingest/adapters/looker/fetch.ts b/packages/cli/src/context/ingest/adapters/looker/fetch.ts index 6959e9b0..9a09ac1f 100644 --- a/packages/cli/src/context/ingest/adapters/looker/fetch.ts +++ b/packages/cli/src/context/ingest/adapters/looker/fetch.ts @@ -31,7 +31,7 @@ import { stagedUserFileSchema, } from './types.js'; -export interface LookerEntityRef { +interface LookerEntityRef { id: string; updatedAt?: string | null; } diff --git a/packages/cli/src/context/ingest/adapters/looker/local-looker.adapter.ts b/packages/cli/src/context/ingest/adapters/looker/local-looker.adapter.ts index 47299373..ea8ba658 100644 --- a/packages/cli/src/context/ingest/adapters/looker/local-looker.adapter.ts +++ b/packages/cli/src/context/ingest/adapters/looker/local-looker.adapter.ts @@ -1,11 +1,8 @@ -import type { KtxLocalProject, KtxProjectConnectionConfig } from '../../../project/index.js'; -import type { LookerClientLogger } from './client.js'; +import type { KtxLocalProject } from '../../../../context/project/project.js'; +import type { KtxProjectConnectionConfig } from '../../../../context/project/config.js'; import { - DefaultLookerClientFactory, - DefaultLookerConnectionClientFactory, type LookerCredentialResolver, } from './factory.js'; -import { LookerSourceAdapter } from './looker.adapter.js'; function stringField(value: unknown): string | null { return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null; @@ -54,16 +51,3 @@ export function createLocalLookerCredentialResolver( }, }; } - -export function createLocalLookerSourceAdapter( - project: KtxLocalProject, - env: NodeJS.ProcessEnv = process.env, - logger?: LookerClientLogger, -): LookerSourceAdapter { - const connectionFactory = new DefaultLookerConnectionClientFactory(createLocalLookerCredentialResolver(project, env), { - ...(logger ? { logger } : {}), - }); - return new LookerSourceAdapter({ - clientFactory: new DefaultLookerClientFactory(connectionFactory), - }); -} diff --git a/packages/cli/src/context/ingest/adapters/looker/local-runtime-store.ts b/packages/cli/src/context/ingest/adapters/looker/local-runtime-store.ts index a7bc7111..fa6560b2 100644 --- a/packages/cli/src/context/ingest/adapters/looker/local-runtime-store.ts +++ b/packages/cli/src/context/ingest/adapters/looker/local-runtime-store.ts @@ -5,7 +5,7 @@ import type { LookerWarehouseConnectionInfo } from './client.js'; import type { LookerConnectionMapping } from './mapping.js'; import type { LookerRuntimeCursors } from './types.js'; -export type LocalLookerMappingSource = 'ktx.yaml' | 'cli' | 'refresh'; +type LocalLookerMappingSource = 'ktx.yaml' | 'cli' | 'refresh'; interface LocalLookerRuntimeStoreOptions { dbPath: string; @@ -41,7 +41,7 @@ export interface ClearLocalLookerMappingsInput { lookerConnectionName?: string; } -export interface LookerSourceStateReader { +interface LookerSourceStateReader { readMappings(lookerConnectionId: string): Promise; readCursors(lookerConnectionId: string): Promise; } diff --git a/packages/cli/src/context/ingest/adapters/looker/mapping.ts b/packages/cli/src/context/ingest/adapters/looker/mapping.ts index abfffcdf..cbd80e80 100644 --- a/packages/cli/src/context/ingest/adapters/looker/mapping.ts +++ b/packages/cli/src/context/ingest/adapters/looker/mapping.ts @@ -2,7 +2,7 @@ import type { ParsedTargetTable } from '../../parsed-target-table.js'; import type { LookerWarehouseConnectionInfo } from './client.js'; import type { LookerPullConfig, LookerRuntimeCursors, StagedExploreFile, StagedLookmlModelsFile } from './types.js'; -export const LOOKER_DIALECT_TO_CONNECTION_TYPE = { +const LOOKER_DIALECT_TO_CONNECTION_TYPE = { bigquery: 'BIGQUERY', bigquery_standard_sql: 'BIGQUERY', snowflake: 'SNOWFLAKE', @@ -16,6 +16,7 @@ export const LOOKER_DIALECT_TO_CONNECTION_TYPE = { clickhouse: 'CLICKHOUSE', } as const; +/** @internal */ export type LookerWarehouseTargetConnectionType = (typeof LOOKER_DIALECT_TO_CONNECTION_TYPE)[keyof typeof LOOKER_DIALECT_TO_CONNECTION_TYPE]; @@ -33,6 +34,7 @@ export interface LookerTargetConnection { connection_params?: Record | null; } +/** @internal */ export interface LookerMappingCandidateConnection extends LookerTargetConnection {} export interface LookerMappingDrift { @@ -89,6 +91,7 @@ export async function discoverLookerConnections( return client.listLookerConnections(); } +/** @internal */ export function lookerDialectToConnectionType(dialect: string | null): LookerWarehouseTargetConnectionType | null { if (!dialect) { return null; @@ -98,10 +101,12 @@ export function lookerDialectToConnectionType(dialect: string | null): LookerWar ); } +/** @internal */ export function sqlglotDialectForConnectionType(connectionType: string): string | null { return SQLGLOT_DIALECT_BY_CONNECTION_TYPE[connectionType as LookerWarehouseTargetConnectionType] ?? null; } +/** @internal */ export function validateLookerWarehouseTarget(connectionType: string): { ok: true } | { ok: false; reason: string } { return sqlglotDialectForConnectionType(connectionType) ? { ok: true } @@ -111,7 +116,7 @@ export function validateLookerWarehouseTarget(connectionType: string): { ok: tru }; } -export function extractWarehouseHost(params: unknown, connectionType: string): string | null { +function extractWarehouseHost(params: unknown, connectionType: string): string | null { const record = isRecord(params) ? params : {}; switch (connectionType) { case 'POSTGRESQL': @@ -126,7 +131,7 @@ export function extractWarehouseHost(params: unknown, connectionType: string): s } } -export function extractWarehouseDatabase(params: unknown, connectionType: string): string | null { +function extractWarehouseDatabase(params: unknown, connectionType: string): string | null { const record = isRecord(params) ? params : {}; switch (connectionType) { case 'POSTGRESQL': @@ -142,14 +147,15 @@ export function extractWarehouseDatabase(params: unknown, connectionType: string } } -export function normalizeHost(value: string | null): string | null { +function normalizeHost(value: string | null): string | null { return value ? value.toLowerCase().replace(/:\d+$/, '') : null; } -export function normalizeName(value: string | null): string | null { +function normalizeName(value: string | null): string | null { return value ? value.toLowerCase() : null; } +/** @internal */ export function suggestKtxConnectionForLookerConnection(args: { lookerConnection: LookerWarehouseConnectionInfo; candidateConnections: LookerMappingCandidateConnection[]; @@ -224,6 +230,7 @@ export function validateLookerMappings(args: { return errors.length === 0 ? { ok: true } : { ok: false, errors }; } +/** @internal */ export function refreshLookerMappingPlaceholders(args: { stored: LookerConnectionMapping[]; live: LookerWarehouseConnectionInfo[]; @@ -264,6 +271,7 @@ export function refreshLookerMappingPlaceholders(args: { return { mappings: [...byName.values()], changed }; } +/** @internal */ export function collectExploreParseItems(args: { explore: StagedExploreFile; connectionMappings: Record; @@ -309,6 +317,7 @@ export function collectExploreParseItems(args: { return { parsedTargetTables, parseItems }; } +/** @internal */ export function projectParsedIdentifier(row: LookerParsedIdentifier | undefined): ParsedTargetTable { if (!row) { return { ok: false, reason: 'parse_error', detail: 'Python parser response was missing this key.' }; diff --git a/packages/cli/src/context/ingest/adapters/looker/scope.ts b/packages/cli/src/context/ingest/adapters/looker/scope.ts index feabde72..f91d31b6 100644 --- a/packages/cli/src/context/ingest/adapters/looker/scope.ts +++ b/packages/cli/src/context/ingest/adapters/looker/scope.ts @@ -15,7 +15,7 @@ export async function describeLookerScope(stagedDir: string): Promise { +async function readLookerScope(stagedDir: string): Promise { try { const body = await readFile(join(stagedDir, STAGED_FILES.scope), 'utf-8'); return stagedLookerScopeFileSchema.parse(JSON.parse(body)); @@ -27,6 +27,7 @@ export async function readLookerScope(stagedDir: string): Promise { diff --git a/packages/cli/src/context/ingest/adapters/looker/tools/looker-query-to-sl.tool.ts b/packages/cli/src/context/ingest/adapters/looker/tools/looker-query-to-sl.tool.ts index fd515bad..400d123e 100644 --- a/packages/cli/src/context/ingest/adapters/looker/tools/looker-query-to-sl.tool.ts +++ b/packages/cli/src/context/ingest/adapters/looker/tools/looker-query-to-sl.tool.ts @@ -1,6 +1,6 @@ import { tool } from 'ai'; import { z } from 'zod'; -import type { ToolOutput } from '../../../../tools/index.js'; +import type { ToolOutput } from '../../../../../context/tools/base-tool.js'; import type { ParsedTargetTable } from '../../../parsed-target-table.js'; import { stagedLookerQuerySchema } from '../types.js'; @@ -9,7 +9,7 @@ const lookerUsageInputSchema = z.object({ uniqueUsers30d: z.number().int().nonnegative().default(0), }); -export const lookerQueryToSlInputSchema = z.object({ +const lookerQueryToSlInputSchema = z.object({ query: stagedLookerQuerySchema, contentTitle: z.string().min(1).optional(), contentType: z.enum(['look', 'dashboard_tile']).default('look'), @@ -20,17 +20,17 @@ export type LookerQueryToSlInput = z.input; type LookerTargetStatus = 'mapped' | 'unmapped' | 'unparseable' | 'missing_target_table'; -export interface LookerSlFieldProposal { +interface LookerSlFieldProposal { name: string; lookerField: string; } -export interface LookerSlMeasureProposal extends LookerSlFieldProposal { +interface LookerSlMeasureProposal extends LookerSlFieldProposal { expr: string; description: string; } -export interface LookerSlSegmentProposal { +interface LookerSlSegmentProposal { name: string; filters: Record; suggestedPredicate: string; @@ -89,6 +89,7 @@ function targetNotes(status: LookerTargetStatus, targetTable: ParsedTargetTable ]; } +/** @internal */ export function buildLookerSlProposal(raw: LookerQueryToSlInput): LookerSlProposal { const input = lookerQueryToSlInputSchema.parse(raw); const sourceName = `looker__${toSlName(input.query.model)}__${toSlName(input.query.view)}`; @@ -182,7 +183,7 @@ export function createLookerQueryToSlTool() { }); } -export function formatLookerSlProposal(proposal: LookerSlProposal): string { +function formatLookerSlProposal(proposal: LookerSlProposal): string { const lines = [ '## Looker query SL proposal', '', diff --git a/packages/cli/src/context/ingest/adapters/looker/types.ts b/packages/cli/src/context/ingest/adapters/looker/types.ts index cda4f91e..37ff362a 100644 --- a/packages/cli/src/context/ingest/adapters/looker/types.ts +++ b/packages/cli/src/context/ingest/adapters/looker/types.ts @@ -5,9 +5,9 @@ import { parsedTargetTableSchema } from '../../parsed-target-table.js'; const lookerIdSchema = z.union([z.string(), z.number().int()]).transform(String); const nullableLookerIdSchema = z.union([lookerIdSchema, z.null()]).default(null); -export const lookerConnectionIdSchema = z.string().min(1).regex(/^[A-Za-z0-9_-]+$/); +const lookerConnectionIdSchema = z.string().min(1).regex(/^[A-Za-z0-9_-]+$/); -export const lookerRuntimeCursorsSchema = z.object({ +const lookerRuntimeCursorsSchema = z.object({ dashboardsLastSyncedAt: z.iso.datetime().nullable().default(null), looksLastSyncedAt: z.iso.datetime().nullable().default(null), }); @@ -215,6 +215,7 @@ const stagedLookerFetchIssueKindSchema = z.enum([ 'lookml_connection_mismatch', ]); +/** @internal */ export const stagedLookerFetchIssueSchema = z.object({ rawPath: z.string().min(1), entityType: z.enum(['dashboard', 'look', 'explore', 'signals', 'lookml_models', 'looker_connection_mapping']), diff --git a/packages/cli/src/context/ingest/adapters/metabase/client-port.ts b/packages/cli/src/context/ingest/adapters/metabase/client-port.ts index a5fdb6ce..47d78283 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/client-port.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/client-port.ts @@ -69,7 +69,7 @@ export interface MetabaseCardSummary { collection_id?: number | 'root' | null; } -export interface MetabaseResultMetadataColumn { +interface MetabaseResultMetadataColumn { name: string; base_type: string; semantic_type?: string | null; @@ -79,7 +79,7 @@ export interface MetabaseResultMetadataColumn { field_ref?: unknown[] | null; } -export interface MetabaseParameter { +interface MetabaseParameter { id: string; name: string; type: string; @@ -103,7 +103,7 @@ export interface MetabaseTemplateTag { 'widget-type'?: string; } -export interface MetabaseResolvedTemplateTag { +interface MetabaseResolvedTemplateTag { name: string; type: string; cardReference?: number | null; diff --git a/packages/cli/src/context/ingest/adapters/metabase/client.ts b/packages/cli/src/context/ingest/adapters/metabase/client.ts index 1962bfe0..7b075991 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/client.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/client.ts @@ -91,6 +91,7 @@ class MetabaseApiError extends Error { * `[[`/`]]` literals in string values or regex predicates are preserved. Metabase's * grammar disallows nested optional blocks (per docs), so non-greedy matching is safe. */ +/** @internal */ export function stripOptionalClauses(sql: string): string { return sql.replace(/\[\[[\s\S]*?\]\]/g, (match) => (match.includes('{{') ? '' : match)); } @@ -163,6 +164,7 @@ function injectNativeSql(datasetQuery: MetabaseDatasetQuery, sql: string): Metab * format, number widgets need a string scalar, identifier/enum widgets accept `[string]`. * Sending `['placeholder']` for a date widget triggers a ClassCastException → HTTP 500. */ +/** @internal */ export function getDummyValueForWidgetType(widgetType: string | undefined): string | string[] { switch (widgetType) { case 'date/range': diff --git a/packages/cli/src/context/ingest/adapters/metabase/fanout-planner.ts b/packages/cli/src/context/ingest/adapters/metabase/fanout-planner.ts index 660e8100..b0916a4c 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/fanout-planner.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/fanout-planner.ts @@ -1,6 +1,6 @@ import { parseMetabasePullConfig, type MetabasePullConfig } from './types.js'; -export interface MetabaseFanoutMappingInput { +interface MetabaseFanoutMappingInput { metabaseDatabaseId: number; targetConnectionId: string | null; syncEnabled: boolean; diff --git a/packages/cli/src/context/ingest/adapters/metabase/local-metabase.adapter.test.ts b/packages/cli/src/context/ingest/adapters/metabase/local-metabase.adapter.test.ts index b25ea18b..c20a65ac 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/local-metabase.adapter.test.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/local-metabase.adapter.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import type { KtxProjectConnectionConfig } from '../../../project/index.js'; +import type { KtxProjectConnectionConfig } from '../../../../context/project/config.js'; import { metabaseRuntimeConfigFromLocalConnection } from './local-metabase.adapter.js'; describe('metabaseRuntimeConfigFromLocalConnection', () => { diff --git a/packages/cli/src/context/ingest/adapters/metabase/local-metabase.adapter.ts b/packages/cli/src/context/ingest/adapters/metabase/local-metabase.adapter.ts index 8d8d5f06..7cb3d843 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/local-metabase.adapter.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/local-metabase.adapter.ts @@ -1,5 +1,6 @@ -import type { KtxLocalProject, KtxProjectConnectionConfig } from '../../../project/index.js'; -import { ktxLocalStateDbPath } from '../../../project/index.js'; +import type { KtxLocalProject } from '../../../../context/project/project.js'; +import type { KtxProjectConnectionConfig } from '../../../../context/project/config.js'; +import { ktxLocalStateDbPath } from '../../../../context/project/local-state-db.js'; import { resolveKtxConfigReference } from '../../../core/config-reference.js'; import { DEFAULT_METABASE_CLIENT_CONFIG, diff --git a/packages/cli/src/context/ingest/adapters/metabase/local-source-state-store.test.ts b/packages/cli/src/context/ingest/adapters/metabase/local-source-state-store.test.ts index 4fdafba5..0225f398 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/local-source-state-store.test.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/local-source-state-store.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { buildDefaultKtxProjectConfig } from '../../../project/index.js'; +import { buildDefaultKtxProjectConfig } from '../../../../context/project/config.js'; import { connectionConfigSchema } from '../../../project/driver-schemas.js'; import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './local-source-state-store.js'; diff --git a/packages/cli/src/context/ingest/adapters/metabase/local-source-state-store.ts b/packages/cli/src/context/ingest/adapters/metabase/local-source-state-store.ts index f8026db6..3393b9f8 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/local-source-state-store.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/local-source-state-store.ts @@ -1,15 +1,12 @@ import { mkdirSync } from 'node:fs'; import { dirname } from 'node:path'; import Database from 'better-sqlite3'; -import { - parseMetabaseMappingBootstrap, - type KtxLocalProject, - type MetabaseMappingBootstrap, -} from '../../../project/index.js'; +import { parseMetabaseMappingBootstrap, type MetabaseMappingBootstrap } from '../../../../context/project/mappings-yaml-schema.js'; +import type { KtxLocalProject } from '../../../../context/project/project.js'; import type { DiscoveredMetabaseDatabase } from './mapping.js'; import type { MetabaseSourceState, MetabaseSourceStateReader, MetabaseSourceStateSelection } from './source-state-port.js'; -export type LocalMetabaseMappingSource = 'ktx.yaml' | 'refresh'; +type LocalMetabaseMappingSource = 'ktx.yaml' | 'refresh'; interface LocalMetabaseDiscoveryCacheOptions { dbPath: string; diff --git a/packages/cli/src/context/ingest/adapters/metabase/mapping.ts b/packages/cli/src/context/ingest/adapters/metabase/mapping.ts index 9dc75006..788ae43b 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/mapping.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/mapping.ts @@ -1,5 +1,6 @@ import type { MetabaseDatabase, MetabaseRuntimeClient } from './client-port.js'; +/** @internal */ export const METABASE_ENGINE_TO_CONNECTION_TYPE = { postgres: 'POSTGRESQL', bigquery: 'BIGQUERY', @@ -9,8 +10,6 @@ export const METABASE_ENGINE_TO_CONNECTION_TYPE = { mysql: 'MYSQL', } as const; -export type MetabaseMappedConnectionType = - (typeof METABASE_ENGINE_TO_CONNECTION_TYPE)[keyof typeof METABASE_ENGINE_TO_CONNECTION_TYPE]; export interface DiscoveredMetabaseDatabase { id: number; @@ -42,26 +41,31 @@ export interface KtxConnectionPhysicalInfo { [key: string]: unknown; } +/** @internal */ export interface PhysicalMismatchInput { mappingId: string; metabase: MappingPhysicalInfo; target: KtxConnectionPhysicalInfo; } +/** @internal */ export interface PhysicalMismatch { mappingId: string; reason: string; } +/** @internal */ export interface MappingRefreshReport { drift: MetabaseMappingDrift; physicalMismatches: PhysicalMismatch[]; } +/** @internal */ export type MetabaseMappingValidationResult = | { ok: true } | { ok: false; errors: Array<{ key: string; reason: string }> }; +/** @internal */ export interface AutoMatchCandidate { id: string; name: string; @@ -69,6 +73,7 @@ export interface AutoMatchCandidate { connection_params: unknown; } +/** @internal */ export interface AutoMatchResult { connectionId: string; connectionName: string; @@ -170,6 +175,7 @@ export function computeMetabaseMappingDrift(args: { return { unmappedDiscovered, staleMappings, inSync }; } +/** @internal */ export function validateMetabaseMappings(args: { mappings: Record; knownKtxConnectionIds: Set; @@ -236,6 +242,7 @@ export function validateMappingPhysicalMatch( return null; } +/** @internal */ export function computeMetabaseMappingPhysicalMismatches(inputs: PhysicalMismatchInput[]): PhysicalMismatch[] { const mismatches: PhysicalMismatch[] = []; for (const input of inputs) { @@ -247,6 +254,7 @@ export function computeMetabaseMappingPhysicalMismatches(inputs: PhysicalMismatc return mismatches; } +/** @internal */ export async function refreshMetabaseMapping(args: { client: Pick; currentMappings: Record; @@ -286,6 +294,7 @@ export async function refreshMetabaseMapping(args: { return { drift, physicalMismatches }; } +/** @internal */ export function findBestMatch(mapping: MappingPhysicalInfo, candidates: AutoMatchCandidate[]): AutoMatchResult | null { const engine = mapping.metabaseEngine?.toLowerCase(); if (!engine) { diff --git a/packages/cli/src/context/ingest/adapters/metabase/source-state-port.ts b/packages/cli/src/context/ingest/adapters/metabase/source-state-port.ts index 16de9369..8b50aae5 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/source-state-port.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/source-state-port.ts @@ -5,7 +5,7 @@ export interface MetabaseSourceStateSelection { metabaseObjectId: number; } -export interface MetabaseSourceStateMapping { +interface MetabaseSourceStateMapping { metabaseDatabaseId: number; metabaseDatabaseName: string | null; metabaseEngine: string | null; diff --git a/packages/cli/src/context/ingest/adapters/metabase/types.ts b/packages/cli/src/context/ingest/adapters/metabase/types.ts index 1021b212..da84de0c 100644 --- a/packages/cli/src/context/ingest/adapters/metabase/types.ts +++ b/packages/cli/src/context/ingest/adapters/metabase/types.ts @@ -3,7 +3,7 @@ import { z } from 'zod'; const metabaseSyncModeSchema = z.enum(['ALL', 'ONLY', 'EXCEPT']); export type MetabaseSyncMode = z.infer; -export const metabaseLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/); +const metabaseLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/); /** * The lean config the adapter needs at `fetch()` time. Lives in the BullMQ payload's @@ -11,6 +11,7 @@ export const metabaseLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a- * job — the persisted state (enabled/disabled, auth, scheduling) lives on the * Metabase connection's `connections.config` JSONB. */ +/** @internal */ export const metabasePullConfigSchema = z.object({ /** The Metabase connection (source) — the thing being swept. */ metabaseConnectionId: metabaseLocalConnectionIdSchema, diff --git a/packages/cli/src/context/ingest/adapters/metricflow/deep-parse.ts b/packages/cli/src/context/ingest/adapters/metricflow/deep-parse.ts index a76e94f7..8c10d638 100644 --- a/packages/cli/src/context/ingest/adapters/metricflow/deep-parse.ts +++ b/packages/cli/src/context/ingest/adapters/metricflow/deep-parse.ts @@ -1,7 +1,7 @@ import { parse as parseYaml } from 'yaml'; -import { noopLogger, type KtxLogger } from '../../../core/index.js'; +import { noopLogger, type KtxLogger } from '../../../../context/core/config.js'; -export interface DimensionDefinition { +interface DimensionDefinition { name: string; column: string; type: string; @@ -9,7 +9,7 @@ export interface DimensionDefinition { description?: string; } -export interface SimpleMeasureDefinition { +interface SimpleMeasureDefinition { type: 'simple'; name: string; column: string; @@ -20,7 +20,7 @@ export interface SimpleMeasureDefinition { cumulative?: boolean; } -export type MeasureDefinition = +type MeasureDefinition = | SimpleMeasureDefinition | { type: 'derived'; @@ -186,6 +186,7 @@ export function parseMetricflowFiles( return parser.parseFiles(files); } +/** @internal */ export function translateMetricflowJinjaFilter(filter: string): string { return new MetricflowDeepParser(noopLogger).translateJinjaFilter(filter); } diff --git a/packages/cli/src/context/ingest/adapters/metricflow/import-semantic-models.ts b/packages/cli/src/context/ingest/adapters/metricflow/import-semantic-models.ts index 7d80a9c0..9aa6181d 100644 --- a/packages/cli/src/context/ingest/adapters/metricflow/import-semantic-models.ts +++ b/packages/cli/src/context/ingest/adapters/metricflow/import-semantic-models.ts @@ -1,10 +1,6 @@ -import type { SemanticLayerService, SemanticLayerSource } from '../../../sl/index.js'; -import { - addTouchedSlSource, - createTouchedSlSources, - listTouchedSlSources, - type TouchedSlSource, -} from '../../../tools/index.js'; +import type { SemanticLayerService } from '../../../../context/sl/semantic-layer.service.js'; +import type { SemanticLayerSource } from '../../../../context/sl/types.js'; +import { addTouchedSlSource, createTouchedSlSources, listTouchedSlSources, type TouchedSlSource } from '../../../../context/tools/touched-sl-sources.js'; import type { MetricFlowParseResult } from './deep-parse.js'; import { buildMetricflowJoinsForModel, @@ -29,12 +25,12 @@ export interface MetricFlowImportResult { touchedSources: TouchedSlSource[]; } -export type MetricflowSemanticLayerWriter = Pick< +type MetricflowSemanticLayerWriter = Pick< SemanticLayerService, 'getManifestEntry' | 'isManifestBacked' | 'loadAllSources' | 'loadSource' | 'writeSource' >; -export type MetricflowSemanticLayerService = MetricflowSemanticLayerWriter & { +type MetricflowSemanticLayerService = MetricflowSemanticLayerWriter & { forWorktree(workdir: string): MetricflowSemanticLayerWriter; }; diff --git a/packages/cli/src/context/ingest/adapters/metricflow/pull-config.ts b/packages/cli/src/context/ingest/adapters/metricflow/pull-config.ts index ad38b033..8fccbeee 100644 --- a/packages/cli/src/context/ingest/adapters/metricflow/pull-config.ts +++ b/packages/cli/src/context/ingest/adapters/metricflow/pull-config.ts @@ -1,7 +1,7 @@ import { z } from 'zod'; import { parsedTargetTableSchema } from '../../parsed-target-table.js'; -export const metricflowPullConfigSchema = z.object({ +const metricflowPullConfigSchema = z.object({ repoUrl: z.string().url(), branch: z.string().default('main'), path: z.string().nullable().default(null), diff --git a/packages/cli/src/context/ingest/adapters/metricflow/semantic-models.test.ts b/packages/cli/src/context/ingest/adapters/metricflow/semantic-models.test.ts index 9ad88563..c22ac97d 100644 --- a/packages/cli/src/context/ingest/adapters/metricflow/semantic-models.test.ts +++ b/packages/cli/src/context/ingest/adapters/metricflow/semantic-models.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { composeOverlay, type SemanticLayerSource } from '../../../sl/index.js'; +import { composeOverlay } from '../../../../context/sl/semantic-layer.service.js'; +import type { SemanticLayerSource } from '../../../../context/sl/types.js'; import type { ParsedCrossModelMetric, ParsedMetricflowRelationship, ParsedSemanticModel } from './deep-parse.js'; import { buildMetricflowColumns, diff --git a/packages/cli/src/context/ingest/adapters/metricflow/semantic-models.ts b/packages/cli/src/context/ingest/adapters/metricflow/semantic-models.ts index f2f84c61..7137bf0d 100644 --- a/packages/cli/src/context/ingest/adapters/metricflow/semantic-models.ts +++ b/packages/cli/src/context/ingest/adapters/metricflow/semantic-models.ts @@ -1,4 +1,4 @@ -import type { SemanticLayerSource } from '../../../sl/index.js'; +import type { SemanticLayerSource } from '../../../../context/sl/types.js'; import type { ParsedCrossModelMetric, ParsedMetricflowRelationship, @@ -25,6 +25,7 @@ export type MetricflowSemanticModelJoin = SemanticLayerSource['joins'][number]; export type MetricflowWritableSemanticLayerSource = Pick & Partial>; +/** @internal */ export function toKebabCaseMetricflowName(str: string): string { return str .toLowerCase() @@ -32,6 +33,7 @@ export function toKebabCaseMetricflowName(str: string): string { .replace(/^-|-$/g, ''); } +/** @internal */ export function mapSemanticModelToSource(model: ParsedSemanticModel, tableRef?: string): SemanticLayerSource { return { name: toKebabCaseMetricflowName(model.modelRef), @@ -177,7 +179,7 @@ export function buildMetricflowSemanticModelSource( return mapMetricflowSemanticModelToStandalone(model, sourceName, matchedTable?.name ?? model.modelRef, joins); } -export function buildMetricflowMeasures(model: ParsedSemanticModel): SemanticLayerSource['measures'] { +function buildMetricflowMeasures(model: ParsedSemanticModel): SemanticLayerSource['measures'] { return model.measures.map((measure) => { if (measure.type === 'simple') { return { @@ -195,6 +197,7 @@ export function buildMetricflowMeasures(model: ParsedSemanticModel): SemanticLay }); } +/** @internal */ export function buildMetricflowColumns(model: ParsedSemanticModel): SemanticLayerSource['columns'] { const columns: SemanticLayerSource['columns'] = model.dimensions.map((dimension) => ({ name: dimension.column, @@ -243,6 +246,7 @@ export function getMetricflowAvailableColumnNames(context: MetricflowSemanticMod return new Set(columns.map((column) => column.name.toLowerCase())); } +/** @internal */ export function countImportableMetricflowRelationships( relationships: ParsedMetricflowRelationship[], hostTables: MetricflowHostTable[], @@ -336,10 +340,11 @@ function mergeMetricflowJoins( return [...baseJoins, ...newJoins]; } -export function normalizeMetricflowJoinOn(on: string): string { +function normalizeMetricflowJoinOn(on: string): string { return on.replace(/\s+/g, ' ').trim(); } +/** @internal */ export function rewriteMetricflowManifestJoins( joins: SemanticLayerSource['joins'], sourceNameByManifestName: Map, @@ -351,7 +356,7 @@ export function rewriteMetricflowManifestJoins( })); } -export function rewriteMetricflowJoinOn(on: string, sourceNameByManifestName: Map): string { +function rewriteMetricflowJoinOn(on: string, sourceNameByManifestName: Map): string { const parts = on.split('='); if (parts.length !== 2) { return on; @@ -366,7 +371,7 @@ export function rewriteMetricflowJoinOn(on: string, sourceNameByManifestName: Ma return `${leftTable}.${left.column} = ${rightTable}.${right.column}`; } -export function parseMetricflowJoinReference(ref: string): { table: string; column: string } | null { +function parseMetricflowJoinReference(ref: string): { table: string; column: string } | null { const lastDot = ref.lastIndexOf('.'); if (lastDot <= 0 || lastDot === ref.length - 1) { return null; diff --git a/packages/cli/src/context/ingest/adapters/notion/chunk.ts b/packages/cli/src/context/ingest/adapters/notion/chunk.ts index bdae8622..10f5d98d 100644 --- a/packages/cli/src/context/ingest/adapters/notion/chunk.ts +++ b/packages/cli/src/context/ingest/adapters/notion/chunk.ts @@ -5,6 +5,7 @@ import type { ChunkResult, DiffSet, ScopeDescriptor, WorkUnit } from '../../type import { notionManifestSchema, notionMetadataSchema } from './types.js'; const MAX_NOTION_WORK_UNIT_CHARS = 40_000; +/** @internal */ export const NOTION_ORG_KNOWLEDGE_WARNING = 'Anything accessible to this Notion integration can become organization knowledge.'; const NOTION_SL_WRITE_GUIDANCE = diff --git a/packages/cli/src/context/ingest/artifact-gates.ts b/packages/cli/src/context/ingest/artifact-gates.ts index 44f7a66f..35735575 100644 --- a/packages/cli/src/context/ingest/artifact-gates.ts +++ b/packages/cli/src/context/ingest/artifact-gates.ts @@ -1,10 +1,10 @@ -import type { SemanticLayerService } from '../sl/index.js'; -import type { TouchedSlSource } from '../tools/index.js'; -import type { KnowledgeWikiService } from '../wiki/index.js'; +import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js'; +import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; import { findMissingWikiRefs } from '../wiki/wiki-ref-validation.js'; import { findInvalidWikiBodyRefs } from './wiki-body-refs.js'; -export interface TouchedValidationResult { +interface TouchedValidationResult { invalidSources: string[]; validSources: string[]; } diff --git a/packages/cli/src/context/ingest/context-candidates/candidate-dedup.service.ts b/packages/cli/src/context/ingest/context-candidates/candidate-dedup.service.ts index e9c0e878..d7f1a07b 100644 --- a/packages/cli/src/context/ingest/context-candidates/candidate-dedup.service.ts +++ b/packages/cli/src/context/ingest/context-candidates/candidate-dedup.service.ts @@ -1,4 +1,4 @@ -import { type KtxLogger, noopLogger } from '../../core/index.js'; +import { type KtxLogger, noopLogger } from '../../../context/core/config.js'; import type { CandidateDedupResult, ContextCandidateForDedup, JsonValue } from '../ports.js'; import { buildContextCandidateEmbeddingText } from './embedding-text.js'; import type { ContextCandidateStorePort } from './store.js'; diff --git a/packages/cli/src/context/ingest/context-candidates/context-candidate-carryforward.service.ts b/packages/cli/src/context/ingest/context-candidates/context-candidate-carryforward.service.ts index 74151e7e..19ec9f39 100644 --- a/packages/cli/src/context/ingest/context-candidates/context-candidate-carryforward.service.ts +++ b/packages/cli/src/context/ingest/context-candidates/context-candidate-carryforward.service.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { type KtxLogger, noopLogger } from '../../core/index.js'; +import { type KtxLogger, noopLogger } from '../../../context/core/config.js'; import type { JsonValue } from '../ports.js'; import type { ContextCandidateStorePort } from './store.js'; import type { diff --git a/packages/cli/src/context/ingest/context-candidates/curator-pagination.service.ts b/packages/cli/src/context/ingest/context-candidates/curator-pagination.service.ts index bd883553..348544ca 100644 --- a/packages/cli/src/context/ingest/context-candidates/curator-pagination.service.ts +++ b/packages/cli/src/context/ingest/context-candidates/curator-pagination.service.ts @@ -1,7 +1,7 @@ -import type { KtxModelRole } from '../../../llm/index.js'; -import { type KtxLogger, noopLogger } from '../../core/index.js'; -import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../llm/index.js'; -import type { MemoryAction } from '../../memory/index.js'; +import type { KtxModelRole } from '../../../llm/types.js'; +import { type KtxLogger, noopLogger } from '../../../context/core/config.js'; +import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; +import type { MemoryAction } from '../../../context/memory/types.js'; import type { ContextCandidateForDedup, CuratorPaginationPort, CuratorPaginationReport } from '../ports.js'; import type { ReconcileCandidateForPrompt, diff --git a/packages/cli/src/context/ingest/context-candidates/index.ts b/packages/cli/src/context/ingest/context-candidates/index.ts deleted file mode 100644 index 7c8e0f78..00000000 --- a/packages/cli/src/context/ingest/context-candidates/index.ts +++ /dev/null @@ -1,29 +0,0 @@ -export type { CandidateDedupServiceDeps } from './candidate-dedup.service.js'; -export { CandidateDedupService } from './candidate-dedup.service.js'; -export type { - ContextCandidateCarryforwardArgs, - ContextCandidateCarryforwardResult, - ContextCandidateCarryforwardServiceDeps, -} from './context-candidate-carryforward.service.js'; -export { ContextCandidateCarryforwardService } from './context-candidate-carryforward.service.js'; -export type { CuratorPaginationInput, CuratorPaginationServiceDeps } from './curator-pagination.service.js'; -export { CuratorPaginationService } from './curator-pagination.service.js'; -export { buildContextCandidateEmbeddingText } from './embedding-text.js'; -export type { ContextCandidateStorePort } from './store.js'; -export type { - BudgetExhaustedCandidateForCarryForward, - CandidateDedupSettings, - ContextCandidateActionHint, - ContextCandidateCarryforwardSettings, - ContextCandidateEmbeddingPort, - ContextCandidateForPrompt, - ContextCandidateLane, - ContextCandidateRejectionReason, - ContextCandidateScoreAggregation, - ContextCandidateStatus, - ContextCandidateVerdictSummary, - CuratorPaginationSettings, - CurrentRunEvidenceChunkForCarryForward, - InsertContextCandidateInput, - MarkContextCandidateClusterInput, -} from './types.js'; diff --git a/packages/cli/src/context/ingest/context-candidates/types.ts b/packages/cli/src/context/ingest/context-candidates/types.ts index 2f8b7db2..c2e3c3c4 100644 --- a/packages/cli/src/context/ingest/context-candidates/types.ts +++ b/packages/cli/src/context/ingest/context-candidates/types.ts @@ -1,7 +1,7 @@ import type { JsonValue } from '../ports.js'; -export type ContextCandidateActionHint = 'create' | 'update' | 'merge' | 'conflict' | 'skip'; -export type ContextCandidateStatus = 'pending' | 'promoted' | 'merged' | 'rejected' | 'conflict'; +type ContextCandidateActionHint = 'create' | 'update' | 'merge' | 'conflict' | 'skip'; +type ContextCandidateStatus = 'pending' | 'promoted' | 'merged' | 'rejected' | 'conflict'; export type ContextCandidateRejectionReason = | 'low_score' | 'duplicates_existing_wiki' @@ -10,20 +10,9 @@ export type ContextCandidateRejectionReason = | 'exceeded_run_budget' | 'exceeded_curator_passes' | 'curator_pass_error'; -export type ContextCandidateLane = 'light' | 'full' | null; -export type ContextCandidateScoreAggregation = 'max' | 'mean' | 'sum'; +type ContextCandidateLane = 'light' | 'full' | null; +type ContextCandidateScoreAggregation = 'max' | 'mean' | 'sum'; -export interface ContextCandidateForPrompt { - candidateKey: string; - topic: string; - assertion: string; - rationale: string; - actionHint: string; - status: string; - promotionScore: number; - suggestedPageKey: string | null; - evidenceRefs: JsonValue; -} export interface ContextCandidateVerdictSummary { pending: number; diff --git a/packages/cli/src/context/ingest/context-evidence/context-evidence-index.service.ts b/packages/cli/src/context/ingest/context-evidence/context-evidence-index.service.ts index 7b54e80f..fe79fef7 100644 --- a/packages/cli/src/context/ingest/context-evidence/context-evidence-index.service.ts +++ b/packages/cli/src/context/ingest/context-evidence/context-evidence-index.service.ts @@ -1,7 +1,7 @@ import { createHash } from 'node:crypto'; import { readdir, readFile } from 'node:fs/promises'; import { basename, dirname, join, relative } from 'node:path'; -import { noopLogger, type KtxLogger } from '../../core/index.js'; +import { noopLogger, type KtxLogger } from '../../../context/core/config.js'; import type { JsonValue } from '../ports.js'; import type { DiffSet } from '../types.js'; import type { ContextEvidenceIndexStorePort } from './store.js'; diff --git a/packages/cli/src/context/ingest/context-evidence/index.ts b/packages/cli/src/context/ingest/context-evidence/index.ts deleted file mode 100644 index 99be0177..00000000 --- a/packages/cli/src/context/ingest/context-evidence/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -export { ContextEvidenceIndexService } from './context-evidence-index.service.js'; -export { SqliteContextEvidenceStore } from './sqlite-context-evidence-store.js'; -export type { - ContextEvidenceDocumentRef, - ContextEvidenceEmbeddingPort, - ContextEvidenceIndexSummary, - EvidencePublishState, - ReplaceContextEvidenceChunk, - UpsertContextEvidenceDocument, -} from './types.js'; -export type { ContextEvidenceIndexStorePort } from './store.js'; -export type { SqliteContextEvidenceStoreOptions } from './sqlite-context-evidence-store.js'; diff --git a/packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.test.ts b/packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.test.ts index 0767a6be..6b575c00 100644 --- a/packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.test.ts +++ b/packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import type { InsertContextCandidateInput } from '../context-candidates/index.js'; +import type { InsertContextCandidateInput } from '../../../context/ingest/context-candidates/types.js'; import type { JsonValue } from '../ports.js'; import { SqliteContextEvidenceStore } from './sqlite-context-evidence-store.js'; diff --git a/packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.ts b/packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.ts index 5d18394f..adba683c 100644 --- a/packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.ts +++ b/packages/cli/src/context/ingest/context-evidence/sqlite-context-evidence-store.ts @@ -2,7 +2,8 @@ import { randomUUID } from 'node:crypto'; import { mkdirSync } from 'node:fs'; import { dirname } from 'node:path'; import Database from 'better-sqlite3'; -import { HybridSearchCore, type SearchCandidateGenerator, type SearchLaneBreakdown } from '../../search/index.js'; +import { HybridSearchCore } from '../../../context/search/hybrid-search-core.js'; +import type { SearchCandidateGenerator, SearchLaneBreakdown } from '../../../context/search/types.js'; import type { ContextCandidateStatusResult, ContextEvidenceChunkForCandidate, @@ -14,16 +15,9 @@ import type { ContextEvidenceSearchResult, ContextEvidenceToolStorePort, } from '../../tools/context-evidence-tool-store.js'; -import type { - BudgetExhaustedCandidateForCarryForward, - ContextCandidateRejectionReason, - ContextCandidateStorePort, - ContextCandidateVerdictSummary, - CurrentRunEvidenceChunkForCarryForward, - InsertContextCandidateInput, - MarkContextCandidateClusterInput, -} from '../context-candidates/index.js'; -import type { PageTriageEvidenceChunk, PageTriageStorePort } from '../page-triage/index.js'; +import type { BudgetExhaustedCandidateForCarryForward, ContextCandidateRejectionReason, ContextCandidateVerdictSummary, CurrentRunEvidenceChunkForCarryForward, InsertContextCandidateInput, MarkContextCandidateClusterInput } from '../../../context/ingest/context-candidates/types.js'; +import type { ContextCandidateStorePort } from '../../../context/ingest/context-candidates/store.js'; +import type { PageTriageEvidenceChunk, PageTriageStorePort } from '../../../context/ingest/page-triage/page-triage.service.js'; import type { ContextCandidateForDedup, ContextCandidateSummary, JsonValue } from '../ports.js'; import type { ContextEvidenceIndexStorePort } from './store.js'; import type { diff --git a/packages/cli/src/context/ingest/dbt-shared/project-vars.ts b/packages/cli/src/context/ingest/dbt-shared/project-vars.ts index 2900e08f..3750e246 100644 --- a/packages/cli/src/context/ingest/dbt-shared/project-vars.ts +++ b/packages/cli/src/context/ingest/dbt-shared/project-vars.ts @@ -18,6 +18,7 @@ export interface ResolveJinjaVariablesResult { unresolvedVars: string[]; } +/** @internal */ export function parseProjectVars(yamlContent: string): Map { const variables = new Map(); const project = parseProjectYaml(yamlContent); @@ -30,6 +31,7 @@ export function parseProjectVars(yamlContent: string): Map { return variables; } +/** @internal */ export function parseProjectName(yamlContent: string): string | null { const project = parseProjectYaml(yamlContent); diff --git a/packages/cli/src/context/ingest/dbt-shared/schema-files.ts b/packages/cli/src/context/ingest/dbt-shared/schema-files.ts index 6c223b83..946d8226 100644 --- a/packages/cli/src/context/ingest/dbt-shared/schema-files.ts +++ b/packages/cli/src/context/ingest/dbt-shared/schema-files.ts @@ -22,6 +22,7 @@ export async function loadDbtSchemaFiles(projectDir: string): Promise { const schemaFiles: string[] = []; diff --git a/packages/cli/src/context/ingest/final-gate-repair.ts b/packages/cli/src/context/ingest/final-gate-repair.ts index 57ff1619..1c373aa6 100644 --- a/packages/cli/src/context/ingest/final-gate-repair.ts +++ b/packages/cli/src/context/ingest/final-gate-repair.ts @@ -1,8 +1,8 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { z } from 'zod'; -import type { AgentRunnerPort, KtxRuntimeToolSet } from '../llm/index.js'; -import type { TouchedSlSource } from '../tools/index.js'; +import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; +import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js'; import type { IngestTraceWriter } from './ingest-trace.js'; import { traceTimed } from './ingest-trace.js'; diff --git a/packages/cli/src/context/ingest/finalization-scope.ts b/packages/cli/src/context/ingest/finalization-scope.ts index 6ecdc83e..b5ace2b9 100644 --- a/packages/cli/src/context/ingest/finalization-scope.ts +++ b/packages/cli/src/context/ingest/finalization-scope.ts @@ -1,5 +1,5 @@ -import type { SemanticLayerSource } from '../sl/index.js'; -import type { TouchedSlSource } from '../tools/index.js'; +import type { SemanticLayerSource } from '../../context/sl/types.js'; +import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js'; import type { IngestReportFinalizationMismatch } from './reports.js'; interface DeriveTouchedSourcesInput { diff --git a/packages/cli/src/context/ingest/index.ts b/packages/cli/src/context/ingest/index.ts deleted file mode 100644 index d8e9c856..00000000 --- a/packages/cli/src/context/ingest/index.ts +++ /dev/null @@ -1,670 +0,0 @@ -export { DbtSourceAdapter } from './adapters/dbt/dbt.adapter.js'; -export { FakeSourceAdapter } from './adapters/fake/fake.adapter.js'; -export type { - DaemonLiveDatabaseIntrospectionOptions, - KtxDaemonDatabaseHttpJsonRunner, - KtxDaemonDatabaseIntrospectionCommand, - KtxDaemonDatabaseJsonRunner, -} from './adapters/live-database/daemon-introspection.js'; -export { createDaemonLiveDatabaseIntrospection } from './adapters/live-database/daemon-introspection.js'; -export type { - LiveDatabaseExtractedColumn, - LiveDatabaseExtractedForeignKey, - LiveDatabaseExtractedSchema, - LiveDatabaseExtractedTable, -} from './adapters/live-database/extracted-schema.js'; -export { - buildLiveDatabaseTableNaturalKey, - ktxSchemaSnapshotToExtractedSchema, -} from './adapters/live-database/extracted-schema.js'; -export { - assertSemanticLayerTargetPathsAllowed, - findDisallowedSemanticLayerTargetPaths, - semanticLayerConnectionIdFromPath, -} from './semantic-layer-target-policy.js'; -export { LiveDatabaseSourceAdapter } from './adapters/live-database/live-database.adapter.js'; -export type { - BuildLiveDatabaseManifestShardsInput, - BuildLiveDatabaseManifestShardsResult, - LiveDatabaseManifestColumn, - LiveDatabaseManifestExistingDescriptions, - LiveDatabaseManifestJoinData, - LiveDatabaseManifestJoinEntry, - LiveDatabaseManifestShard, - LiveDatabaseManifestTableData, - LiveDatabaseManifestTableEntry, -} from './adapters/live-database/manifest.js'; -export { buildLiveDatabaseManifestShards } from './adapters/live-database/manifest.js'; -export type { - LiveDatabaseStructuralChanges, - LiveDatabaseStructuralSyncOperations, - LiveDatabaseStructuralSyncPlan, - LiveDatabaseStructuralSyncStats, - LiveDatabaseSyncedColumn, - LiveDatabaseSyncedLink, - LiveDatabaseSyncedSchema, - LiveDatabaseSyncedTable, - PlanLiveDatabaseStructuralSyncInput, -} from './adapters/live-database/structural-sync.js'; -export { planLiveDatabaseStructuralSync } from './adapters/live-database/structural-sync.js'; -export type { - LiveDatabaseIntrospectionPort, - LiveDatabaseSourceAdapterDeps, -} from './adapters/live-database/types.js'; -export { getLookerTriageSignals, writeLookerEvidenceDocuments } from './adapters/looker/evidence-documents.js'; -export { LookerClient } from './adapters/looker/client.js'; -export type { - LookerClientDeps, - LookerClientLogger, - LookerConnectionParams, - LookerSdkPort, - LookerWarehouseConnectionInfo, - TestConnectionResult as LookerTestConnectionResult, -} from './adapters/looker/client.js'; -export type { - LookerClientFactory, - LookerEntityRef, - LookerRuntimeClient, -} from './adapters/looker/fetch.js'; -export { - DefaultLookerClientFactory, - DefaultLookerConnectionClientFactory, -} from './adapters/looker/factory.js'; -export { - createDaemonLookerTableIdentifierParser, - type DaemonLookerTableIdentifierParserOptions, - type KtxDaemonTableIdentifierHttpJsonRunner, -} from './adapters/looker/daemon-table-identifier-parser.js'; -export type { - LookerConnectionClientFactory, - LookerCredentialResolver, -} from './adapters/looker/factory.js'; -export { - createLocalLookerCredentialResolver, - createLocalLookerSourceAdapter, - lookerCredentialsFromLocalConnection, -} from './adapters/looker/local-looker.adapter.js'; -export { - LocalLookerRuntimeStore, - type ClearLocalLookerMappingsInput, - type LocalLookerConnectionMappingListRow, - type LocalLookerMappingSource, - type LookerSourceStateReader, - type RefreshLocalLookerDiscoveredConnectionsInput, - type UpsertLocalLookerConnectionMappingInput, -} from './adapters/looker/local-runtime-store.js'; -export { - LOOKER_DIALECT_TO_CONNECTION_TYPE, - buildLookerPullConfigFromInputs, - collectExploreParseItems, - computeLookerMappingDrift, - discoverLookerConnections, - extractWarehouseDatabase, - extractWarehouseHost, - lookerDialectToConnectionType, - normalizeHost, - normalizeName, - projectParsedIdentifier, - refreshLookerMappingPlaceholders, - sqlglotDialectForConnectionType, - suggestKtxConnectionForLookerConnection, - validateLookerMappings, - validateLookerWarehouseTarget, -} from './adapters/looker/mapping.js'; -export type { - LookerConnectionMapping as KtxLookerConnectionMapping, - LookerMappingCandidateConnection, - LookerMappingClient, - LookerMappingDrift, - LookerMappingValidationResult, - LookerParsedIdentifier, - LookerTableIdentifierParseItem, - LookerTableIdentifierParser, - LookerTargetConnection, - LookerWarehouseTargetConnectionType, -} from './adapters/looker/mapping.js'; -export { - readLookerFetchReport, - writeLookerFetchReport, -} from './adapters/looker/fetch-report.js'; -export { LookerSourceAdapter, type LookerSourceAdapterDeps } from './adapters/looker/looker.adapter.js'; -export { - describeLookerScope, - hashLookerScope, - isPathInLookerScope, - readLookerScope, -} from './adapters/looker/scope.js'; -export type { - LookerQueryToSlInput, - LookerSlFieldProposal, - LookerSlMeasureProposal, - LookerSlProposal, - LookerSlSegmentProposal, -} from './adapters/looker/tools/looker-query-to-sl.tool.js'; -export { - buildLookerSlProposal, - createLookerQueryToSlTool, - formatLookerSlProposal, - lookerQueryToSlInputSchema, -} from './adapters/looker/tools/looker-query-to-sl.tool.js'; -export type { - LookerPullConfig, - LookerRuntimeCursors, - StagedDashboardFile, - StagedExploreFile, - StagedFoldersTreeFile, - StagedGroupFile, - StagedLookerFetchIssue, - StagedLookerFetchReport, - StagedLookerQuery, - StagedLookerScopeFile, - StagedLookerSignalsFile, - StagedLookFile, - StagedLookmlModelsFile, - StagedUserFile, -} from './adapters/looker/types.js'; -export { - lookerConnectionIdSchema, - lookerRuntimeCursorsSchema, - stagedLookerFetchIssueSchema, - stagedLookerFetchReportSchema, - stagedLookerScopeFileSchema, - stagedSyncConfigSchema, -} from './adapters/looker/types.js'; -export { LookmlSourceAdapter } from './adapters/lookml/lookml.adapter.js'; -export { parseLookmlStagedDir } from './adapters/lookml/parse.js'; -export type { ParsedLookmlProject } from './adapters/lookml/parse.js'; -export { - DEFAULT_METABASE_CLIENT_CONFIG, - DefaultMetabaseConnectionClientFactory, - MetabaseClient, - getDummyValueForWidgetType, - stripOptionalClauses, -} from './adapters/metabase/client.js'; -export { CardReferenceCycleError, expandCardReferences } from './adapters/metabase/card-references.js'; -export { IngestMetabaseClientFactory } from './adapters/metabase/client-port.js'; -export type { MetabaseClientLogger } from './adapters/metabase/client.js'; -export type { - MetabaseCard, - MetabaseCardSummary, - MetabaseClientConfig, - MetabaseClientFactory, - MetabaseClientRuntimeConfig, - MetabaseCollection, - MetabaseCollectionItem, - MetabaseConnectionClientFactory, - MetabaseDatabase, - MetabaseDatasetQuery, - MetabaseNativeQueryResult, - MetabaseParameter, - MetabaseResolvedTemplateTag, - MetabaseResultMetadataColumn, - MetabaseRuntimeClient, - MetabaseTemplateTag, - MetabaseUser, - ResolvedSqlResult, - TestConnectionResult, -} from './adapters/metabase/client-port.js'; -export type { - MetabaseSourceState, - MetabaseSourceStateMapping, - MetabaseSourceStateReader, - MetabaseSourceStateSelection, -} from './adapters/metabase/source-state-port.js'; -export { - METABASE_ENGINE_TO_CONNECTION_TYPE, - computeMetabaseMappingDrift, - computeMetabaseMappingPhysicalMismatches, - discoverMetabaseDatabases, - findBestMatch, - refreshMetabaseMapping, - validateMappingPhysicalMatch, - validateMetabaseMappings, -} from './adapters/metabase/mapping.js'; -export type { - AutoMatchCandidate, - AutoMatchResult as MetabaseAutoMatchResult, - DiscoveredMetabaseDatabase, - KtxConnectionPhysicalInfo, - MappingPhysicalInfo, - MappingRefreshReport, - MetabaseMappedConnectionType, - MetabaseMappingDrift, - MetabaseMappingValidationResult, - PhysicalMismatch, - PhysicalMismatchInput, -} from './adapters/metabase/mapping.js'; -export { planMetabaseFanoutChildren } from './adapters/metabase/fanout-planner.js'; -export type { - MetabaseFanoutChildPlan, - MetabaseFanoutMappingInput, - PlanMetabaseFanoutChildrenInput, -} from './adapters/metabase/fanout-planner.js'; -export { MetabaseSourceAdapter } from './adapters/metabase/metabase.adapter.js'; -export { - createLocalMetabaseSourceAdapter, - metabaseRuntimeConfigFromLocalConnection, -} from './adapters/metabase/local-metabase.adapter.js'; -export { - KtxYamlMetabaseSourceStateReader, - LocalMetabaseDiscoveryCache, -} from './adapters/metabase/local-source-state-store.js'; -export type { - LocalMetabaseDiscoveredDatabaseRow, - LocalMetabaseMappingListRow, - LocalMetabaseMappingSource, - RefreshLocalMetabaseDiscoveredDatabasesInput, -} from './adapters/metabase/local-source-state-store.js'; -export { metabaseLocalConnectionIdSchema, metabasePullConfigSchema, parseMetabasePullConfig } from './adapters/metabase/types.js'; -export type { MetabasePullConfig, MetabaseSyncMode } from './adapters/metabase/types.js'; -export { - fetchMetricflowRepo, -} from './adapters/metricflow/fetch.js'; -export type { FetchMetricflowRepoParams, FetchMetricflowRepoResult } from './adapters/metricflow/fetch.js'; -export { - parseMetricflowFiles, - translateMetricflowJinjaFilter, -} from './adapters/metricflow/deep-parse.js'; -export type { - DimensionDefinition, - MeasureDefinition, - MetricFlowParseResult, - MetricflowParseOptions, - ParsedCrossModelMetric, - ParsedMetricflowRelationship, - ParsedSemanticModel, - SimpleMeasureDefinition, -} from './adapters/metricflow/deep-parse.js'; -export { - buildMetricflowColumns, - buildMetricflowJoinsForModel, - buildMetricflowMeasures, - buildMetricflowSemanticModelSource, - countImportableMetricflowRelationships, - filterValidMetricflowRelationships, - findMatchingMetricflowTable, - getMetricflowAvailableColumnNames, - mapCrossModelMetricToSource, - mapSemanticModelToSource, - normalizeMetricflowJoinOn, - parseMetricflowJoinReference, - resolveMetricflowSemanticModelSourceName, - rewriteMetricflowJoinOn, - rewriteMetricflowManifestJoins, - toKebabCaseMetricflowName, -} from './adapters/metricflow/semantic-models.js'; -export { importMetricflowSemanticModels } from './adapters/metricflow/import-semantic-models.js'; -export type { - ImportMetricflowSemanticModelsDeps, - ImportMetricflowSemanticModelsInput, - MetricFlowImportResult, - MetricflowSemanticLayerService, - MetricflowSemanticLayerWriter, -} from './adapters/metricflow/import-semantic-models.js'; -export type { - MetricflowHostTable, - MetricflowSemanticModelImportContext, - MetricflowSemanticModelJoin, - MetricflowWritableSemanticLayerSource, -} from './adapters/metricflow/semantic-models.js'; -export { MetricflowSourceAdapter, type MetricflowSourceAdapterDeps } from './adapters/metricflow/metricflow.adapter.js'; -export { - metricflowPullConfigSchema, - parseMetricflowPullConfig, - pullConfigFromMetricflowIntegration, -} from './adapters/metricflow/pull-config.js'; -export type { - MetricflowIntegrationLike, - MetricflowPullConfig, -} from './adapters/metricflow/pull-config.js'; -export { NOTION_ORG_KNOWLEDGE_WARNING } from './adapters/notion/chunk.js'; -export { NOTION_DEFAULT_MAX_KNOWLEDGE_CREATES_PER_RUN } from './adapters/notion/types.js'; -export { LocalNotionRuntimeStore } from './adapters/notion/local-state-store.js'; -export { NotionSourceAdapter, type NotionSourceAdapterDeps } from './adapters/notion/notion.adapter.js'; -export { NotionClient, type NotionApi, type NotionBotInfo } from './adapters/notion/notion-client.js'; -export { bucketDistinctUsers, bucketErrorRate, bucketExecutions, bucketP95Runtime, bucketRecency } from './adapters/historic-sql/buckets.js'; -export { chunkHistoricSqlUnifiedStagedDir, describeHistoricSqlUnifiedScope } from './adapters/historic-sql/chunk-unified.js'; -export { detectHistoricSqlStagedDir } from './adapters/historic-sql/detect.js'; -export { - HistoricSqlExtensionMissingError, - HistoricSqlGrantsMissingError, - HistoricSqlVersionUnsupportedError, -} from './adapters/historic-sql/errors.js'; -export { HistoricSqlSourceAdapter } from './adapters/historic-sql/historic-sql.adapter.js'; -export { BigQueryHistoricSqlQueryHistoryReader } from './adapters/historic-sql/bigquery-query-history-reader.js'; -export type { BigQueryHistoricSqlQueryHistoryReaderOptions } from './adapters/historic-sql/bigquery-query-history-reader.js'; -export { PostgresPgssReader } from './adapters/historic-sql/postgres-pgss-reader.js'; -export { SnowflakeHistoricSqlQueryHistoryReader } from './adapters/historic-sql/snowflake-query-history-reader.js'; -export { stageHistoricSqlAggregatedSnapshot } from './adapters/historic-sql/stage-unified.js'; -export { - historicSqlEvidenceEnvelopeSchema, - historicSqlEvidencePath, - historicSqlPatternEvidenceSchema, - historicSqlTableUsageEvidenceSchema, - serializeHistoricSqlEvidence, -} from './adapters/historic-sql/evidence.js'; -export type { - HistoricSqlEvidenceEnvelope, - HistoricSqlPatternEvidence, - HistoricSqlTableUsageEvidence, -} from './adapters/historic-sql/evidence.js'; -export { createEmitHistoricSqlEvidenceTool } from './adapters/historic-sql/evidence-tool.js'; -export { projectHistoricSqlEvidence } from './adapters/historic-sql/projection.js'; -export type { HistoricSqlProjectionInput, HistoricSqlProjectionResult } from './adapters/historic-sql/projection.js'; -export { - patternOutputSchema, - patternsArraySchema, - tableUsageOutputSchema, -} from './adapters/historic-sql/skill-schemas.js'; -export type { - PatternOutput, - TableUsageOutput, -} from './adapters/historic-sql/skill-schemas.js'; -export type { - AggregatedTemplate, - HistoricSqlDialect, - HistoricSqlProbeResult, - HistoricSqlReader, - HistoricSqlSourceAdapterDeps, - HistoricSqlTimeWindow, - HistoricSqlUnifiedPullConfig, - KtxPostgresQueryClient, - PostgresPgssProbeResult, - StagedManifest, - StagedPatternsInput, - StagedTableInput, -} from './adapters/historic-sql/types.js'; -export { - HISTORIC_SQL_SOURCE_KEY, - aggregatedTemplateSchema, - historicSqlUnifiedPullConfigSchema, - stagedManifestSchema, - stagedPatternsInputSchema, - stagedTableInputSchema, -} from './adapters/historic-sql/types.js'; -export type { CanonicalPin } from './canonical-pins.js'; -export { buildCanonicalPinsPromptBlock, selectRelevantCanonicalPins } from './canonical-pins.js'; -export type { - BudgetExhaustedCandidateForCarryForward, - CandidateDedupServiceDeps, - CandidateDedupSettings, - ContextCandidateActionHint, - ContextCandidateCarryforwardArgs, - ContextCandidateCarryforwardResult, - ContextCandidateCarryforwardServiceDeps, - ContextCandidateCarryforwardSettings, - ContextCandidateEmbeddingPort, - ContextCandidateForPrompt, - ContextCandidateLane, - ContextCandidateRejectionReason, - ContextCandidateScoreAggregation, - ContextCandidateStatus, - ContextCandidateStorePort, - ContextCandidateVerdictSummary, - CuratorPaginationInput, - CuratorPaginationServiceDeps, - CuratorPaginationSettings, - CurrentRunEvidenceChunkForCarryForward, - InsertContextCandidateInput, - MarkContextCandidateClusterInput, -} from './context-candidates/index.js'; -export { - buildContextCandidateEmbeddingText, - CandidateDedupService, - ContextCandidateCarryforwardService, - CuratorPaginationService, -} from './context-candidates/index.js'; -export type { - ContextEvidenceDocumentRef, - ContextEvidenceEmbeddingPort, - ContextEvidenceIndexStorePort, - ContextEvidenceIndexSummary as PackageContextEvidenceIndexSummary, - EvidencePublishState, - ReplaceContextEvidenceChunk, - SqliteContextEvidenceStoreOptions, - UpsertContextEvidenceDocument, -} from './context-evidence/index.js'; -export { - ContextEvidenceIndexService, - SqliteContextEvidenceStore, -} from './context-evidence/index.js'; -export { DiffSetService } from './diff-set.service.js'; -export { IngestBundleRunner } from './ingest-bundle.runner.js'; -export type { DefaultLocalIngestAdaptersOptions } from './local-adapters.js'; -export { createDefaultLocalIngestAdapters, localPullConfigForAdapter } from './local-adapters.js'; -export type { - LocalIngestMcpOptions, - LocalIngestResult, - LocalMetabaseFanoutChild, - LocalMetabaseFanoutProgress, - LocalMetabaseFanoutProgressChild, - LocalMetabaseFanoutResult, - RunLocalIngestOptions, - RunLocalMetabaseIngestOptions, -} from './local-ingest.js'; -export { getLatestLocalIngestStatus, getLocalIngestStatus, runLocalIngest, runLocalMetabaseIngest } from './local-ingest.js'; -export { seedLocalMappingStateFromKtxYaml } from './local-mapping-reconcile.js'; -export type { - CreateLocalBundleIngestRuntimeOptions, - LocalBundleIngestRuntime, -} from './local-bundle-runtime.js'; -export { createLocalBundleIngestRuntime } from './local-bundle-runtime.js'; -export type { - LocalIngestDiffPaths, - LocalIngestRunRecord, - LocalIngestStatus, - RunLocalStageOnlyIngestOptions, -} from './local-stage-ingest.js'; -export { getLocalStageOnlyIngestStatus, runLocalStageOnlyIngest } from './local-stage-ingest.js'; -export { - ingestReportToMemoryFlowReplay, - localIngestRunToMemoryFlowReplay, -} from './memory-flow/events.js'; -export { - buildAuthenticatedUrl, - cleanupRepoDir, - cloneOrPull, - RepoConfigError, - RepoFetchError, - repoDirExists, - sanitizeRepoError, - testRepoConnection, - validateRepoConfig, -} from './repo-fetch.js'; -export type { RepoFetchConfig } from './repo-fetch.js'; -export { - loadProjectInfo, - parseProjectName, - parseProjectVars, - resolveJinjaVariables, -} from './dbt-shared/project-vars.js'; -export type { DbtProjectInfo, ResolveJinjaVariablesResult } from './dbt-shared/project-vars.js'; -export { findDbtSchemaFiles, loadDbtSchemaFiles } from './dbt-shared/schema-files.js'; -export { - computeDbtSchemaHash, - parseDbtSchemaFile, - parseDbtSchemaFiles, -} from './adapters/dbt-descriptions/parse-schema.js'; -export type { - DbtParsedColumn, - DbtColumnConstraints, - DbtDataTestRef, - DbtParsedRelationship, - DbtParsedTable, - DbtSchemaFile, - DbtSchemaParseResult, -} from './adapters/dbt-descriptions/parse-schema.js'; -export { findMatchingKtxTable, matchDbtTables } from './adapters/dbt-descriptions/match-tables.js'; -export type { DbtHostTableLite, DbtTableMatch } from './adapters/dbt-descriptions/match-tables.js'; -export { toDescriptionUpdates } from './adapters/dbt-descriptions/to-description-updates.js'; -export type { DbtDescriptionUpdates } from './adapters/dbt-descriptions/to-description-updates.js'; -export { toRelationshipUpdates } from './adapters/dbt-descriptions/to-relationship-updates.js'; -export type { DbtRelationshipUpdates } from './adapters/dbt-descriptions/to-relationship-updates.js'; -export { toMetadataUpdates } from './adapters/dbt-descriptions/to-metadata-updates.js'; -export { mergeSemanticModelTables } from './adapters/dbt-descriptions/merge-semantic-model-tables.js'; -export type { KtxJoinUpdate, KtxMetadataUpdate } from '../scan/enrichment-types.js'; -export { - createInitialMemoryFlowInteractionState, - findMemoryFlowSearchMatches, - reduceMemoryFlowInteractionState, - selectedMemoryFlowColumn, - selectedMemoryFlowDetails, - selectMemoryFlowChip, - selectMemoryFlowColumn, - visibleMemoryFlowChips, -} from './memory-flow/interaction.js'; -export { renderMemoryFlowInteractive } from './memory-flow/interactive-render.js'; -export { createMemoryFlowLiveBuffer, sanitizeMemoryFlowError } from './memory-flow/live-buffer.js'; -export { renderMemoryFlowReplay } from './memory-flow/render.js'; -export { formatMemoryFlowFinalSummary } from './memory-flow/summary.js'; -export type { MemoryFlowStreamEvent } from './memory-flow/schema.js'; -export { - memoryFlowActionDetailSchema, - memoryFlowDetailSectionsSchema, - memoryFlowEventSchema, - memoryFlowPlannedWorkUnitSchema, - memoryFlowReplayInputSchema, - memoryFlowRunStatusSchema, - memoryFlowStreamEventSchema, - parseMemoryFlowReplayInput, -} from './memory-flow/schema.js'; -export type { - MemoryFlowChip, - MemoryFlowColumnId, - MemoryFlowColumnView, - MemoryFlowDisplayStatus, - MemoryFlowEvent, - MemoryFlowEventSink, - MemoryFlowFilterMode, - MemoryFlowInteractionCommand, - MemoryFlowInteractionState, - MemoryFlowLiveBufferOptions, - MemoryFlowPaneId, - MemoryFlowPlannedWorkUnit, - MemoryFlowRenderOptions, - MemoryFlowReplayInput, - MemoryFlowReplayPatch, - MemoryFlowRunStatus, - MemoryFlowViewModel, -} from './memory-flow/types.js'; -export { buildMemoryFlowViewModel } from './memory-flow/view-model.js'; -export type { - MemoryFlowStatusBadge, - MemoryFlowVisualColumn, - MemoryFlowVisualModel, -} from './memory-flow/visuals.js'; -export { - buildMemoryFlowVisualModel, - memoryFlowStatusBadge, - renderMemoryFlowConnectorLine, -} from './memory-flow/visuals.js'; -export type { - PageTriageEvidenceChunk, - PageTriageReport, - PageTriageRunArgs, - PageTriageServiceDeps, - PageTriageSettings, - PageTriageStorePort, -} from './page-triage/index.js'; -export { PageTriageService } from './page-triage/index.js'; -export type { - CandidateDedupPort, - CandidateDedupResult, - ContextCandidateCarryforwardPort, - ContextCandidateForDedup, - ContextCandidateSummary, - ContextEvidenceCandidatesPort, - ContextEvidenceIndexPort, - ContextEvidenceIndexSummary, - CreateIngestRunArgs, - CuratorPaginationPort, - CuratorPaginationReport, - DiffSetComputerPort, - IngestBundleRunnerDeps, - IngestCanonicalPinsPort, - IngestCommitMessagePort, - IngestFileStorePort, - IngestGitAuthor, - IngestKnowledgeIndexPort, - IngestLockPort, - IngestProvenanceInsert, - IngestProvenancePort, - IngestProvenanceRow, - IngestReportsPort, - IngestRunnerJob, - IngestRunRecord, - IngestRunsPort, - IngestSessionWorktree, - IngestSessionWorktreePort, - IngestSettingsPort, - IngestStoragePort, - IngestToolsetFactoryPort, - IngestToolsetLike, - PageTriagePort, - PageTriageRunResult, - ProvenanceActionType, - SourceAdapterRegistryPort, -} from './ports.js'; -export { - buildSyncId, - provenanceMarker, - rawSourcesDirForSync, - rawSourcesRoot, -} from './raw-sources-paths.js'; -export { ingestReportSnapshotSchema, parseIngestReportSnapshot } from './report-snapshot.js'; -export type { IngestReportBody, IngestReportSnapshot } from './reports.js'; -export * from './artifact-gates.js'; -export * from './ingest-trace.js'; -export * from './isolated-diff/git-patch.js'; -export * from './isolated-diff/patch-integrator.js'; -export * from './isolated-diff/work-unit-executor.js'; -export * from './reports.js'; -export { SourceAdapterRegistry } from './source-adapter-registry.js'; -export type { SqliteBundleIngestStoreOptions } from './sqlite-bundle-ingest-store.js'; -export { SqliteBundleIngestStore } from './sqlite-bundle-ingest-store.js'; -export type { - SaveCompletedLocalIngestRunInput, - SqliteLocalIngestStoreOptions, -} from './sqlite-local-ingest-store.js'; -export { SqliteLocalIngestStore } from './sqlite-local-ingest-store.js'; -export type { - ReconcileCandidateForPrompt, - ReconcileCandidateSummary, - ReconcilePromptRunState, - WikiPageRef, -} from './stages/build-reconcile-context.js'; -export { - buildReconcileSystemPrompt, - buildReconcileToolSet, - buildReconcileUserPrompt, -} from './stages/build-reconcile-context.js'; -export type { ReconciliationOutcome } from './stages/stage-4-reconciliation.js'; -export { runReconciliationStage4 } from './stages/stage-4-reconciliation.js'; -export type { StageIndex } from './stages/stage-index.types.js'; -export type { - ChunkResult, - DiffSet, - EvictionUnit, - FetchContext, - IngestBundleJob, - IngestBundleRef, - IngestBundleResult, - IngestDiffSummary, - IngestJobContext, - IngestJobPhase, - IngestTrigger, - ScopeDescriptor, - SourceAdapter, - SourceFetchIssue, - SourceFetchReport, - TriageLane, - TriageSignals, - UnresolvedCardInfo, - WorkUnit, - DeterministicProjectionContext, - ProjectionResult, - DeterministicFinalizationContext, - FinalizationOverrideReplay, - FinalizationResult, -} from './types.js'; -export * from './wiki-body-refs.js'; diff --git a/packages/cli/src/context/ingest/ingest-bundle.runner.isolated-diff.test.ts b/packages/cli/src/context/ingest/ingest-bundle.runner.isolated-diff.test.ts index fee60600..0201b881 100644 --- a/packages/cli/src/context/ingest/ingest-bundle.runner.isolated-diff.test.ts +++ b/packages/cli/src/context/ingest/ingest-bundle.runner.isolated-diff.test.ts @@ -2,9 +2,10 @@ import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promis import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it, vi } from 'vitest'; -import { GitService, SessionWorktreeService } from '../core/index.js'; +import { GitService } from '../../context/core/git.service.js'; +import { SessionWorktreeService } from '../../context/core/session-worktree.service.js'; import { LocalGitFileStore } from '../project/local-git-file-store.js'; -import { addTouchedSlSource } from '../tools/index.js'; +import { addTouchedSlSource } from '../../context/tools/touched-sl-sources.js'; import { IngestBundleRunner } from './ingest-bundle.runner.js'; import type { IngestBundleRunnerDeps } from './ports.js'; diff --git a/packages/cli/src/context/ingest/ingest-bundle.runner.test.ts b/packages/cli/src/context/ingest/ingest-bundle.runner.test.ts index ae6f4c14..85f45049 100644 --- a/packages/cli/src/context/ingest/ingest-bundle.runner.test.ts +++ b/packages/cli/src/context/ingest/ingest-bundle.runner.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { addTouchedSlSource } from '../tools/index.js'; +import { addTouchedSlSource } from '../../context/tools/touched-sl-sources.js'; import { IngestBundleRunner } from './ingest-bundle.runner.js'; import { createMemoryFlowLiveBuffer } from './memory-flow/live-buffer.js'; import type { MemoryFlowReplayInput } from './memory-flow/types.js'; diff --git a/packages/cli/src/context/ingest/ingest-bundle.runner.ts b/packages/cli/src/context/ingest/ingest-bundle.runner.ts index 43a2b251..510e88d0 100644 --- a/packages/cli/src/context/ingest/ingest-bundle.runner.ts +++ b/packages/cli/src/context/ingest/ingest-bundle.runner.ts @@ -2,12 +2,17 @@ import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import pLimit from 'p-limit'; import { z } from 'zod'; -import { type KtxLogger, noopLogger } from '../core/index.js'; -import { createRuntimeToolDescriptorFromAiTool, type KtxRuntimeToolSet } from '../llm/index.js'; -import type { CaptureSession, MemoryAction } from '../memory/index.js'; -import type { SemanticLayerService, SemanticLayerSource, SlValidationDeps } from '../sl/index.js'; -import { createTouchedSlSources, type ToolContext, type ToolSession, type TouchedSlSource } from '../tools/index.js'; -import type { KnowledgeWikiService } from '../wiki/index.js'; +import { type KtxLogger, noopLogger } from '../../context/core/config.js'; +import { createRuntimeToolDescriptorFromAiTool } from '../../context/llm/runtime-tools.js'; +import type { KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; +import type { CaptureSession, MemoryAction } from '../../context/memory/types.js'; +import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import type { SemanticLayerSource } from '../../context/sl/types.js'; +import type { SlValidationDeps } from '../../context/sl/tools/sl-warehouse-validation.js'; +import { createTouchedSlSources, type TouchedSlSource } from '../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../context/tools/base-tool.js'; +import type { ToolSession } from '../../context/tools/tool-session.js'; +import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; import { findDanglingWikiRefsForActions } from '../wiki/wiki-ref-validation.js'; import { actionTargetConnectionId } from './action-identity.js'; import { NOTION_DEFAULT_MAX_KNOWLEDGE_CREATES_PER_RUN } from './adapters/notion/types.js'; diff --git a/packages/cli/src/context/ingest/ingest-runtime-assets.test.ts b/packages/cli/src/context/ingest/ingest-runtime-assets.test.ts index 6b9d83ba..f6a46111 100644 --- a/packages/cli/src/context/ingest/ingest-runtime-assets.test.ts +++ b/packages/cli/src/context/ingest/ingest-runtime-assets.test.ts @@ -2,8 +2,8 @@ import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; -import { PromptService } from '../prompts/index.js'; -import { SkillsRegistryService } from '../skills/index.js'; +import { PromptService } from '../../context/prompts/prompt.service.js'; +import { SkillsRegistryService } from '../../context/skills/skills-registry.service.js'; const promptsDir = fileURLToPath(new URL('../../prompts', import.meta.url)); const skillsDir = fileURLToPath(new URL('../../skills', import.meta.url)); diff --git a/packages/cli/src/context/ingest/ingest-trace.ts b/packages/cli/src/context/ingest/ingest-trace.ts index eed0cfd5..bac114be 100644 --- a/packages/cli/src/context/ingest/ingest-trace.ts +++ b/packages/cli/src/context/ingest/ingest-trace.ts @@ -20,7 +20,7 @@ export interface IngestTraceContext { level?: IngestTraceLevel; } -export interface IngestTraceEvent { +interface IngestTraceEvent { schemaVersion: 1; at: string; level: IngestTraceLevel; @@ -121,22 +121,6 @@ export class FileIngestTraceWriter implements IngestTraceWriter { } } -export class NoopIngestTraceWriter implements IngestTraceWriter { - readonly tracePath = ''; - readonly context: IngestTraceContext = { - tracePath: '', - jobId: '', - connectionId: '', - sourceKey: '', - level: 'error', - }; - - withContext(): IngestTraceWriter { - return this; - } - - async event(): Promise {} -} export async function traceTimed( trace: IngestTraceWriter, diff --git a/packages/cli/src/context/ingest/isolated-diff/git-patch.ts b/packages/cli/src/context/ingest/isolated-diff/git-patch.ts index ee7f0020..e0250b11 100644 --- a/packages/cli/src/context/ingest/isolated-diff/git-patch.ts +++ b/packages/cli/src/context/ingest/isolated-diff/git-patch.ts @@ -1,5 +1,6 @@ import { assertSemanticLayerTargetPathsAllowed } from '../semantic-layer-target-policy.js'; +/** @internal */ export const textArtifactRoots = ['wiki/', 'semantic-layer/'] as const; export interface PatchTouchedPath { diff --git a/packages/cli/src/context/ingest/isolated-diff/patch-integrator.test.ts b/packages/cli/src/context/ingest/isolated-diff/patch-integrator.test.ts index d55cfc5b..e547e22e 100644 --- a/packages/cli/src/context/ingest/isolated-diff/patch-integrator.test.ts +++ b/packages/cli/src/context/ingest/isolated-diff/patch-integrator.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it, vi } from 'vitest'; -import { GitService } from '../../core/index.js'; +import { GitService } from '../../../context/core/git.service.js'; import { FileIngestTraceWriter } from '../ingest-trace.js'; import { integrateWorkUnitPatch } from './patch-integrator.js'; diff --git a/packages/cli/src/context/ingest/isolated-diff/patch-integrator.ts b/packages/cli/src/context/ingest/isolated-diff/patch-integrator.ts index a4542576..869c019e 100644 --- a/packages/cli/src/context/ingest/isolated-diff/patch-integrator.ts +++ b/packages/cli/src/context/ingest/isolated-diff/patch-integrator.ts @@ -1,12 +1,12 @@ import { readFile } from 'node:fs/promises'; -import type { GitService } from '../../core/index.js'; +import type { GitService } from '../../../context/core/git.service.js'; import type { FinalGateRepairResult } from '../final-gate-repair.js'; import type { IngestTraceWriter } from '../ingest-trace.js'; import { traceTimed } from '../ingest-trace.js'; import { assertPatchAllowedForWorkUnit, parsePatchTouchedPaths } from './git-patch.js'; import type { TextualConflictResolutionResult } from './textual-conflict-resolver.js'; -export type PatchIntegrationTextualResolution = +type PatchIntegrationTextualResolution = | { status: 'repaired'; attempts: number; changedPaths: string[] } | { status: 'failed'; attempts: number; reason: string }; diff --git a/packages/cli/src/context/ingest/isolated-diff/textual-conflict-resolver.ts b/packages/cli/src/context/ingest/isolated-diff/textual-conflict-resolver.ts index c5128291..5ae551d1 100644 --- a/packages/cli/src/context/ingest/isolated-diff/textual-conflict-resolver.ts +++ b/packages/cli/src/context/ingest/isolated-diff/textual-conflict-resolver.ts @@ -1,7 +1,7 @@ import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { z } from 'zod'; -import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../llm/index.js'; +import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; import type { IngestTraceWriter } from '../ingest-trace.js'; import { traceTimed } from '../ingest-trace.js'; diff --git a/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.test.ts b/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.test.ts index 6f8a7599..5975dee8 100644 --- a/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.test.ts +++ b/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it, vi } from 'vitest'; -import { GitService } from '../../core/index.js'; +import { GitService } from '../../../context/core/git.service.js'; import { FileIngestTraceWriter } from '../ingest-trace.js'; import { runIsolatedWorkUnit } from './work-unit-executor.js'; diff --git a/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.ts b/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.ts index ac013d5a..81e6edfa 100644 --- a/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.ts +++ b/packages/cli/src/context/ingest/isolated-diff/work-unit-executor.ts @@ -1,6 +1,6 @@ import { mkdir, readFile } from 'node:fs/promises'; import { join } from 'node:path'; -import type { SessionOutcome } from '../../core/index.js'; +import type { SessionOutcome } from '../../../context/core/session-worktree.service.js'; import type { IngestSessionWorktree, IngestSessionWorktreePort } from '../ports.js'; import type { WorkUnit } from '../types.js'; import type { IngestTraceWriter } from '../ingest-trace.js'; diff --git a/packages/cli/src/context/ingest/local-adapters.test.ts b/packages/cli/src/context/ingest/local-adapters.test.ts index a4e9eea6..5f61c739 100644 --- a/packages/cli/src/context/ingest/local-adapters.test.ts +++ b/packages/cli/src/context/ingest/local-adapters.test.ts @@ -2,8 +2,8 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js'; -import type { SqlAnalysisPort } from '../sql-analysis/index.js'; +import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js'; +import type { SqlAnalysisPort } from '../../context/sql-analysis/ports.js'; import type { HistoricSqlReader } from './adapters/historic-sql/types.js'; import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js'; import { LocalNotionRuntimeStore } from './adapters/notion/local-state-store.js'; diff --git a/packages/cli/src/context/ingest/local-adapters.ts b/packages/cli/src/context/ingest/local-adapters.ts index 2b13327a..ea7556e7 100644 --- a/packages/cli/src/context/ingest/local-adapters.ts +++ b/packages/cli/src/context/ingest/local-adapters.ts @@ -1,8 +1,10 @@ import { join } from 'node:path'; -import { localConnectionToWarehouseDescriptor, notionConnectionToPullConfig, parseNotionConnectionConfig } from '../connections/index.js'; +import { localConnectionToWarehouseDescriptor } from '../../context/connections/local-warehouse-descriptor.js'; +import { notionConnectionToPullConfig, parseNotionConnectionConfig } from '../../context/connections/notion-config.js'; import { resolveKtxConfigReference } from '../core/config-reference.js'; -import { ktxLocalStateDbPath, type KtxLocalProject } from '../project/index.js'; -import type { SqlAnalysisPort } from '../sql-analysis/index.js'; +import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import type { SqlAnalysisPort } from '../../context/sql-analysis/ports.js'; import { DbtSourceAdapter } from './adapters/dbt/dbt.adapter.js'; import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js'; import { HistoricSqlSourceAdapter } from './adapters/historic-sql/historic-sql.adapter.js'; diff --git a/packages/cli/src/context/ingest/local-bundle-ingest.test.ts b/packages/cli/src/context/ingest/local-bundle-ingest.test.ts index 5a336f35..4b4b834c 100644 --- a/packages/cli/src/context/ingest/local-bundle-ingest.test.ts +++ b/packages/cli/src/context/ingest/local-bundle-ingest.test.ts @@ -3,8 +3,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import Database from 'better-sqlite3'; import YAML from 'yaml'; -import type { AgentRunnerPort, RunLoopParams } from '../llm/index.js'; -import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js'; +import type { AgentRunnerPort, RunLoopParams } from '../../context/llm/runtime-port.js'; +import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js'; import { makeLocalGitRepo } from '../test/make-local-git-repo.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js'; diff --git a/packages/cli/src/context/ingest/local-bundle-runtime.test.ts b/packages/cli/src/context/ingest/local-bundle-runtime.test.ts index 89dd2ce5..db0e5e11 100644 --- a/packages/cli/src/context/ingest/local-bundle-runtime.test.ts +++ b/packages/cli/src/context/ingest/local-bundle-runtime.test.ts @@ -1,8 +1,8 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import type { AgentRunnerPort } from '../llm/index.js'; -import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js'; +import type { AgentRunnerPort } from '../../context/llm/runtime-port.js'; +import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js'; import { createLocalBundleIngestRuntime } from './local-bundle-runtime.js'; diff --git a/packages/cli/src/context/ingest/local-bundle-runtime.ts b/packages/cli/src/context/ingest/local-bundle-runtime.ts index 41c2fa55..73e152cb 100644 --- a/packages/cli/src/context/ingest/local-bundle-runtime.ts +++ b/packages/cli/src/context/ingest/local-bundle-runtime.ts @@ -2,82 +2,64 @@ import { mkdirSync } from 'node:fs'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import YAML from 'yaml'; -import { localConnectionInfoFromConfig, type KtxSqlQueryExecutorPort } from '../connections/index.js'; -import type { KtxEmbeddingPort, KtxLogger } from '../core/index.js'; -import { noopLogger, SessionWorktreeService } from '../core/index.js'; -import type { KtxSemanticLayerComputePort } from '../daemon/index.js'; -import { - createRuntimeToolDescriptorFromAiTool, - createLocalKtxLlmRuntimeFromConfig, - KtxIngestEmbeddingPortAdapter, - RuntimeAgentRunner, - type AgentRunnerPort, - type KtxLlmRuntimePort, - type KtxRuntimeToolSet, -} from '../llm/index.js'; -import type { KtxEmbeddingProvider } from '../../llm/index.js'; -import type { KtxLocalProject } from '../project/index.js'; -import { ktxLocalStateDbPath } from '../project/index.js'; -import { PromptService } from '../prompts/index.js'; -import { SkillsRegistryService } from '../skills/index.js'; -import { - type KtxConnectionInfo, - type KtxQueryResult, - SemanticLayerService, - type SlConnectionCatalogPort, - SlDiscoverTool, - SlEditSourceTool, - type SlPythonPort, - SlReadSourceTool, - SlRollbackTool, - SlSearchService, - type SlSourcesIndexPort, - SlValidateTool, - type SlValidationDeps, - type SlValidatorPort, - SlWriteSourceTool, - SqliteSlSourcesIndex, - sourceDefinitionSchema, - sourceOverlaySchema, -} from '../sl/index.js'; -import { - BaseTool, - ContextCandidateMarkTool, - ContextCandidateWriteTool, - ContextEvidenceNeighborsTool, - ContextEvidenceReadTool, - ContextEvidenceSearchTool, - type GitAuthorResolverPort, - type ToolContext, - type ToolSession, -} from '../tools/index.js'; -import { - buildKnowledgeSearchText, - type KnowledgeEventPort, - type KnowledgeIndexPort, - type KnowledgeIndexPageListing, - KnowledgeWikiService, - searchLocalKnowledgePages, - SqliteKnowledgeIndex, - type SqliteKnowledgeIndexPage, - WikiListTagsTool, - WikiReadTool, - WikiRemoveTool, - WikiSearchTool, - WikiWriteTool, -} from '../wiki/index.js'; -import { - CandidateDedupService, - ContextCandidateCarryforwardService, - CuratorPaginationService, -} from './context-candidates/index.js'; +import { localConnectionInfoFromConfig } from '../../context/connections/local-warehouse-descriptor.js'; +import type { KtxSqlQueryExecutorPort } from '../../context/connections/query-executor.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxLogger } from '../../context/core/config.js'; +import { noopLogger } from '../../context/core/config.js'; +import { SessionWorktreeService } from '../../context/core/session-worktree.service.js'; +import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js'; +import { createRuntimeToolDescriptorFromAiTool } from '../../context/llm/runtime-tools.js'; +import { createLocalKtxLlmRuntimeFromConfig } from '../../context/llm/local-config.js'; +import { KtxIngestEmbeddingPortAdapter } from '../../context/llm/embedding-port.js'; +import { RuntimeAgentRunner, type AgentRunnerPort, type KtxLlmRuntimePort, type KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; +import type { KtxEmbeddingProvider } from '../../llm/types.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js'; +import { PromptService } from '../../context/prompts/prompt.service.js'; +import { SkillsRegistryService } from '../../context/skills/skills-registry.service.js'; +import type { KtxConnectionInfo, KtxQueryResult, SlConnectionCatalogPort, SlPythonPort, SlSourcesIndexPort } from '../../context/sl/ports.js'; +import { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import { SlDiscoverTool } from '../../context/sl/tools/sl-discover.tool.js'; +import { SlEditSourceTool } from '../../context/sl/tools/sl-edit-source.tool.js'; +import { SlReadSourceTool } from '../../context/sl/tools/sl-read-source.tool.js'; +import { SlRollbackTool } from '../../context/sl/tools/sl-rollback.tool.js'; +import { SlSearchService } from '../../context/sl/sl-search.service.js'; +import { SlValidateTool } from '../../context/sl/tools/sl-validate.tool.js'; +import type { SlValidationDeps } from '../../context/sl/tools/sl-warehouse-validation.js'; +import type { SlValidatorPort } from '../../context/sl/sl-validator.port.js'; +import { SlWriteSourceTool } from '../../context/sl/tools/sl-write-source.tool.js'; +import { SqliteSlSourcesIndex } from '../../context/sl/sqlite-sl-sources-index.js'; +import { sourceDefinitionSchema, sourceOverlaySchema } from '../../context/sl/schemas.js'; +import { BaseTool, type ToolContext } from '../../context/tools/base-tool.js'; +import { ContextCandidateMarkTool } from '../../context/tools/context-candidate-mark.tool.js'; +import { ContextCandidateWriteTool } from '../../context/tools/context-candidate-write.tool.js'; +import { ContextEvidenceNeighborsTool } from '../../context/tools/context-evidence-neighbors.tool.js'; +import { ContextEvidenceReadTool } from '../../context/tools/context-evidence-read.tool.js'; +import { ContextEvidenceSearchTool } from '../../context/tools/context-evidence-search.tool.js'; +import type { GitAuthorResolverPort } from '../../context/tools/authors.js'; +import type { ToolSession } from '../../context/tools/tool-session.js'; +import { buildKnowledgeSearchText } from '../../context/wiki/knowledge-search-text.js'; +import type { KnowledgeEventPort, KnowledgeIndexPort, KnowledgeIndexPageListing } from '../../context/wiki/ports.js'; +import { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; +import { searchLocalKnowledgePages } from '../../context/wiki/local-knowledge.js'; +import { SqliteKnowledgeIndex, type SqliteKnowledgeIndexPage } from '../../context/wiki/sqlite-knowledge-index.js'; +import { WikiListTagsTool } from '../../context/wiki/tools/wiki-list-tags.tool.js'; +import { WikiReadTool } from '../../context/wiki/tools/wiki-read.tool.js'; +import { WikiRemoveTool } from '../../context/wiki/tools/wiki-remove.tool.js'; +import { WikiSearchTool } from '../../context/wiki/tools/wiki-search.tool.js'; +import { WikiWriteTool } from '../../context/wiki/tools/wiki-write.tool.js'; +import { CandidateDedupService } from '../../context/ingest/context-candidates/candidate-dedup.service.js'; +import { ContextCandidateCarryforwardService } from '../../context/ingest/context-candidates/context-candidate-carryforward.service.js'; +import { CuratorPaginationService } from '../../context/ingest/context-candidates/curator-pagination.service.js'; import { createEmitHistoricSqlEvidenceTool } from './adapters/historic-sql/evidence-tool.js'; -import { ContextEvidenceIndexService, SqliteContextEvidenceStore } from './context-evidence/index.js'; +import { ContextEvidenceIndexService } from '../../context/ingest/context-evidence/context-evidence-index.service.js'; +import { SqliteContextEvidenceStore } from '../../context/ingest/context-evidence/sqlite-context-evidence-store.js'; import { DiffSetService } from './diff-set.service.js'; import { ingestTracePathForJob, type IngestTraceLevel } from './ingest-trace.js'; import { IngestBundleRunner } from './ingest-bundle.runner.js'; -import { PageTriageService } from './page-triage/index.js'; -import { createWarehouseVerificationTools } from './tools/warehouse-verification/index.js'; +import { PageTriageService } from '../../context/ingest/page-triage/page-triage.service.js'; +import { createWarehouseVerificationTools } from '../../context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.js'; import type { IngestBundleRunnerDeps, IngestCommitMessagePort, diff --git a/packages/cli/src/context/ingest/local-ingest.ts b/packages/cli/src/context/ingest/local-ingest.ts index 044d41de..2832d9ff 100644 --- a/packages/cli/src/context/ingest/local-ingest.ts +++ b/packages/cli/src/context/ingest/local-ingest.ts @@ -1,12 +1,12 @@ import { randomUUID } from 'node:crypto'; import { cp, mkdir, rm } from 'node:fs/promises'; import { isAbsolute, resolve } from 'node:path'; -import type { KtxSqlQueryExecutorPort } from '../connections/index.js'; -import type { KtxLogger } from '../core/index.js'; -import type { KtxSemanticLayerComputePort } from '../daemon/index.js'; -import type { AgentRunnerPort, KtxLlmRuntimePort } from '../llm/index.js'; -import type { KtxLocalProject } from '../project/index.js'; -import { ktxLocalStateDbPath } from '../project/index.js'; +import type { KtxSqlQueryExecutorPort } from '../../context/connections/query-executor.js'; +import type { KtxLogger } from '../../context/core/config.js'; +import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js'; +import type { AgentRunnerPort, KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js'; import { planMetabaseFanoutChildren } from './adapters/metabase/fanout-planner.js'; import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './adapters/metabase/local-source-state-store.js'; import { localPullConfigForAdapter, type DefaultLocalIngestAdaptersOptions } from './local-adapters.js'; @@ -34,24 +34,7 @@ export interface RunLocalIngestOptions { semanticLayerCompute?: KtxSemanticLayerComputePort; queryExecutor?: KtxSqlQueryExecutorPort; logger?: KtxLogger; - embeddingProvider?: import('../../llm/index.js').KtxEmbeddingProvider | null; -} - -export interface LocalIngestMcpOptions - extends Pick< - RunLocalIngestOptions, - | 'agentRunner' - | 'llmRuntime' - | 'memoryModel' - | 'semanticLayerCompute' - | 'queryExecutor' - | 'logger' - | 'pullConfigOptions' - > { - adapters?: SourceAdapter[]; - jobIdFactory?: () => string; - runLocalIngest?: (options: RunLocalIngestOptions) => Promise; - runLocalMetabaseIngest?: (options: RunLocalMetabaseIngestOptions) => Promise; + embeddingProvider?: import('../../llm/types.js').KtxEmbeddingProvider | null; } export interface LocalIngestResult { @@ -59,7 +42,7 @@ export interface LocalIngestResult { report: IngestReportSnapshot; } -export interface LocalMetabaseFanoutChild { +interface LocalMetabaseFanoutChild { jobId: string; metabaseConnectionId: string; metabaseDatabaseId: number; @@ -75,7 +58,7 @@ export interface LocalMetabaseFanoutResult { totals?: { workUnits: number; failedWorkUnits: number }; } -export interface LocalMetabaseFanoutProgressChild { +interface LocalMetabaseFanoutProgressChild { metabaseDatabaseId: number; targetConnectionId: string; } @@ -173,7 +156,7 @@ async function runScheduledPullJob(options: { semanticLayerCompute?: KtxSemanticLayerComputePort; queryExecutor?: KtxSqlQueryExecutorPort; logger?: KtxLogger; - embeddingProvider?: import('../../llm/index.js').KtxEmbeddingProvider | null; + embeddingProvider?: import('../../llm/types.js').KtxEmbeddingProvider | null; }): Promise { const runtime = createLocalBundleIngestRuntime(options); const jobId = options.jobId ?? runtime.nextJobId(); diff --git a/packages/cli/src/context/ingest/local-mapping-reconcile.test.ts b/packages/cli/src/context/ingest/local-mapping-reconcile.test.ts index 903ef43f..3eed9d53 100644 --- a/packages/cli/src/context/ingest/local-mapping-reconcile.test.ts +++ b/packages/cli/src/context/ingest/local-mapping-reconcile.test.ts @@ -2,7 +2,8 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; -import { ktxLocalStateDbPath, type KtxLocalProject } from '../project/index.js'; +import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js'; import { seedLocalMappingStateFromKtxYaml } from './local-mapping-reconcile.js'; diff --git a/packages/cli/src/context/ingest/local-mapping-reconcile.ts b/packages/cli/src/context/ingest/local-mapping-reconcile.ts index 1a58af95..3b9d5d8f 100644 --- a/packages/cli/src/context/ingest/local-mapping-reconcile.ts +++ b/packages/cli/src/context/ingest/local-mapping-reconcile.ts @@ -1,9 +1,6 @@ -import { - ktxLocalStateDbPath, - parseConnectionMappingBootstrap, - type KtxLocalProject, - type LookerMappingBootstrap, -} from '../project/index.js'; +import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js'; +import { parseConnectionMappingBootstrap, type LookerMappingBootstrap } from '../../context/project/mappings-yaml-schema.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js'; function lookerMappings(bootstrap: LookerMappingBootstrap) { diff --git a/packages/cli/src/context/ingest/local-metabase-ingest.test.ts b/packages/cli/src/context/ingest/local-metabase-ingest.test.ts index 04f18923..ff91d827 100644 --- a/packages/cli/src/context/ingest/local-metabase-ingest.test.ts +++ b/packages/cli/src/context/ingest/local-metabase-ingest.test.ts @@ -1,9 +1,9 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import type { AgentRunnerPort, RunLoopParams } from '../llm/index.js'; +import type { AgentRunnerPort, RunLoopParams } from '../../context/llm/runtime-port.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { LocalMetabaseDiscoveryCache } from './adapters/metabase/local-source-state-store.js'; import { getLocalIngestStatus, runLocalMetabaseIngest } from './local-ingest.js'; import type { ChunkResult, FetchContext, SourceAdapter } from './types.js'; diff --git a/packages/cli/src/context/ingest/local-stage-ingest.test.ts b/packages/cli/src/context/ingest/local-stage-ingest.test.ts index 66fea320..7a2c5a6a 100644 --- a/packages/cli/src/context/ingest/local-stage-ingest.test.ts +++ b/packages/cli/src/context/ingest/local-stage-ingest.test.ts @@ -2,7 +2,7 @@ import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promise import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js'; import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js'; import { createDefaultLocalIngestAdapters } from './local-adapters.js'; import { diff --git a/packages/cli/src/context/ingest/local-stage-ingest.ts b/packages/cli/src/context/ingest/local-stage-ingest.ts index 0365b071..5897281f 100644 --- a/packages/cli/src/context/ingest/local-stage-ingest.ts +++ b/packages/cli/src/context/ingest/local-stage-ingest.ts @@ -1,7 +1,7 @@ import { createHash } from 'node:crypto'; import { cp, mkdir, readdir, readFile, rm } from 'node:fs/promises'; import { isAbsolute, join, relative, resolve, sep } from 'node:path'; -import type { KtxLocalProject } from '../project/index.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import { ktxLocalStateDbPath } from '../project/local-state-db.js'; import { computeDiffSetFromHashes } from './diff-set.service.js'; import { localPullConfigForAdapter } from './local-adapters.js'; @@ -11,9 +11,9 @@ import { buildSyncId } from './raw-sources-paths.js'; import { SqliteLocalIngestStore } from './sqlite-local-ingest-store.js'; import type { IngestTrigger, SourceAdapter, WorkUnit } from './types.js'; -export type LocalIngestStatus = 'running' | 'done' | 'error'; +type LocalIngestStatus = 'running' | 'done' | 'error'; -export interface LocalIngestDiffPaths { +interface LocalIngestDiffPaths { added: string[]; modified: string[]; deleted: string[]; diff --git a/packages/cli/src/context/ingest/memory-flow/events.ts b/packages/cli/src/context/ingest/memory-flow/events.ts index 7692e710..020ce5ae 100644 --- a/packages/cli/src/context/ingest/memory-flow/events.ts +++ b/packages/cli/src/context/ingest/memory-flow/events.ts @@ -1,4 +1,4 @@ -import type { MemoryAction } from '../../memory/index.js'; +import type { MemoryAction } from '../../../context/memory/types.js'; import type { LocalIngestRunRecord } from '../local-stage-ingest.js'; import type { IngestReportSnapshot } from '../reports.js'; import type { @@ -132,6 +132,7 @@ function detailSectionsFromReport(report: IngestReportSnapshot): MemoryFlowDetai }; } +/** @internal */ export function localIngestRunToMemoryFlowReplay(record: LocalIngestRunRecord): MemoryFlowReplayInput { const events: MemoryFlowEvent[] = [ { type: 'source_acquired', adapter: record.adapter, trigger: 'manual_resync', fileCount: record.rawFileCount }, diff --git a/packages/cli/src/context/ingest/memory-flow/index.ts b/packages/cli/src/context/ingest/memory-flow/index.ts deleted file mode 100644 index fa4f8fc5..00000000 --- a/packages/cli/src/context/ingest/memory-flow/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -export { - memoryFlowReplayInputSchema, - memoryFlowStreamEventSchema, - parseMemoryFlowReplayInput, -} from './schema.js'; -export type { MemoryFlowStreamEvent } from './schema.js'; -export { buildMemoryFlowViewModel } from './view-model.js'; -export { renderMemoryFlowReplay } from './render.js'; -export { formatMemoryFlowFinalSummary } from './summary.js'; -export type { - MemoryFlowDetailSections, - MemoryFlowEvent, - MemoryFlowPlannedWorkUnit, - MemoryFlowReplayInput, - MemoryFlowRunStatus, - MemoryFlowViewModel, -} from './types.js'; diff --git a/packages/cli/src/context/ingest/memory-flow/interaction.ts b/packages/cli/src/context/ingest/memory-flow/interaction.ts index a7703c36..6d1591af 100644 --- a/packages/cli/src/context/ingest/memory-flow/interaction.ts +++ b/packages/cli/src/context/ingest/memory-flow/interaction.ts @@ -233,6 +233,7 @@ export function createInitialMemoryFlowInteractionState(view: MemoryFlowViewMode }; } +/** @internal */ export function selectMemoryFlowColumn( view: MemoryFlowViewModel, state: MemoryFlowInteractionState, @@ -253,6 +254,7 @@ export function selectMemoryFlowColumn( return { ...nextState, selectedChipIndex: clampChipIndex(column, nextState, view) }; } +/** @internal */ export function selectMemoryFlowChip( view: MemoryFlowViewModel, state: MemoryFlowInteractionState, diff --git a/packages/cli/src/context/ingest/memory-flow/schema.ts b/packages/cli/src/context/ingest/memory-flow/schema.ts index 09cba418..0268a53f 100644 --- a/packages/cli/src/context/ingest/memory-flow/schema.ts +++ b/packages/cli/src/context/ingest/memory-flow/schema.ts @@ -1,7 +1,7 @@ import * as z from 'zod'; import type { MemoryFlowReplayInput } from './types.js'; -export const memoryFlowRunStatusSchema = z.enum(['running', 'done', 'error']); +const memoryFlowRunStatusSchema = z.enum(['running', 'done', 'error']); const memoryFlowEventTimestampShape = { emittedAt: z.string().datetime().optional(), @@ -22,7 +22,7 @@ const memoryFlowReplayMetadataSchema = z.object({ fallbackReason: z.string().min(1).nullable(), }); -export const memoryFlowEventSchema = z.discriminatedUnion('type', [ +const memoryFlowEventSchema = z.discriminatedUnion('type', [ eventSchema({ type: z.literal('source_acquired'), adapter: z.string().min(1), @@ -114,14 +114,14 @@ export const memoryFlowEventSchema = z.discriminatedUnion('type', [ }), ]); -export const memoryFlowPlannedWorkUnitSchema = z.object({ +const memoryFlowPlannedWorkUnitSchema = z.object({ unitKey: z.string().min(1), rawFiles: z.array(z.string()), peerFileCount: z.number().int().min(0), dependencyCount: z.number().int().min(0), }); -export const memoryFlowActionDetailSchema = z.object({ +const memoryFlowActionDetailSchema = z.object({ unitKey: z.string().min(1), target: z.enum(['wiki', 'sl']), action: z.enum(['created', 'updated', 'removed']), @@ -146,7 +146,7 @@ const memoryFlowTranscriptDetailSchema = z.object({ toolNames: z.array(z.string()), }); -export const memoryFlowDetailSectionsSchema = z.object({ +const memoryFlowDetailSectionsSchema = z.object({ actions: z.array(memoryFlowActionDetailSchema), provenance: z.array(memoryFlowProvenanceDetailSchema), transcripts: z.array(memoryFlowTranscriptDetailSchema), @@ -168,6 +168,7 @@ export const memoryFlowReplayInputSchema: z.ZodType = z.o details: memoryFlowDetailSectionsSchema, }); +/** @internal */ export const memoryFlowStreamEventSchema = z.discriminatedUnion('type', [ z.object({ type: z.literal('snapshot'), snapshot: memoryFlowReplayInputSchema }), z.object({ @@ -177,8 +178,8 @@ export const memoryFlowStreamEventSchema = z.discriminatedUnion('type', [ }), ]); -export type MemoryFlowStreamEvent = z.infer; +/** @internal */ export function parseMemoryFlowReplayInput(value: unknown): MemoryFlowReplayInput { const result = memoryFlowReplayInputSchema.safeParse(value); if (!result.success) { diff --git a/packages/cli/src/context/ingest/memory-flow/visuals.ts b/packages/cli/src/context/ingest/memory-flow/visuals.ts index 84eb6113..e3221383 100644 --- a/packages/cli/src/context/ingest/memory-flow/visuals.ts +++ b/packages/cli/src/context/ingest/memory-flow/visuals.ts @@ -5,12 +5,13 @@ import type { MemoryFlowViewModel, } from './types.js'; +/** @internal */ export interface MemoryFlowStatusBadge { label: '..' | '>>' | 'OK' | '!!' | 'XX'; text: 'waiting' | 'active' | 'complete' | 'warning' | 'failed'; } -export interface MemoryFlowVisualColumn { +interface MemoryFlowVisualColumn { id: MemoryFlowColumnId; title: string; status: MemoryFlowDisplayStatus; @@ -18,12 +19,14 @@ export interface MemoryFlowVisualColumn { pulse: boolean; } +/** @internal */ export interface MemoryFlowVisualModel { columns: MemoryFlowVisualColumn[]; connectorLine: string; pulseColumnId: MemoryFlowColumnId; } +/** @internal */ export function memoryFlowStatusBadge(status: MemoryFlowDisplayStatus): MemoryFlowStatusBadge { if (status === 'active') return { label: '>>', text: 'active' }; if (status === 'complete') return { label: 'OK', text: 'complete' }; @@ -56,6 +59,7 @@ function renderColumn(column: MemoryFlowVisualColumn): string { return `${column.badge.label} ${column.title}`; } +/** @internal */ export function buildMemoryFlowVisualModel(view: MemoryFlowViewModel): MemoryFlowVisualModel { const pulseColumn = selectPulseColumn(view.columns); const columns = view.columns.map((column) => ({ diff --git a/packages/cli/src/context/ingest/page-triage/index.ts b/packages/cli/src/context/ingest/page-triage/index.ts deleted file mode 100644 index e589165e..00000000 --- a/packages/cli/src/context/ingest/page-triage/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export type { - PageTriageEvidenceChunk, - PageTriageReport, - PageTriageRunArgs, - PageTriageServiceDeps, - PageTriageSettings, - PageTriageStorePort, -} from './page-triage.service.js'; -export { PageTriageService } from './page-triage.service.js'; diff --git a/packages/cli/src/context/ingest/page-triage/page-triage.service.ts b/packages/cli/src/context/ingest/page-triage/page-triage.service.ts index cb9ea471..289a6aeb 100644 --- a/packages/cli/src/context/ingest/page-triage/page-triage.service.ts +++ b/packages/cli/src/context/ingest/page-triage/page-triage.service.ts @@ -3,10 +3,10 @@ import { readdir, readFile } from 'node:fs/promises'; import { dirname, join, relative } from 'node:path'; import pLimit from 'p-limit'; import { z } from 'zod'; -import { type KtxLogger, noopLogger } from '../../core/index.js'; -import type { KtxLlmRuntimePort } from '../../llm/index.js'; -import type { PromptService } from '../../prompts/index.js'; -import type { InsertContextCandidateInput } from '../context-candidates/index.js'; +import { type KtxLogger, noopLogger } from '../../../context/core/config.js'; +import type { KtxLlmRuntimePort } from '../../../context/llm/runtime-port.js'; +import type { PromptService } from '../../../context/prompts/prompt.service.js'; +import type { InsertContextCandidateInput } from '../../../context/ingest/context-candidates/types.js'; import type { JsonValue } from '../ports.js'; import type { DiffSet, SourceAdapter, TriageLane, TriageSignals } from '../types.js'; @@ -42,7 +42,7 @@ interface StagedTriageDocument { markdown: string; } -export interface PageTriageReport { +interface PageTriageReport { pageCount: number; skip: number; light: number; @@ -89,7 +89,7 @@ export interface PageTriageStorePort { insertCandidate(input: InsertContextCandidateInput): Promise; } -export interface PageTriageSettings { +interface PageTriageSettings { enabled: boolean; maxConcurrency: number; lightExtractionEnabled: boolean; diff --git a/packages/cli/src/context/ingest/ports.ts b/packages/cli/src/context/ingest/ports.ts index d579e6ec..54d28fe6 100644 --- a/packages/cli/src/context/ingest/ports.ts +++ b/packages/cli/src/context/ingest/ports.ts @@ -1,20 +1,22 @@ -import type { KtxModelRole } from '../../llm/index.js'; +import type { KtxModelRole } from '../../llm/types.js'; import type { KtxEmbeddingPort } from '../core/embedding.js'; -import type { GitService, KtxFileStorePort, KtxLogger, SessionOutcome } from '../core/index.js'; -import type { AgentRunnerPort, KtxLlmRuntimePort, KtxRuntimeToolSet } from '../llm/index.js'; -import type { MemoryAction, MemoryKnowledgeSlRefsPort } from '../memory/index.js'; -import type { PromptService } from '../prompts/index.js'; -import type { SkillsRegistryService } from '../skills/index.js'; -import type { - SemanticLayerService, - SlConnectionCatalogPort, - SlSearchService, - SlSourcesIndexPort, - SlValidationDeps, - SlValidatorPort, -} from '../sl/index.js'; -import type { ToolContext, ToolSession } from '../tools/index.js'; -import type { KnowledgeIndexPort, KnowledgeWikiService } from '../wiki/index.js'; +import type { GitService } from '../../context/core/git.service.js'; +import type { KtxFileStorePort } from '../../context/core/file-store.js'; +import type { KtxLogger } from '../../context/core/config.js'; +import type { SessionOutcome } from '../../context/core/session-worktree.service.js'; +import type { AgentRunnerPort, KtxLlmRuntimePort, KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; +import type { MemoryAction, MemoryKnowledgeSlRefsPort } from '../../context/memory/types.js'; +import type { PromptService } from '../../context/prompts/prompt.service.js'; +import type { SkillsRegistryService } from '../../context/skills/skills-registry.service.js'; +import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import type { SlConnectionCatalogPort, SlSourcesIndexPort } from '../../context/sl/ports.js'; +import type { SlSearchService } from '../../context/sl/sl-search.service.js'; +import type { SlValidationDeps } from '../../context/sl/tools/sl-warehouse-validation.js'; +import type { SlValidatorPort } from '../../context/sl/sl-validator.port.js'; +import type { ToolContext } from '../../context/tools/base-tool.js'; +import type { ToolSession } from '../../context/tools/tool-session.js'; +import type { KnowledgeIndexPort } from '../../context/wiki/ports.js'; +import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; import type { CanonicalPin } from './canonical-pins.js'; import type { IngestTraceLevel } from './ingest-trace.js'; import type { IngestReportSnapshot } from './reports.js'; @@ -28,7 +30,6 @@ import type { StageIndex } from './stages/stage-index.types.js'; import type { DiffSet, EvictionUnit, - IngestBundleJob, IngestDiffSummary, IngestTrigger, SourceAdapter, @@ -120,7 +121,7 @@ export interface IngestLockPort { withLock(key: string, fn: () => Promise): Promise; } -export interface IngestFileStorePort extends KtxFileStorePort {} +interface IngestFileStorePort extends KtxFileStorePort {} export interface IngestSessionWorktree { chatId: string; @@ -137,7 +138,7 @@ export interface IngestSessionWorktreePort { cleanup(session: IngestSessionWorktree, outcome: SessionOutcome): Promise; } -export interface IngestSettingsPort { +interface IngestSettingsPort { memoryIngestionModel: string; probeRowCount: number; workUnitMaxConcurrency?: number; @@ -146,7 +147,7 @@ export interface IngestSettingsPort { ingestTraceLevel?: IngestTraceLevel; } -export interface IngestGitAuthor { +interface IngestGitAuthor { name: string; email: string; } @@ -172,7 +173,7 @@ export interface IngestToolsetFactoryPort { createIngestWuToolset(session: ToolSession, options?: { includeContextEvidenceTools?: boolean }): IngestToolsetLike; } -export type IngestKnowledgeIndexPort = Pick; +type IngestKnowledgeIndexPort = Pick; export interface SourceAdapterRegistryPort { register(adapter: SourceAdapter): void; @@ -181,7 +182,7 @@ export interface SourceAdapterRegistryPort { list(): string[]; } -export interface DiffSetComputerPort { +interface DiffSetComputerPort { compute( connectionId: string, sourceKey: string, @@ -203,7 +204,7 @@ export interface ContextEvidenceIndexSummary { warnings: string[]; } -export interface ContextEvidenceIndexPort { +interface ContextEvidenceIndexPort { indexStagedDir(args: { stagedDir: string; runId: string; @@ -230,7 +231,7 @@ export interface PageTriageRunResult { warnings: string[]; } -export interface PageTriagePort { +interface PageTriagePort { triageRun(args: { stagedDir: string; runId: string; @@ -243,7 +244,7 @@ export interface PageTriagePort { }): Promise; } -export interface ContextCandidateCarryforwardPort { +interface ContextCandidateCarryforwardPort { carryForward(args: { runId: string; connectionId: string; sourceKey: string }): Promise<{ warnings: string[] }>; } @@ -271,7 +272,7 @@ export interface CandidateDedupResult { warnings: string[]; } -export interface CandidateDedupPort { +interface CandidateDedupPort { deduplicateRun(runId: string): Promise; } @@ -284,7 +285,7 @@ export interface ContextCandidateSummary { conflict: number; } -export interface ContextEvidenceCandidatesPort { +interface ContextEvidenceCandidatesPort { getCandidateSummary(runId: string): Promise; } @@ -359,4 +360,3 @@ export interface IngestBundleRunnerDeps { logger?: KtxLogger; } -export type IngestRunnerJob = IngestBundleJob; diff --git a/packages/cli/src/context/ingest/raw-sources-paths.ts b/packages/cli/src/context/ingest/raw-sources-paths.ts index 654ce8ba..9e9f847f 100644 --- a/packages/cli/src/context/ingest/raw-sources-paths.ts +++ b/packages/cli/src/context/ingest/raw-sources-paths.ts @@ -1,3 +1,4 @@ +/** @internal */ export const rawSourcesRoot = 'raw-sources'; export function buildSyncId(now: Date, jobId: string): string { @@ -14,6 +15,7 @@ export function rawSourcesDirForSync(connectionId: string, sourceKey: string, sy return `${rawSourcesRoot}/${connectionId}/${sourceKey}/${syncId}`; } +/** @internal */ export function provenanceMarker(rawPath: string, startLine: number, endLine: number): string { return ``; } diff --git a/packages/cli/src/context/ingest/repo-fetch.ts b/packages/cli/src/context/ingest/repo-fetch.ts index cf394675..1fecb422 100644 --- a/packages/cli/src/context/ingest/repo-fetch.ts +++ b/packages/cli/src/context/ingest/repo-fetch.ts @@ -3,12 +3,14 @@ import { join } from 'node:path'; import type { CloneOptions } from 'simple-git'; import { createSimpleGit } from './git-env.js'; +/** @internal */ export interface RepoFetchConfig { repoUrl: string; branch?: string; authToken?: string | null; } +/** @internal */ export class RepoConfigError extends Error { constructor(message: string) { super(message); @@ -16,6 +18,7 @@ export class RepoConfigError extends Error { } } +/** @internal */ export class RepoFetchError extends Error { constructor(message: string) { super(message); @@ -23,6 +26,7 @@ export class RepoFetchError extends Error { } } +/** @internal */ export function validateRepoConfig(config: RepoFetchConfig): void { if (!config.repoUrl) { throw new RepoConfigError('Repository URL is required'); @@ -35,6 +39,7 @@ export function validateRepoConfig(config: RepoFetchConfig): void { } } +/** @internal */ export function buildAuthenticatedUrl(repoUrl: string, authToken: string | null | undefined): string { if (!authToken) { return repoUrl; @@ -70,6 +75,7 @@ export function sanitizeRepoError(err: unknown, authToken: string | null | undef return sanitized; } +/** @internal */ export async function repoDirExists(dir: string): Promise { try { await access(join(dir, '.git')); @@ -130,6 +136,7 @@ export async function testRepoConnection(args: { } } +/** @internal */ export async function cleanupRepoDir(dir: string): Promise { await rm(dir, { recursive: true, force: true }); } diff --git a/packages/cli/src/context/ingest/report-snapshot.ts b/packages/cli/src/context/ingest/report-snapshot.ts index 3cde6a4e..d91f374b 100644 --- a/packages/cli/src/context/ingest/report-snapshot.ts +++ b/packages/cli/src/context/ingest/report-snapshot.ts @@ -158,7 +158,7 @@ const finalizationOutcomeSchema = z.object({ provenanceExclusions: z.array(finalizationProvenanceExclusionSchema).default([]), }); -export const ingestReportSnapshotSchema = z +const ingestReportSnapshotSchema = z .object({ id: z.string().min(1), runId: z.string().min(1), diff --git a/packages/cli/src/context/ingest/reports.ts b/packages/cli/src/context/ingest/reports.ts index 280b224f..ea02a31a 100644 --- a/packages/cli/src/context/ingest/reports.ts +++ b/packages/cli/src/context/ingest/reports.ts @@ -1,5 +1,5 @@ -import type { MemoryAction } from '../memory/index.js'; -import type { TouchedSlSource } from '../tools/index.js'; +import type { MemoryAction } from '../../context/memory/types.js'; +import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js'; import type { MemoryFlowReplayInput } from './memory-flow/types.js'; import type { IngestProvenanceInsert } from './ports.js'; import type { @@ -31,7 +31,7 @@ export interface IngestReportProvenanceDetail { actionType: IngestProvenanceInsert['actionType']; } -export interface IngestReportToolTranscriptSummary { +interface IngestReportToolTranscriptSummary { unitKey: string; path: string; toolCallCount: number; @@ -68,7 +68,7 @@ export interface IngestReportFinalizationOutcome { provenanceExclusions: IngestReportFinalizationProvenanceExclusion[]; } -export interface IngestReportFailure { +interface IngestReportFailure { phase: string; message: string; details?: Record; @@ -135,16 +135,6 @@ export interface IngestSavedMemoryCounts { slCount: number; } -export function finalizationSavedMemoryCounts( - finalization: IngestReportFinalizationOutcome | undefined, -): IngestSavedMemoryCounts { - const actions = finalization?.actions ?? []; - return { - wikiCount: actions.filter((action) => action.target === 'wiki').length, - slCount: actions.filter((action) => action.target === 'sl').length, - }; -} - export function savedMemoryCountsForReport(report: IngestReportSnapshot): IngestSavedMemoryCounts { const workUnitActions = report.body.workUnits.flatMap((workUnit) => workUnit.actions); const reconciliationActions = report.body.reconciliationActions ?? []; diff --git a/packages/cli/src/context/ingest/semantic-layer-target-policy.ts b/packages/cli/src/context/ingest/semantic-layer-target-policy.ts index 1fffbbc5..58cfc19e 100644 --- a/packages/cli/src/context/ingest/semantic-layer-target-policy.ts +++ b/packages/cli/src/context/ingest/semantic-layer-target-policy.ts @@ -9,12 +9,14 @@ export interface SemanticLayerTargetPolicyViolation { connectionId: string; } +/** @internal */ export function semanticLayerConnectionIdFromPath(path: string): string | null { const normalized = path.replace(/^[ab]\//, ''); const match = /^semantic-layer\/([^/]+)\//.exec(normalized); return match?.[1] ?? null; } +/** @internal */ export function findDisallowedSemanticLayerTargetPaths( input: SemanticLayerTargetPolicyInput, ): SemanticLayerTargetPolicyViolation[] { diff --git a/packages/cli/src/context/ingest/sqlite-bundle-ingest-store.test.ts b/packages/cli/src/context/ingest/sqlite-bundle-ingest-store.test.ts index cd6d2385..0cee47d0 100644 --- a/packages/cli/src/context/ingest/sqlite-bundle-ingest-store.test.ts +++ b/packages/cli/src/context/ingest/sqlite-bundle-ingest-store.test.ts @@ -3,7 +3,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { DiffSetService } from './diff-set.service.js'; -import type { IngestDiffSummary, IngestReportBody, IngestTrigger } from './index.js'; +import type { IngestDiffSummary, IngestTrigger } from '../../context/ingest/types.js'; +import type { IngestReportBody } from '../../context/ingest/reports.js'; import { SqliteBundleIngestStore } from './sqlite-bundle-ingest-store.js'; function idFactory(ids: string[]): () => string { diff --git a/packages/cli/src/context/ingest/stages/build-reconcile-context.ts b/packages/cli/src/context/ingest/stages/build-reconcile-context.ts index 366eeb4f..0f033616 100644 --- a/packages/cli/src/context/ingest/stages/build-reconcile-context.ts +++ b/packages/cli/src/context/ingest/stages/build-reconcile-context.ts @@ -1,5 +1,5 @@ import { buildCanonicalPinsPromptBlock, type CanonicalPin } from '../canonical-pins.js'; -import type { KtxRuntimeToolSet } from '../../llm/index.js'; +import type { KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; import { createVerificationLedgerState, VERIFICATION_LEDGER_PROMPT, diff --git a/packages/cli/src/context/ingest/stages/build-wu-context.ts b/packages/cli/src/context/ingest/stages/build-wu-context.ts index 14c2912b..f8fb4af4 100644 --- a/packages/cli/src/context/ingest/stages/build-wu-context.ts +++ b/packages/cli/src/context/ingest/stages/build-wu-context.ts @@ -1,6 +1,7 @@ import { buildCanonicalPinsPromptBlock, type CanonicalPin } from '../canonical-pins.js'; import { createLookerQueryToSlTool } from '../adapters/looker/tools/looker-query-to-sl.tool.js'; -import { createRuntimeToolDescriptorFromAiTool, type KtxRuntimeToolSet } from '../../llm/index.js'; +import { createRuntimeToolDescriptorFromAiTool } from '../../../context/llm/runtime-tools.js'; +import type { KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; import type { IngestProvenanceRow } from '../ports.js'; import { createReadRawFileTool } from '../tools/read-raw-file.tool.js'; import { createReadRawSpanTool } from '../tools/read-raw-span.tool.js'; diff --git a/packages/cli/src/context/ingest/stages/stage-3-work-units.test.ts b/packages/cli/src/context/ingest/stages/stage-3-work-units.test.ts index 2cb0b432..fc39fd9b 100644 --- a/packages/cli/src/context/ingest/stages/stage-3-work-units.test.ts +++ b/packages/cli/src/context/ingest/stages/stage-3-work-units.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import type { CaptureSession, MemoryAction } from '../../memory/index.js'; -import { addTouchedSlSource, createTouchedSlSources } from '../../tools/index.js'; +import type { CaptureSession, MemoryAction } from '../../../context/memory/types.js'; +import { addTouchedSlSource, createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; import type { WorkUnit } from '../types.js'; import { executeWorkUnit, type WorkUnitExecutionDeps } from './stage-3-work-units.js'; diff --git a/packages/cli/src/context/ingest/stages/stage-3-work-units.ts b/packages/cli/src/context/ingest/stages/stage-3-work-units.ts index 250cba4d..96c0e65c 100644 --- a/packages/cli/src/context/ingest/stages/stage-3-work-units.ts +++ b/packages/cli/src/context/ingest/stages/stage-3-work-units.ts @@ -1,7 +1,7 @@ -import type { KtxModelRole } from '../../../llm/index.js'; -import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../index.js'; -import type { CaptureSession, MemoryAction } from '../../memory/index.js'; -import { listTouchedSlSources, type TouchedSlSource } from '../../tools/index.js'; +import type { KtxModelRole } from '../../../llm/types.js'; +import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; +import type { CaptureSession, MemoryAction } from '../../../context/memory/types.js'; +import { listTouchedSlSources, type TouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; import type { WorkUnit } from '../types.js'; const MAX_WORK_UNIT_PROMPT_CHARS = 240_000; diff --git a/packages/cli/src/context/ingest/stages/stage-4-reconciliation.ts b/packages/cli/src/context/ingest/stages/stage-4-reconciliation.ts index 90dae74d..61fc1efe 100644 --- a/packages/cli/src/context/ingest/stages/stage-4-reconciliation.ts +++ b/packages/cli/src/context/ingest/stages/stage-4-reconciliation.ts @@ -1,5 +1,5 @@ -import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../index.js'; -import type { KtxModelRole } from '../../../llm/index.js'; +import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; +import type { KtxModelRole } from '../../../llm/types.js'; import type { EvictionUnit } from '../types.js'; import type { StageIndex } from './stage-index.types.js'; diff --git a/packages/cli/src/context/ingest/stages/stage-index.types.ts b/packages/cli/src/context/ingest/stages/stage-index.types.ts index cae9526d..fd6fd59c 100644 --- a/packages/cli/src/context/ingest/stages/stage-index.types.ts +++ b/packages/cli/src/context/ingest/stages/stage-index.types.ts @@ -1,5 +1,5 @@ -import type { MemoryAction } from '../../memory/index.js'; -import type { TouchedSlSource } from '../../tools/index.js'; +import type { MemoryAction } from '../../../context/memory/types.js'; +import type { TouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; interface StageIndexWorkUnit { unitKey: string; diff --git a/packages/cli/src/context/ingest/stages/validate-wu-sources.ts b/packages/cli/src/context/ingest/stages/validate-wu-sources.ts index bf5476b0..4bc3aaa0 100644 --- a/packages/cli/src/context/ingest/stages/validate-wu-sources.ts +++ b/packages/cli/src/context/ingest/stages/validate-wu-sources.ts @@ -1,5 +1,6 @@ -import type { SlValidationDeps, SlValidatorPort } from '../../sl/index.js'; -import type { TouchedSlSource } from '../../tools/index.js'; +import type { SlValidationDeps } from '../../../context/sl/tools/sl-warehouse-validation.js'; +import type { SlValidatorPort } from '../../../context/sl/sl-validator.port.js'; +import type { TouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; export interface WuValidationResult { validSources: string[]; diff --git a/packages/cli/src/context/ingest/tools/tool-call-logger.ts b/packages/cli/src/context/ingest/tools/tool-call-logger.ts index c91a74c7..61d020c6 100644 --- a/packages/cli/src/context/ingest/tools/tool-call-logger.ts +++ b/packages/cli/src/context/ingest/tools/tool-call-logger.ts @@ -1,6 +1,6 @@ import { appendFile, mkdir } from 'node:fs/promises'; import { dirname } from 'node:path'; -import type { KtxRuntimeToolSet } from '../../llm/index.js'; +import type { KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; export interface ToolCallLogEntry { ts: string; diff --git a/packages/cli/src/context/ingest/tools/verification-ledger.tool.ts b/packages/cli/src/context/ingest/tools/verification-ledger.tool.ts index 7dd3b56c..4ab87131 100644 --- a/packages/cli/src/context/ingest/tools/verification-ledger.tool.ts +++ b/packages/cli/src/context/ingest/tools/verification-ledger.tool.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import type { KtxRuntimeToolDescriptor, KtxRuntimeToolSet } from '../../llm/index.js'; +import type { KtxRuntimeToolDescriptor, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js'; const verificationLedgerInputSchema = z.object({ summary: z.string().min(1).max(2000), diff --git a/packages/cli/src/context/ingest/tools/warehouse-verification/index.ts b/packages/cli/src/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.ts similarity index 81% rename from packages/cli/src/context/ingest/tools/warehouse-verification/index.ts rename to packages/cli/src/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.ts index 0478305c..166713b9 100644 --- a/packages/cli/src/context/ingest/tools/warehouse-verification/index.ts +++ b/packages/cli/src/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.ts @@ -1,7 +1,7 @@ -import type { KtxFileStorePort } from '../../../core/index.js'; -import type { SlConnectionCatalogPort } from '../../../sl/index.js'; +import type { KtxFileStorePort } from '../../../core/file-store.js'; +import type { SlConnectionCatalogPort } from '../../../sl/ports.js'; import { WarehouseCatalogService } from '../../../scan/warehouse-catalog.js'; -import type { BaseTool, ToolContext } from '../../../tools/index.js'; +import type { BaseTool, ToolContext } from '../../../tools/base-tool.js'; import { DiscoverDataTool } from './discover-data.tool.js'; import { EntityDetailsTool } from './entity-details.tool.js'; import { SqlExecutionTool } from './sql-execution.tool.js'; diff --git a/packages/cli/src/context/ingest/tools/warehouse-verification/discover-data.tool.test.ts b/packages/cli/src/context/ingest/tools/warehouse-verification/discover-data.tool.test.ts index 8982e300..7aebc101 100644 --- a/packages/cli/src/context/ingest/tools/warehouse-verification/discover-data.tool.test.ts +++ b/packages/cli/src/context/ingest/tools/warehouse-verification/discover-data.tool.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { WarehouseCatalogService } from '../../../scan/warehouse-catalog.js'; -import type { BaseTool, ToolContext } from '../../../tools/index.js'; +import type { BaseTool, ToolContext } from '../../../../context/tools/base-tool.js'; import { DiscoverDataTool } from './discover-data.tool.js'; describe('DiscoverDataTool', () => { diff --git a/packages/cli/src/context/ingest/tools/warehouse-verification/discover-data.tool.ts b/packages/cli/src/context/ingest/tools/warehouse-verification/discover-data.tool.ts index 4d13ea6b..e358e970 100644 --- a/packages/cli/src/context/ingest/tools/warehouse-verification/discover-data.tool.ts +++ b/packages/cli/src/context/ingest/tools/warehouse-verification/discover-data.tool.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; import { WarehouseCatalogService, type RawSchemaHit } from '../../../scan/warehouse-catalog.js'; -import { BaseTool, type ToolContext, type ToolOutput } from '../../../tools/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../../context/tools/base-tool.js'; const discoverDataInputSchema = z.object({ query: z.string().optional(), diff --git a/packages/cli/src/context/ingest/tools/warehouse-verification/entity-details.tool.test.ts b/packages/cli/src/context/ingest/tools/warehouse-verification/entity-details.tool.test.ts index e6cdbdc8..fcef38df 100644 --- a/packages/cli/src/context/ingest/tools/warehouse-verification/entity-details.tool.test.ts +++ b/packages/cli/src/context/ingest/tools/warehouse-verification/entity-details.tool.test.ts @@ -2,9 +2,9 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../../../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../../../context/project/project.js'; import { WarehouseCatalogService } from '../../../scan/warehouse-catalog.js'; -import type { ToolContext } from '../../../tools/index.js'; +import type { ToolContext } from '../../../../context/tools/base-tool.js'; import { EntityDetailsTool } from './entity-details.tool.js'; describe('EntityDetailsTool', () => { diff --git a/packages/cli/src/context/ingest/tools/warehouse-verification/entity-details.tool.ts b/packages/cli/src/context/ingest/tools/warehouse-verification/entity-details.tool.ts index 79ce92b2..45ecba2b 100644 --- a/packages/cli/src/context/ingest/tools/warehouse-verification/entity-details.tool.ts +++ b/packages/cli/src/context/ingest/tools/warehouse-verification/entity-details.tool.ts @@ -1,7 +1,7 @@ import { z } from 'zod'; import type { KtxTableRef } from '../../../scan/types.js'; import { WarehouseCatalogService, type TableDetail } from '../../../scan/warehouse-catalog.js'; -import { BaseTool, type ToolContext, type ToolOutput } from '../../../tools/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../../context/tools/base-tool.js'; const targetSchema = z.union([ z.object({ display: z.string().min(1) }), diff --git a/packages/cli/src/context/ingest/tools/warehouse-verification/sql-execution.tool.test.ts b/packages/cli/src/context/ingest/tools/warehouse-verification/sql-execution.tool.test.ts index ec7ef0ba..4458471a 100644 --- a/packages/cli/src/context/ingest/tools/warehouse-verification/sql-execution.tool.test.ts +++ b/packages/cli/src/context/ingest/tools/warehouse-verification/sql-execution.tool.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import type { SlConnectionCatalogPort } from '../../../sl/index.js'; -import type { ToolContext } from '../../../tools/index.js'; +import type { SlConnectionCatalogPort } from '../../../../context/sl/ports.js'; +import type { ToolContext } from '../../../../context/tools/base-tool.js'; import { SqlExecutionTool } from './sql-execution.tool.js'; describe('SqlExecutionTool', () => { diff --git a/packages/cli/src/context/ingest/tools/warehouse-verification/sql-execution.tool.ts b/packages/cli/src/context/ingest/tools/warehouse-verification/sql-execution.tool.ts index 8b2e3b5c..76911ce7 100644 --- a/packages/cli/src/context/ingest/tools/warehouse-verification/sql-execution.tool.ts +++ b/packages/cli/src/context/ingest/tools/warehouse-verification/sql-execution.tool.ts @@ -1,7 +1,7 @@ import { z } from 'zod'; -import { assertReadOnlySql, limitSqlForExecution } from '../../../connections/index.js'; -import type { SlConnectionCatalogPort } from '../../../sl/index.js'; -import { BaseTool, type ToolContext, type ToolOutput } from '../../../tools/index.js'; +import { assertReadOnlySql, limitSqlForExecution } from '../../../../context/connections/read-only-sql.js'; +import type { SlConnectionCatalogPort } from '../../../../context/sl/ports.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../../context/tools/base-tool.js'; const sqlExecutionInputSchema = z.object({ connectionId: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), diff --git a/packages/cli/src/context/ingest/types.ts b/packages/cli/src/context/ingest/types.ts index e0317141..991670f6 100644 --- a/packages/cli/src/context/ingest/types.ts +++ b/packages/cli/src/context/ingest/types.ts @@ -1,7 +1,7 @@ import type { KtxEmbeddingPort } from '../core/embedding.js'; -import type { MemoryAction } from '../memory/index.js'; -import type { SemanticLayerService } from '../sl/index.js'; -import type { TouchedSlSource } from '../tools/index.js'; +import type { MemoryAction } from '../../context/memory/types.js'; +import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js'; import type { MemoryFlowEventSink } from './memory-flow/types.js'; import type { StageIndex } from './stages/stage-index.types.js'; import type { WorkUnitOutcome } from './stages/stage-3-work-units.js'; @@ -62,7 +62,7 @@ type SourceFetchIssueKind = | 'derived_table_not_supported' | 'lookml_connection_mismatch'; -export interface SourceFetchIssue { +interface SourceFetchIssue { rawPath: string; entityType: string; entityId: string | null; @@ -179,7 +179,7 @@ export interface SourceAdapter { }): Promise; } -export type IngestBundleRef = +type IngestBundleRef = | { kind: 'upload'; uploadId: string } | { kind: 'scheduled_pull'; config: unknown } | { kind: 'override'; priorJobId: string }; diff --git a/packages/cli/src/context/ingest/wiki-body-refs.ts b/packages/cli/src/context/ingest/wiki-body-refs.ts index 25f25eb3..fa62aefa 100644 --- a/packages/cli/src/context/ingest/wiki-body-refs.ts +++ b/packages/cli/src/context/ingest/wiki-body-refs.ts @@ -1,5 +1,6 @@ -import type { SemanticLayerSource } from '../sl/index.js'; +import type { SemanticLayerSource } from '../../context/sl/types.js'; +/** @internal */ export type WikiBodyRef = | { kind: 'sl_entity'; connectionId: string | null; sourceName: string; entityName: string } | { kind: 'sl_source'; connectionId: string | null; sourceName: string } @@ -42,6 +43,7 @@ function isIdentifierToken(value: string): boolean { return /^[A-Za-z_][A-Za-z0-9_]*$/.test(value); } +/** @internal */ export function parseWikiBodyRefs(body: string): WikiBodyRef[] { const refs: WikiBodyRef[] = []; for (const line of visibleLinesOutsideFences(body)) { diff --git a/packages/cli/src/context/ingest/wiki-sl-ref-repair.ts b/packages/cli/src/context/ingest/wiki-sl-ref-repair.ts index 3205ebbd..7dcd1bd1 100644 --- a/packages/cli/src/context/ingest/wiki-sl-ref-repair.ts +++ b/packages/cli/src/context/ingest/wiki-sl-ref-repair.ts @@ -1,7 +1,9 @@ -import type { KtxFileStorePort } from '../core/index.js'; -import type { SemanticLayerService, SemanticLayerSource } from '../sl/index.js'; +import type { KtxFileStorePort } from '../../context/core/file-store.js'; +import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import type { SemanticLayerSource } from '../../context/sl/types.js'; import { isFlatWikiKey } from '../wiki/keys.js'; -import type { KnowledgeWikiService, WikiFrontmatter } from '../wiki/index.js'; +import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; +import type { WikiFrontmatter } from '../../context/wiki/types.js'; const SYSTEM_AUTHOR = 'System User'; const SYSTEM_EMAIL = 'system@example.com'; diff --git a/packages/cli/src/context/llm/ai-sdk-runtime.ts b/packages/cli/src/context/llm/ai-sdk-runtime.ts index db38eae6..33a55c11 100644 --- a/packages/cli/src/context/llm/ai-sdk-runtime.ts +++ b/packages/cli/src/context/llm/ai-sdk-runtime.ts @@ -1,7 +1,8 @@ -import { KtxMessageBuilder, splitKtxSystemMessages, type KtxLlmProvider } from '../../llm/index.js'; +import { KtxMessageBuilder, splitKtxSystemMessages } from '../../llm/message-builder.js'; +import type { KtxLlmProvider } from '../../llm/types.js'; import { generateText, Output, stepCountIs, type FlexibleSchema, type TelemetrySettings, type ToolSet } from 'ai'; import type { z } from 'zod'; -import { noopLogger, type KtxLogger } from '../core/index.js'; +import { noopLogger, type KtxLogger } from '../../context/core/config.js'; import { summarizeKtxLlmDebugRequest, type KtxLlmDebugRequestRecorder } from './debug-request-recorder.js'; import { createAiSdkToolSet } from './runtime-tools.js'; import type { @@ -12,7 +13,7 @@ import type { RunLoopResult, } from './runtime-port.js'; -export interface AgentTelemetryPort { +interface AgentTelemetryPort { createTelemetry(tags: Record): TelemetrySettings; } diff --git a/packages/cli/src/context/llm/claude-code-env.ts b/packages/cli/src/context/llm/claude-code-env.ts index 285113e4..a80f633f 100644 --- a/packages/cli/src/context/llm/claude-code-env.ts +++ b/packages/cli/src/context/llm/claude-code-env.ts @@ -1,3 +1,4 @@ +/** @internal */ export const CLAUDE_CODE_PROVIDER_ENV_DENYLIST = [ 'ANTHROPIC_API_KEY', 'ANTHROPIC_AUTH_TOKEN', diff --git a/packages/cli/src/context/llm/claude-code-runtime.ts b/packages/cli/src/context/llm/claude-code-runtime.ts index bf815445..c6783d71 100644 --- a/packages/cli/src/context/llm/claude-code-runtime.ts +++ b/packages/cli/src/context/llm/claude-code-runtime.ts @@ -6,7 +6,7 @@ import { type SDKResultMessage, } from '@anthropic-ai/claude-agent-sdk'; import { z } from 'zod'; -import { noopLogger, type KtxLogger } from '../core/index.js'; +import { noopLogger, type KtxLogger } from '../../context/core/config.js'; import { createKtxClaudeCodeEnv } from './claude-code-env.js'; import { resolveClaudeCodeModel } from './claude-code-models.js'; import { createClaudeSdkTools, mcpToolIds } from './runtime-tools.js'; @@ -59,6 +59,7 @@ function resultError(result: SDKResultMessage): Error | undefined { return new Error(`Claude Code query failed (${result.subtype})${details}`); } +/** @internal */ export function mapClaudeCodeStopReason(result: SDKResultMessage): RunLoopStopReason { if (result.subtype === 'error_max_turns') { return 'budget'; diff --git a/packages/cli/src/context/llm/debug-request-recorder.ts b/packages/cli/src/context/llm/debug-request-recorder.ts index 17937d90..92ae6ab9 100644 --- a/packages/cli/src/context/llm/debug-request-recorder.ts +++ b/packages/cli/src/context/llm/debug-request-recorder.ts @@ -1,12 +1,12 @@ import { appendFile, mkdir } from 'node:fs/promises'; import { dirname } from 'node:path'; import type { ModelMessage } from 'ai'; -import type { KtxModelRole } from '../../llm/index.js'; +import type { KtxModelRole } from '../../llm/types.js'; type ProviderOptionsCarrier = { providerOptions?: unknown; [key: string]: unknown }; type ToolMap = Record; -export interface KtxLlmDebugProviderOptionsEntry { +interface KtxLlmDebugProviderOptionsEntry { target: 'message' | 'message-part' | 'tool'; index?: number; role?: string; @@ -121,6 +121,7 @@ export function summarizeKtxLlmDebugRequest(input: SummarizeKtxLlmDebugRequestIn }; } +/** @internal */ export function createJsonlKtxLlmDebugRequestRecorder(filePath: string): KtxLlmDebugRequestRecorder { return { async record(request) { diff --git a/packages/cli/src/context/llm/embedding-port.ts b/packages/cli/src/context/llm/embedding-port.ts index 8d250146..e4501053 100644 --- a/packages/cli/src/context/llm/embedding-port.ts +++ b/packages/cli/src/context/llm/embedding-port.ts @@ -1,4 +1,4 @@ -import type { KtxEmbeddingProvider } from '../../llm/index.js'; +import type { KtxEmbeddingProvider } from '../../llm/types.js'; import type { KtxEmbeddingPort as KtxIngestEmbeddingPort } from '../core/embedding.js'; import type { KtxEmbeddingPort as KtxScanEmbeddingPort } from '../scan/types.js'; diff --git a/packages/cli/src/context/llm/index.ts b/packages/cli/src/context/llm/index.ts deleted file mode 100644 index 96da8dcf..00000000 --- a/packages/cli/src/context/llm/index.ts +++ /dev/null @@ -1,44 +0,0 @@ -export { KtxIngestEmbeddingPortAdapter, KtxScanEmbeddingPortAdapter } from './embedding-port.js'; -export { AiSdkKtxLlmRuntime } from './ai-sdk-runtime.js'; -export type { AgentTelemetryPort, AiSdkKtxLlmRuntimeDeps } from './ai-sdk-runtime.js'; -export { createKtxClaudeCodeEnv, CLAUDE_CODE_PROVIDER_ENV_DENYLIST } from './claude-code-env.js'; -export { resolveClaudeCodeModel } from './claude-code-models.js'; -export { ClaudeCodeKtxLlmRuntime, mapClaudeCodeStopReason, runClaudeCodeAuthProbe } from './claude-code-runtime.js'; -export type { - AgentRunnerPort, - KtxGenerateObjectInput, - KtxGenerateTextInput, - KtxLlmRuntimePort, - KtxRuntimeToolDescriptor, - KtxRuntimeToolOutput, - KtxRuntimeToolSet, - RunLoopParams, - RunLoopResult, - RunLoopStepInfo, - RunLoopStopReason, -} from './runtime-port.js'; -export { RuntimeAgentRunner } from './runtime-port.js'; -export { - createAiSdkToolSet, - createClaudeSdkTools, - createRuntimeToolDescriptorFromAiTool, - createRuntimeToolSetFromAiSdkTools, - normalizeKtxRuntimeToolOutput, -} from './runtime-tools.js'; -export type { - KtxLlmDebugProviderOptionsEntry, - KtxLlmDebugRequest, - KtxLlmDebugRequestRecorder, - SummarizeKtxLlmDebugRequestInput, -} from './debug-request-recorder.js'; -export { - createJsonlKtxLlmDebugRequestRecorder, - summarizeKtxLlmDebugRequest, -} from './debug-request-recorder.js'; -export { - createLocalKtxEmbeddingProviderFromConfig, - createLocalKtxLlmProviderFromConfig, - createLocalKtxLlmRuntimeFromConfig, - resolveLocalKtxEmbeddingConfig, - resolveLocalKtxLlmConfig, -} from './local-config.js'; diff --git a/packages/cli/src/context/llm/local-config.ts b/packages/cli/src/context/llm/local-config.ts index 64228489..c64a85cf 100644 --- a/packages/cli/src/context/llm/local-config.ts +++ b/packages/cli/src/context/llm/local-config.ts @@ -1,12 +1,6 @@ -import { - createKtxEmbeddingProvider, - createKtxLlmProvider, - type KtxEmbeddingConfig, - type KtxEmbeddingProvider, - type KtxLlmConfig, - type KtxLlmProvider, - type KtxModelRole, -} from '../../llm/index.js'; +import { createKtxEmbeddingProvider } from '../../llm/embedding-provider.js'; +import { createKtxLlmProvider } from '../../llm/model-provider.js'; +import type { KtxEmbeddingConfig, KtxEmbeddingProvider, KtxLlmConfig, KtxLlmProvider, KtxModelRole } from '../../llm/types.js'; import { resolveKtxConfigReference } from '../core/config-reference.js'; import type { KtxProjectEmbeddingConfig, KtxProjectLlmConfig } from '../project/config.js'; import { AiSdkKtxLlmRuntime } from './ai-sdk-runtime.js'; @@ -104,6 +98,7 @@ export function resolveLocalKtxLlmConfig(config: KtxProjectLlmConfig, env: NodeJ }; } +/** @internal */ export function createLocalKtxLlmProviderFromConfig( config: KtxProjectLlmConfig, deps: LocalConfigDeps = {}, @@ -177,6 +172,7 @@ export function resolveLocalKtxEmbeddingConfig( throw new Error(`Unsupported KTX embedding backend: ${String((config as { backend?: string }).backend)}`); } +/** @internal */ export function createLocalKtxEmbeddingProviderFromConfig( config: KtxProjectEmbeddingConfig, deps: LocalConfigDeps = {}, diff --git a/packages/cli/src/context/llm/runtime-port.ts b/packages/cli/src/context/llm/runtime-port.ts index 5a9cbabb..c1f5ca10 100644 --- a/packages/cli/src/context/llm/runtime-port.ts +++ b/packages/cli/src/context/llm/runtime-port.ts @@ -1,4 +1,4 @@ -import type { KtxModelRole } from '../../llm/index.js'; +import type { KtxModelRole } from '../../llm/types.js'; import type { z } from 'zod'; export interface KtxRuntimeToolOutput { @@ -17,6 +17,7 @@ export type KtxRuntimeToolSet = Record; export type RunLoopStopReason = 'budget' | 'natural' | 'error'; +/** @internal */ export interface RunLoopStepInfo { stepIndex: number; stepBudget: number; diff --git a/packages/cli/src/context/llm/runtime-tools.ts b/packages/cli/src/context/llm/runtime-tools.ts index ab2e088d..0c83c8d4 100644 --- a/packages/cli/src/context/llm/runtime-tools.ts +++ b/packages/cli/src/context/llm/runtime-tools.ts @@ -84,8 +84,3 @@ export function createRuntimeToolDescriptorFromAiTool(name: string, aiSdkTool: T }; } -export function createRuntimeToolSetFromAiSdkTools(tools: ToolSet = {}): KtxRuntimeToolSet { - return Object.fromEntries( - Object.entries(tools).map(([name, aiSdkTool]) => [name, createRuntimeToolDescriptorFromAiTool(name, aiSdkTool as Tool)]), - ); -} diff --git a/packages/cli/src/context/mcp/context-tools.ts b/packages/cli/src/context/mcp/context-tools.ts index e040df87..6778bb64 100644 --- a/packages/cli/src/context/mcp/context-tools.ts +++ b/packages/cli/src/context/mcp/context-tools.ts @@ -1,7 +1,7 @@ import { randomUUID } from 'node:crypto'; import type { ToolAnnotations } from '@modelcontextprotocol/sdk/types.js'; import { z } from 'zod'; -import type { MemoryAgentInput } from '../memory/index.js'; +import type { MemoryAgentInput } from '../../context/memory/types.js'; import type { KtxMcpContextPorts, KtxMcpProgressCallback, @@ -445,6 +445,7 @@ const memoryIngestStatusOutputSchema = z.object({ signalDetected: z.boolean(), }); +/** @internal */ export function jsonToolResult(structuredContent: T): KtxMcpToolResult { return { content: [{ type: 'text', text: JSON.stringify(structuredContent, null, 2) }], @@ -452,7 +453,7 @@ export function jsonToolResult(structuredContent: T): }; } -export function jsonErrorToolResult(text: string): KtxMcpToolResult> { +function jsonErrorToolResult(text: string): KtxMcpToolResult> { return { content: [{ type: 'text', text }], isError: true, diff --git a/packages/cli/src/context/mcp/index.ts b/packages/cli/src/context/mcp/index.ts deleted file mode 100644 index f241c68e..00000000 --- a/packages/cli/src/context/mcp/index.ts +++ /dev/null @@ -1,25 +0,0 @@ -export type { RegisterKtxContextToolsDeps } from './context-tools.js'; -export { jsonErrorToolResult, jsonToolResult, registerKtxContextTools } from './context-tools.js'; -export { createLocalProjectMcpContextPorts } from './local-project-ports.js'; -export { createDefaultKtxMcpServer, createKtxMcpServer } from './server.js'; -export type { - KtxConnectionSummary, - KtxConnectionsMcpPort, - KtxDiscoverDataMcpPort, - KtxDictionarySearchMcpPort, - KtxEntityDetailsMcpPort, - KtxKnowledgeMcpPort, - KtxKnowledgePage, - KtxKnowledgeSearchResponse, - KtxKnowledgeSearchResult, - KtxMcpContextPorts, - KtxMcpServerDeps, - KtxMcpServerLike, - KtxMcpTextContent, - KtxMcpToolResult, - KtxMcpUserContext, - KtxSemanticLayerMcpPort, - KtxSemanticLayerQueryResponse, - KtxSemanticLayerReadResponse, - MemoryIngestPort, -} from './types.js'; diff --git a/packages/cli/src/context/mcp/local-project-ports.test.ts b/packages/cli/src/context/mcp/local-project-ports.test.ts index b9ca1fc0..aa06b47e 100644 --- a/packages/cli/src/context/mcp/local-project-ports.test.ts +++ b/packages/cli/src/context/mcp/local-project-ports.test.ts @@ -2,14 +2,9 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { initKtxProject } from '../project/index.js'; -import { - createKtxConnectorCapabilities, - type KtxQueryResult, - type KtxScanConnector, - type KtxSchemaSnapshot, -} from '../scan/index.js'; -import { writeLocalSlSource } from '../sl/index.js'; +import { initKtxProject } from '../../context/project/project.js'; +import { createKtxConnectorCapabilities, type KtxQueryResult, type KtxScanConnector, type KtxSchemaSnapshot } from '../../context/scan/types.js'; +import { writeLocalSlSource } from '../../context/sl/local-sl.js'; import { createLocalProjectMcpContextPorts } from './local-project-ports.js'; describe('createLocalProjectMcpContextPorts', () => { diff --git a/packages/cli/src/context/mcp/local-project-ports.ts b/packages/cli/src/context/mcp/local-project-ports.ts index bf40dc80..dbbb36d2 100644 --- a/packages/cli/src/context/mcp/local-project-ports.ts +++ b/packages/cli/src/context/mcp/local-project-ports.ts @@ -1,11 +1,15 @@ -import { type KtxSqlQueryExecutorPort, localConnectionInfoFromConfig } from '../connections/index.js'; -import type { KtxEmbeddingPort } from '../core/index.js'; -import type { KtxSemanticLayerComputePort } from '../daemon/index.js'; -import type { KtxLocalProject } from '../project/index.js'; -import { createKtxEntityDetailsService, type KtxScanConnector, type LocalScanMcpOptions } from '../scan/index.js'; -import { createKtxDiscoverDataService } from '../search/index.js'; -import type { SqlAnalysisDialect, SqlAnalysisPort } from '../sql-analysis/index.js'; -import { compileLocalSlQuery, createKtxDictionarySearchService } from '../sl/index.js'; +import type { KtxSqlQueryExecutorPort } from '../../context/connections/query-executor.js'; +import { localConnectionInfoFromConfig } from '../../context/connections/local-warehouse-descriptor.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { createKtxEntityDetailsService } from '../../context/scan/entity-details.js'; +import type { KtxScanConnector } from '../../context/scan/types.js'; +import type { LocalScanMcpOptions } from '../../context/scan/local-scan.js'; +import { createKtxDiscoverDataService } from '../../context/search/discover.js'; +import type { SqlAnalysisDialect, SqlAnalysisPort } from '../../context/sql-analysis/ports.js'; +import { compileLocalSlQuery } from '../../context/sl/local-query.js'; +import { createKtxDictionarySearchService } from '../../context/sl/dictionary-search.js'; import { readLocalKnowledgePage, searchLocalKnowledgePages } from '../wiki/local-knowledge.js'; import type { KtxMcpContextPorts, KtxMcpProgressCallback, KtxSqlExecutionResponse } from './types.js'; diff --git a/packages/cli/src/context/mcp/server.test.ts b/packages/cli/src/context/mcp/server.test.ts index f02b4adb..3532a327 100644 --- a/packages/cli/src/context/mcp/server.test.ts +++ b/packages/cli/src/context/mcp/server.test.ts @@ -4,12 +4,10 @@ import { join } from 'node:path'; import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; import { describe, expect, it, vi } from 'vitest'; -import { - createLocalProjectMemoryIngest, - detectCaptureSignals, - type MemoryAgentInput, -} from '../memory/index.js'; -import { initKtxProject } from '../project/index.js'; +import { createLocalProjectMemoryIngest } from '../../context/memory/local-memory.js'; +import { detectCaptureSignals } from '../../context/memory/capture-signals.js'; +import type { MemoryAgentInput } from '../../context/memory/types.js'; +import { initKtxProject } from '../../context/project/project.js'; import { jsonToolResult } from './context-tools.js'; import { createDefaultKtxMcpServer, createKtxMcpServer } from './server.js'; import type { diff --git a/packages/cli/src/context/mcp/server.ts b/packages/cli/src/context/mcp/server.ts index 8931650f..e1236e9c 100644 --- a/packages/cli/src/context/mcp/server.ts +++ b/packages/cli/src/context/mcp/server.ts @@ -2,6 +2,7 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { registerKtxContextTools } from './context-tools.js'; import type { KtxMcpServerDeps, KtxMcpServerLike } from './types.js'; +/** @internal */ export function createKtxMcpServer(deps: KtxMcpServerDeps): KtxMcpServerDeps['server'] { if (deps.contextTools) { registerKtxContextTools({ diff --git a/packages/cli/src/context/mcp/types.ts b/packages/cli/src/context/mcp/types.ts index 0697e3fb..223aa394 100644 --- a/packages/cli/src/context/mcp/types.ts +++ b/packages/cli/src/context/mcp/types.ts @@ -1,10 +1,11 @@ -import type { MemoryIngestService } from '../memory/index.js'; +import type { MemoryIngestService } from '../../context/memory/memory-runs.js'; import type { KtxEntityDetailsInput, KtxEntityDetailsResponse } from '../scan/entity-details.js'; -import type { KtxDiscoverDataInput, KtxDiscoverDataResponse } from '../search/index.js'; -import type { KtxDictionarySearchInput, KtxDictionarySearchResponse, SemanticLayerQueryInput } from '../sl/index.js'; -import type { WikiSearchLaneSummary, WikiSearchMatchReason } from '../wiki/index.js'; +import type { KtxDiscoverDataInput, KtxDiscoverDataResponse } from '../../context/search/discover.js'; +import type { KtxDictionarySearchInput, KtxDictionarySearchResponse } from '../../context/sl/dictionary-search.js'; +import type { SemanticLayerQueryInput } from '../../context/sl/types.js'; +import type { WikiSearchLaneSummary, WikiSearchMatchReason } from '../../context/wiki/types.js'; -export interface KtxMcpTextContent { +interface KtxMcpTextContent { type: 'text'; text: string; } @@ -38,6 +39,7 @@ export interface KtxMcpToolHandlerContext { }) => Promise; } +/** @internal */ export interface MemoryIngestPort { ingest: MemoryIngestService['ingest']; status: MemoryIngestService['status']; @@ -61,17 +63,17 @@ export interface KtxMcpServerLike { ): void; } -export interface KtxConnectionSummary { +interface KtxConnectionSummary { id: string; name: string; connectionType: string; } -export interface KtxConnectionsMcpPort { +interface KtxConnectionsMcpPort { list(): Promise; } -export interface KtxKnowledgeSearchResult { +interface KtxKnowledgeSearchResult { key: string; path: string; scope: 'GLOBAL' | 'USER'; @@ -81,12 +83,12 @@ export interface KtxKnowledgeSearchResult { lanes?: WikiSearchLaneSummary[]; } -export interface KtxKnowledgeSearchResponse { +interface KtxKnowledgeSearchResponse { results: KtxKnowledgeSearchResult[]; totalFound: number; } -export interface KtxKnowledgePage { +interface KtxKnowledgePage { key: string; summary: string; content: string; @@ -96,17 +98,18 @@ export interface KtxKnowledgePage { slRefs?: string[]; } +/** @internal */ export interface KtxKnowledgeMcpPort { search(input: { userId: string; query: string; limit: number }): Promise; read(input: { userId: string; key: string }): Promise; } -export interface KtxSemanticLayerReadResponse { +interface KtxSemanticLayerReadResponse { sourceName: string; yaml: string; } -export interface KtxSemanticLayerQueryResponse { +interface KtxSemanticLayerQueryResponse { sql: string; headers: string[]; rows: unknown[][]; @@ -114,6 +117,7 @@ export interface KtxSemanticLayerQueryResponse { plan?: Record; } +/** @internal */ export interface KtxSemanticLayerMcpPort { readSource(input: { connectionId: string; sourceName: string }): Promise; query( @@ -122,14 +126,17 @@ export interface KtxSemanticLayerMcpPort { ): Promise; } +/** @internal */ export interface KtxEntityDetailsMcpPort { read(input: KtxEntityDetailsInput): Promise; } +/** @internal */ export interface KtxDictionarySearchMcpPort { search(input: KtxDictionarySearchInput): Promise; } +/** @internal */ export interface KtxDiscoverDataMcpPort { search(input: KtxDiscoverDataInput): Promise; } diff --git a/packages/cli/src/context/memory/capture-signals.ts b/packages/cli/src/context/memory/capture-signals.ts index 360f0b7c..86a123e0 100644 --- a/packages/cli/src/context/memory/capture-signals.ts +++ b/packages/cli/src/context/memory/capture-signals.ts @@ -103,6 +103,7 @@ export function prefilterSkipReason(input: MemoryAgentInput, signals = detectCap return null; } +/** @internal */ export function isWorthAnalyzing(input: MemoryAgentInput): boolean { return prefilterSkipReason(input, detectCaptureSignals(input)) === null; } diff --git a/packages/cli/src/context/memory/index.ts b/packages/cli/src/context/memory/index.ts deleted file mode 100644 index 28efcd2e..00000000 --- a/packages/cli/src/context/memory/index.ts +++ /dev/null @@ -1,41 +0,0 @@ -export { - buildRequiredSkillsBlock, - DEFAULT_SKILL_NAMES, - detectCaptureSignals, - isWorthAnalyzing, - prefilterSkipReason, - promptNameFor, - stepBudgetFor, -} from './capture-signals.js'; -export { MemoryAgentService } from './memory-agent.service.js'; -export { createLocalProjectMemoryIngest, type CreateLocalProjectMemoryIngestOptions } from './local-memory.js'; -export { LocalMemoryRunStore, type LocalMemoryRunStoreOptions } from './local-memory-runs.js'; -export { - MemoryIngestService, - type MemoryIngestServiceDeps, - type MemoryIngestStartResult, - type MemoryIngestStatus, - type MemoryRunRecord, - type MemoryRunStatus, - type MemoryRunStorePort, -} from './memory-runs.js'; - -export type { - CaptureSession, - CaptureSignals, - MemoryAction, - MemoryAgentInput, - MemoryAgentResult, - MemoryAgentServiceDeps, - MemoryAgentSettings, - MemoryAgentSourceType, - MemoryCommitMessagePort, - MemoryConnectionPort, - MemoryFileStorePort, - MemoryKnowledgeSlRefsPort, - MemoryLockPort, - MemorySlSourceReconcilerPort, - MemoryTelemetryPort, - MemoryToolSetLike, - MemoryToolsetFactoryPort, -} from './types.js'; diff --git a/packages/cli/src/context/memory/local-memory.test.ts b/packages/cli/src/context/memory/local-memory.test.ts index 24e0df14..69d8e1f2 100644 --- a/packages/cli/src/context/memory/local-memory.test.ts +++ b/packages/cli/src/context/memory/local-memory.test.ts @@ -2,7 +2,7 @@ import { access, mkdtemp, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { initKtxProject } from '../project/index.js'; +import { initKtxProject } from '../../context/project/project.js'; import { createLocalProjectMemoryIngest } from './local-memory.js'; import { LocalMemoryRunStore } from './local-memory-runs.js'; diff --git a/packages/cli/src/context/memory/local-memory.ts b/packages/cli/src/context/memory/local-memory.ts index c12dec74..121696e6 100644 --- a/packages/cli/src/context/memory/local-memory.ts +++ b/packages/cli/src/context/memory/local-memory.ts @@ -1,54 +1,41 @@ import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import YAML from 'yaml'; -import { localConnectionInfoFromConfig } from '../connections/index.js'; -import type { KtxEmbeddingPort, KtxFileStorePort, KtxFileWriteResult } from '../core/index.js'; -import { type KtxLogger, noopLogger, SessionWorktreeService } from '../core/index.js'; -import type { KtxSemanticLayerComputePort } from '../daemon/index.js'; -import { - createLocalKtxLlmRuntimeFromConfig, - RuntimeAgentRunner, - type AgentRunnerPort, - type KtxLlmRuntimePort, - type KtxRuntimeToolSet, -} from '../llm/index.js'; -import type { KtxLocalProject } from '../project/index.js'; -import { PromptService } from '../prompts/index.js'; -import { SkillsRegistryService } from '../skills/index.js'; -import { - type KtxConnectionInfo, - type KtxQueryResult, - SemanticLayerService, - type SemanticLayerSource, - type SlConnectionCatalogPort, - SlDiscoverTool, - SlEditSourceTool, - type SlPythonPort, - SlReadSourceTool, - SlRollbackTool, - SlSearchService, - type SlSourcesIndexPort, - SlValidateTool, - type SlValidationDeps, - type SlValidatorPort, - SlWriteSourceTool, - SqliteSlSourcesIndex, - sourceDefinitionSchema, - sourceOverlaySchema, -} from '../sl/index.js'; -import { BaseTool, type GitAuthorResolverPort, type ToolContext } from '../tools/index.js'; -import { - type KnowledgeEventPort, - type KnowledgeIndexPort, - type KnowledgeIndexPageListing, - KnowledgeWikiService, - searchLocalKnowledgePages, - WikiListTagsTool, - WikiReadTool, - WikiRemoveTool, - WikiSearchTool, - WikiWriteTool, -} from '../wiki/index.js'; +import { localConnectionInfoFromConfig } from '../../context/connections/local-warehouse-descriptor.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxFileStorePort, KtxFileWriteResult } from '../../context/core/file-store.js'; +import { type KtxLogger, noopLogger } from '../../context/core/config.js'; +import { SessionWorktreeService } from '../../context/core/session-worktree.service.js'; +import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js'; +import { createLocalKtxLlmRuntimeFromConfig } from '../../context/llm/local-config.js'; +import { RuntimeAgentRunner, type AgentRunnerPort, type KtxLlmRuntimePort, type KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { PromptService } from '../../context/prompts/prompt.service.js'; +import { SkillsRegistryService } from '../../context/skills/skills-registry.service.js'; +import type { KtxConnectionInfo, KtxQueryResult, SlConnectionCatalogPort, SlPythonPort, SlSourcesIndexPort } from '../../context/sl/ports.js'; +import { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import type { SemanticLayerSource } from '../../context/sl/types.js'; +import { SlDiscoverTool } from '../../context/sl/tools/sl-discover.tool.js'; +import { SlEditSourceTool } from '../../context/sl/tools/sl-edit-source.tool.js'; +import { SlReadSourceTool } from '../../context/sl/tools/sl-read-source.tool.js'; +import { SlRollbackTool } from '../../context/sl/tools/sl-rollback.tool.js'; +import { SlSearchService } from '../../context/sl/sl-search.service.js'; +import { SlValidateTool } from '../../context/sl/tools/sl-validate.tool.js'; +import type { SlValidationDeps } from '../../context/sl/tools/sl-warehouse-validation.js'; +import type { SlValidatorPort } from '../../context/sl/sl-validator.port.js'; +import { SlWriteSourceTool } from '../../context/sl/tools/sl-write-source.tool.js'; +import { SqliteSlSourcesIndex } from '../../context/sl/sqlite-sl-sources-index.js'; +import { sourceDefinitionSchema, sourceOverlaySchema } from '../../context/sl/schemas.js'; +import { BaseTool, type ToolContext } from '../../context/tools/base-tool.js'; +import type { GitAuthorResolverPort } from '../../context/tools/authors.js'; +import type { KnowledgeEventPort, KnowledgeIndexPort, KnowledgeIndexPageListing } from '../../context/wiki/ports.js'; +import { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; +import { searchLocalKnowledgePages } from '../../context/wiki/local-knowledge.js'; +import { WikiListTagsTool } from '../../context/wiki/tools/wiki-list-tags.tool.js'; +import { WikiReadTool } from '../../context/wiki/tools/wiki-read.tool.js'; +import { WikiRemoveTool } from '../../context/wiki/tools/wiki-remove.tool.js'; +import { WikiSearchTool } from '../../context/wiki/tools/wiki-search.tool.js'; +import { WikiWriteTool } from '../../context/wiki/tools/wiki-write.tool.js'; import { LocalMemoryRunStore } from './local-memory-runs.js'; import { MemoryAgentService } from './memory-agent.service.js'; import { MemoryIngestService } from './memory-runs.js'; diff --git a/packages/cli/src/context/memory/memory-agent.service.ingest.test.ts b/packages/cli/src/context/memory/memory-agent.service.ingest.test.ts index e9985c20..1c13bdd2 100644 --- a/packages/cli/src/context/memory/memory-agent.service.ingest.test.ts +++ b/packages/cli/src/context/memory/memory-agent.service.ingest.test.ts @@ -13,7 +13,7 @@ vi.mock('ai', () => ({ // Imported AFTER vi.mock so the mocked module is used. import { generateText } from 'ai'; -import { SYSTEM_GIT_AUTHOR } from '../tools/index.js'; +import { SYSTEM_GIT_AUTHOR } from '../../context/tools/authors.js'; import { MemoryAgentService } from './memory-agent.service.js'; interface BuiltMocks { diff --git a/packages/cli/src/context/memory/memory-agent.service.test.ts b/packages/cli/src/context/memory/memory-agent.service.test.ts index 63f97e14..cea83674 100644 --- a/packages/cli/src/context/memory/memory-agent.service.test.ts +++ b/packages/cli/src/context/memory/memory-agent.service.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { validateSingleSource } from '../sl/index.js'; -import { createTouchedSlSources, hasTouchedSlSource } from '../tools/index.js'; +import { validateSingleSource } from '../../context/sl/tools/sl-warehouse-validation.js'; +import { createTouchedSlSources, hasTouchedSlSource } from '../../context/tools/touched-sl-sources.js'; import { detectCaptureSignals, isWorthAnalyzing } from './capture-signals.js'; import { MemoryAgentService } from './memory-agent.service.js'; diff --git a/packages/cli/src/context/memory/memory-agent.service.ts b/packages/cli/src/context/memory/memory-agent.service.ts index e29ad2f1..7f6438d3 100644 --- a/packages/cli/src/context/memory/memory-agent.service.ts +++ b/packages/cli/src/context/memory/memory-agent.service.ts @@ -3,24 +3,15 @@ import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import * as YAML from 'yaml'; import { z } from 'zod'; -import { type KtxLogger, noopLogger } from '../core/index.js'; -import type { KtxRuntimeToolSet } from '../llm/index.js'; -import { - revertSourceToPreHead, - type SemanticLayerSource, - type SlValidationDeps, - type SlValidatorPort, -} from '../sl/index.js'; -import { - createTouchedSlSources, - deleteTouchedSlSource, - listTouchedSlSources, - SYSTEM_GIT_AUTHOR, - type ToolContext, - type ToolSession, - touchedSlSourceCount, - touchedSlSourceNamesForConnection, -} from '../tools/index.js'; +import { type KtxLogger, noopLogger } from '../../context/core/config.js'; +import type { KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; +import { revertSourceToPreHead, type SlValidationDeps } from '../../context/sl/tools/sl-warehouse-validation.js'; +import type { SemanticLayerSource } from '../../context/sl/types.js'; +import type { SlValidatorPort } from '../../context/sl/sl-validator.port.js'; +import { createTouchedSlSources, deleteTouchedSlSource, listTouchedSlSources, touchedSlSourceCount, touchedSlSourceNamesForConnection } from '../../context/tools/touched-sl-sources.js'; +import { SYSTEM_GIT_AUTHOR } from '../../context/tools/authors.js'; +import type { ToolContext } from '../../context/tools/base-tool.js'; +import type { ToolSession } from '../../context/tools/tool-session.js'; import { buildRequiredSkillsBlock, DEFAULT_SKILL_NAMES, diff --git a/packages/cli/src/context/memory/memory-runs.test.ts b/packages/cli/src/context/memory/memory-runs.test.ts index 9fd9c36d..e4d0d4ba 100644 --- a/packages/cli/src/context/memory/memory-runs.test.ts +++ b/packages/cli/src/context/memory/memory-runs.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import type { MemoryAgentInput, MemoryAgentResult, MemoryAgentService } from './index.js'; +import type { MemoryAgentInput, MemoryAgentResult } from '../../context/memory/types.js'; +import type { MemoryAgentService } from '../../context/memory/memory-agent.service.js'; import { MemoryIngestService, type MemoryRunStorePort } from './memory-runs.js'; class InMemoryRunStore implements MemoryRunStorePort { diff --git a/packages/cli/src/context/memory/memory-runs.ts b/packages/cli/src/context/memory/memory-runs.ts index 2b34657e..9bf5dedf 100644 --- a/packages/cli/src/context/memory/memory-runs.ts +++ b/packages/cli/src/context/memory/memory-runs.ts @@ -1,5 +1,6 @@ import { createHash } from 'node:crypto'; -import type { MemoryAction, MemoryAgentInput, MemoryAgentResult, MemoryAgentService } from './index.js'; +import type { MemoryAction, MemoryAgentInput, MemoryAgentResult } from '../../context/memory/types.js'; +import type { MemoryAgentService } from '../../context/memory/memory-agent.service.js'; export type MemoryRunStatus = 'running' | 'done' | 'error'; diff --git a/packages/cli/src/context/memory/memory-runtime-assets.test.ts b/packages/cli/src/context/memory/memory-runtime-assets.test.ts index 68a53cfd..55d9047c 100644 --- a/packages/cli/src/context/memory/memory-runtime-assets.test.ts +++ b/packages/cli/src/context/memory/memory-runtime-assets.test.ts @@ -2,9 +2,10 @@ import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; -import { PromptService } from '../prompts/index.js'; -import { SkillsRegistryService } from '../skills/index.js'; -import { DEFAULT_SKILL_NAMES, type MemoryAgentSourceType, promptNameFor } from './index.js'; +import { PromptService } from '../../context/prompts/prompt.service.js'; +import { SkillsRegistryService } from '../../context/skills/skills-registry.service.js'; +import { DEFAULT_SKILL_NAMES, promptNameFor } from '../../context/memory/capture-signals.js'; +import type { MemoryAgentSourceType } from '../../context/memory/types.js'; const promptsDir = fileURLToPath(new URL('../../prompts', import.meta.url)); const skillsDir = fileURLToPath(new URL('../../skills', import.meta.url)); diff --git a/packages/cli/src/context/memory/types.ts b/packages/cli/src/context/memory/types.ts index bb2f3bad..5cc7dab2 100644 --- a/packages/cli/src/context/memory/types.ts +++ b/packages/cli/src/context/memory/types.ts @@ -1,19 +1,21 @@ -import type { AgentRunnerPort, KtxRuntimeToolSet } from '../llm/index.js'; -import type { GitService, KtxFileStorePort, KtxLogger, SessionWorktreeService } from '../core/index.js'; -import type { PromptService } from '../prompts/index.js'; -import type { SkillsRegistryService } from '../skills/index.js'; -import type { - KtxConnectionInfo, - KtxQueryResult, - SemanticLayerService, - SemanticLayerSource, - SlSearchService, - SlSourcesIndexPort, - SlValidationDeps, - SlValidatorPort, -} from '../sl/index.js'; -import type { ToolContext, ToolSession, TouchedSlSourceSet } from '../tools/index.js'; -import type { KnowledgeIndexPort, KnowledgeWikiService } from '../wiki/index.js'; +import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; +import type { GitService } from '../../context/core/git.service.js'; +import type { KtxFileStorePort } from '../../context/core/file-store.js'; +import type { KtxLogger } from '../../context/core/config.js'; +import type { SessionWorktreeService } from '../../context/core/session-worktree.service.js'; +import type { PromptService } from '../../context/prompts/prompt.service.js'; +import type { SkillsRegistryService } from '../../context/skills/skills-registry.service.js'; +import type { KtxConnectionInfo, KtxQueryResult, SlSourcesIndexPort } from '../../context/sl/ports.js'; +import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import type { SemanticLayerSource } from '../../context/sl/types.js'; +import type { SlSearchService } from '../../context/sl/sl-search.service.js'; +import type { SlValidationDeps } from '../../context/sl/tools/sl-warehouse-validation.js'; +import type { SlValidatorPort } from '../../context/sl/sl-validator.port.js'; +import type { ToolContext } from '../../context/tools/base-tool.js'; +import type { ToolSession } from '../../context/tools/tool-session.js'; +import type { TouchedSlSourceSet } from '../../context/tools/touched-sl-sources.js'; +import type { KnowledgeIndexPort } from '../../context/wiki/ports.js'; +import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; export type MemoryAgentSourceType = 'research' | 'external_ingest' | 'backfill'; @@ -62,7 +64,7 @@ export interface CaptureSession { preHead: string | null; } -export interface MemoryAgentSettings { +interface MemoryAgentSettings { knowledge: { userScopedKnowledgeEnabled: boolean; }; @@ -74,7 +76,7 @@ export interface MemoryAgentSettings { }; } -export interface MemoryTelemetryPort { +interface MemoryTelemetryPort { trackMemoryIngestion( userId: string, properties: { @@ -106,7 +108,7 @@ export interface MemoryConnectionPort { executeQuery(connectionId: string, sql: string): Promise; } -export interface MemoryCommitMessagePort { +interface MemoryCommitMessagePort { enqueueCommitMessageJobForExternalCommit( commit: { commitHash: string }, message: string, @@ -129,7 +131,7 @@ export interface MemorySlSourceReconcilerPort { upsertRow(parsed: SemanticLayerSource, path: string, contentHash: string): Promise; } -export interface MemoryLockPort { +interface MemoryLockPort { withLock(key: 'config:repo', fn: () => Promise): Promise; } diff --git a/packages/cli/src/context/project/config.ts b/packages/cli/src/context/project/config.ts index 65a884e0..e19ee6af 100644 --- a/packages/cli/src/context/project/config.ts +++ b/packages/cli/src/context/project/config.ts @@ -1,4 +1,4 @@ -import { KTX_MODEL_ROLES } from '../../llm/index.js'; +import { KTX_MODEL_ROLES } from '../../llm/types.js'; import YAML from 'yaml'; import * as z from 'zod'; import { connectionConfigSchema } from './driver-schemas.js'; @@ -260,8 +260,6 @@ export type KtxProjectEmbeddingConfig = z.infer; export type KtxScanEnrichmentConfig = z.infer; export type KtxScanRelationshipConfig = z.infer; export type KtxProjectConnectionConfig = z.infer; -export type KtxStorageState = z.infer['state']; -export type KtxSearchBackend = z.infer['search']; export interface KtxConfigIssue { path: string; diff --git a/packages/cli/src/context/project/driver-schemas.ts b/packages/cli/src/context/project/driver-schemas.ts index c3b819ea..2e690295 100644 --- a/packages/cli/src/context/project/driver-schemas.ts +++ b/packages/cli/src/context/project/driver-schemas.ts @@ -208,4 +208,3 @@ export const connectionConfigSchema = z.discriminatedUnion('driver', [ metricflowConnectionSchema, ]); -export type KtxConnectionConfig = z.infer; diff --git a/packages/cli/src/context/project/index.ts b/packages/cli/src/context/project/index.ts deleted file mode 100644 index a0c08767..00000000 --- a/packages/cli/src/context/project/index.ts +++ /dev/null @@ -1,45 +0,0 @@ -export type { - KtxConfigIssue, - KtxConfigValidation, - KtxProjectConfig, - KtxProjectConnectionConfig, - KtxProjectEmbeddingConfig, - KtxProjectLlmConfig, - KtxSearchBackend, - KtxStorageState, -} from './config.js'; -export { - buildDefaultKtxProjectConfig, - generateKtxProjectConfigJsonSchema, - parseKtxProjectConfig, - serializeKtxProjectConfig, - validateKtxProjectConfig, -} from './config.js'; -export type { KtxConnectionConfig } from './driver-schemas.js'; -export type { LocalGitFileStoreDeps } from './local-git-file-store.js'; -export { LocalGitFileStore } from './local-git-file-store.js'; -export { ktxLocalStateDbPath } from './local-state-db.js'; -export type { - ConnectionMappingBootstrap, - LookerMappingBootstrap, - LookmlMappingBootstrap, - MetabaseMappingBootstrap, -} from './mappings-yaml-schema.js'; -export { - parseConnectionMappingBootstrap, - parseLookerMappingBootstrap, - parseLookmlMappingBootstrap, - parseMetabaseMappingBootstrap, -} from './mappings-yaml-schema.js'; -export type { InitKtxProjectOptions, InitKtxProjectResult, KtxLocalProject, LoadKtxProjectOptions } from './project.js'; -export { initKtxProject, loadKtxProject } from './project.js'; -export type { KtxSetupStep } from './setup-config.js'; -export { - KTX_SETUP_STEPS, - ktxSetupStatePath, - markKtxSetupStateStepComplete, - mergeKtxSetupGitignoreEntries, - readKtxSetupState, - setKtxSetupDatabaseConnectionIds, - writeKtxSetupState, -} from './setup-config.js'; diff --git a/packages/cli/src/context/project/local-git-file-store.test.ts b/packages/cli/src/context/project/local-git-file-store.test.ts index 62b7fc8c..1bee3c1e 100644 --- a/packages/cli/src/context/project/local-git-file-store.test.ts +++ b/packages/cli/src/context/project/local-git-file-store.test.ts @@ -2,7 +2,8 @@ import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { GitService, type KtxCoreConfig } from '../core/index.js'; +import { GitService } from '../../context/core/git.service.js'; +import type { KtxCoreConfig } from '../../context/core/config.js'; import { LocalGitFileStore } from './local-git-file-store.js'; describe('LocalGitFileStore', () => { diff --git a/packages/cli/src/context/project/local-git-file-store.ts b/packages/cli/src/context/project/local-git-file-store.ts index 8d1bd065..51757cb2 100644 --- a/packages/cli/src/context/project/local-git-file-store.ts +++ b/packages/cli/src/context/project/local-git-file-store.ts @@ -1,14 +1,7 @@ import { promises as fs } from 'node:fs'; import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; -import type { - GitCommitInfo, - GitService, - KtxFileHistoryEntry, - KtxFileListResult, - KtxFileReadResult, - KtxFileStorePort, - KtxFileWriteResult, -} from '../core/index.js'; +import type { GitCommitInfo, GitService } from '../../context/core/git.service.js'; +import type { KtxFileHistoryEntry, KtxFileListResult, KtxFileReadResult, KtxFileStorePort, KtxFileWriteResult } from '../../context/core/file-store.js'; export interface LocalGitFileStoreDeps { rootDir: string; diff --git a/packages/cli/src/context/project/mappings-yaml-schema.ts b/packages/cli/src/context/project/mappings-yaml-schema.ts index 43ad6515..9c69e1b3 100644 --- a/packages/cli/src/context/project/mappings-yaml-schema.ts +++ b/packages/cli/src/context/project/mappings-yaml-schema.ts @@ -69,6 +69,7 @@ export type LookerMappingBootstrap = { connectionMappings: Record; }; +/** @internal */ export type LookmlMappingBootstrap = { adapter: 'lookml'; connectionId: string; @@ -117,6 +118,7 @@ export function parseMetabaseMappingBootstrap( }; } +/** @internal */ export function parseLookerMappingBootstrap( connectionId: string, connection: MappingConnectionInput, @@ -129,6 +131,7 @@ export function parseLookerMappingBootstrap( }; } +/** @internal */ export function parseLookmlMappingBootstrap( connectionId: string, connection: MappingConnectionInput, diff --git a/packages/cli/src/context/project/project.ts b/packages/cli/src/context/project/project.ts index 572ac325..56fca5b2 100644 --- a/packages/cli/src/context/project/project.ts +++ b/packages/cli/src/context/project/project.ts @@ -1,6 +1,7 @@ import { promises as fs } from 'node:fs'; import { basename, dirname, join, resolve } from 'node:path'; -import { GitService, type KtxCoreConfig, type KtxLogger, noopLogger } from '../core/index.js'; +import { GitService } from '../../context/core/git.service.js'; +import { type KtxCoreConfig, type KtxLogger, noopLogger } from '../../context/core/config.js'; import type { KtxProjectConfig } from './config.js'; import { buildDefaultKtxProjectConfig, parseKtxProjectConfig, serializeKtxProjectConfig } from './config.js'; import { LocalGitFileStore } from './local-git-file-store.js'; diff --git a/packages/cli/src/context/project/setup-config.ts b/packages/cli/src/context/project/setup-config.ts index 3f2d6534..f790597e 100644 --- a/packages/cli/src/context/project/setup-config.ts +++ b/packages/cli/src/context/project/setup-config.ts @@ -2,7 +2,7 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import type { KtxProjectConfig } from './config.js'; -export const KTX_SETUP_STEPS = [ +const KTX_SETUP_STEPS = [ 'project', 'llm', 'embeddings', @@ -40,7 +40,7 @@ function uniqueSetupSteps(steps: unknown): KtxSetupStep[] { return [...new Set(steps.filter(isKtxSetupStep))]; } -export function ktxSetupStatePath(projectDir: string): string { +function ktxSetupStatePath(projectDir: string): string { return join(projectDir, '.ktx', 'setup', 'state.json'); } @@ -56,6 +56,7 @@ export async function readKtxSetupState(projectDir: string): Promise { await mkdir(join(projectDir, '.ktx', 'setup'), { recursive: true }); await writeFile( diff --git a/packages/cli/src/context/prompts/index.ts b/packages/cli/src/context/prompts/index.ts deleted file mode 100644 index c70cbfc2..00000000 --- a/packages/cli/src/context/prompts/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export type { PromptContext, PromptServiceOptions } from './prompt.service.js'; -export { PromptService } from './prompt.service.js'; diff --git a/packages/cli/src/context/prompts/prompt.service.ts b/packages/cli/src/context/prompts/prompt.service.ts index 1864f120..b18fc0b0 100644 --- a/packages/cli/src/context/prompts/prompt.service.ts +++ b/packages/cli/src/context/prompts/prompt.service.ts @@ -1,7 +1,7 @@ import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import Handlebars from 'handlebars'; -import { type KtxLogger, noopLogger } from '../core/index.js'; +import { type KtxLogger, noopLogger } from '../../context/core/config.js'; export interface PromptContext { current_date?: string; diff --git a/packages/cli/src/context/scan/credentials.test.ts b/packages/cli/src/context/scan/credentials.test.ts index 60b6c2e9..891c58a9 100644 --- a/packages/cli/src/context/scan/credentials.test.ts +++ b/packages/cli/src/context/scan/credentials.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; +import { REDACTED_KTX_CREDENTIAL_VALUE } from '../core/redaction.js'; import { - REDACTED_KTX_CREDENTIAL_VALUE, redactKtxCredentialEnvelope, redactKtxCredentialValue, redactKtxScanMetadata, diff --git a/packages/cli/src/context/scan/credentials.ts b/packages/cli/src/context/scan/credentials.ts index 23fd634e..07ad8e24 100644 --- a/packages/cli/src/context/scan/credentials.ts +++ b/packages/cli/src/context/scan/credentials.ts @@ -2,20 +2,20 @@ import { redactKtxSensitiveMetadata, redactKtxSensitiveText, redactKtxSensitiveValue, - REDACTED_KTX_CREDENTIAL_VALUE, } from '../core/redaction.js'; import type { KtxCredentialEnvelope, KtxScanReport, KtxScanWarning } from './types.js'; -export { REDACTED_KTX_CREDENTIAL_VALUE }; - +/** @internal */ export function redactKtxCredentialValue(key: string, value: unknown): unknown { return redactKtxSensitiveValue(key, value); } +/** @internal */ export function redactKtxScanMetadata(metadata: Record): Record { return redactKtxSensitiveMetadata(metadata); } +/** @internal */ export function redactKtxCredentialEnvelope(envelope: KtxCredentialEnvelope): KtxCredentialEnvelope { if (envelope.kind !== 'resolved') { return envelope; @@ -28,6 +28,7 @@ export function redactKtxCredentialEnvelope(envelope: KtxCredentialEnvelope): Kt }; } +/** @internal */ export function redactKtxScanWarning(warning: KtxScanWarning): KtxScanWarning { if (!warning.metadata) { return { diff --git a/packages/cli/src/context/scan/data-dictionary.ts b/packages/cli/src/context/scan/data-dictionary.ts index 6c1acab7..859b9cc6 100644 --- a/packages/cli/src/context/scan/data-dictionary.ts +++ b/packages/cli/src/context/scan/data-dictionary.ts @@ -36,17 +36,19 @@ export const defaultKtxDataDictionarySettings: KtxDataDictionarySettings = { ], }; -export type KtxDataDictionarySkipReason = +type KtxDataDictionarySkipReason = | 'not_candidate' | 'already_populated' | 'empty_column' | 'high_cardinality'; +/** @internal */ export interface KtxDataDictionarySampleDecision { sample: boolean; reason?: KtxDataDictionarySkipReason; } +/** @internal */ export interface KtxDataDictionaryColumnState { columnType: string; columnName: string; @@ -83,6 +85,7 @@ export function isKtxDataDictionaryCandidate( return true; } +/** @internal */ export function shouldKtxSampleColumnForDictionary( input: KtxDataDictionaryColumnState, ): KtxDataDictionarySampleDecision { diff --git a/packages/cli/src/context/scan/description-generation.ts b/packages/cli/src/context/scan/description-generation.ts index ba61fb62..4526215d 100644 --- a/packages/cli/src/context/scan/description-generation.ts +++ b/packages/cli/src/context/scan/description-generation.ts @@ -1,4 +1,4 @@ -import type { KtxLlmRuntimePort } from '../llm/index.js'; +import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; import type { KtxColumnSampleInput, KtxColumnSampleResult, @@ -24,13 +24,13 @@ export interface KtxDescriptionCachePort { set(key: string, value: string): Promise; } -export interface KtxDescriptionSamplingPort { +interface KtxDescriptionSamplingPort { id: string; sampleColumn?(input: KtxColumnSampleInput, ctx: KtxScanContext): Promise; sampleTable?(input: KtxTableSampleInput, ctx: KtxScanContext): Promise; } -export interface KtxDescriptionGenerationSettings { +interface KtxDescriptionGenerationSettings { columnMaxWords: number; tableMaxWords: number; dataSourceMaxWords: number; @@ -57,7 +57,7 @@ export interface KtxDescriptionColumnTable extends KtxTableRef { columns: KtxDescriptionColumn[]; } -export interface KtxDescriptionTableInput extends KtxTableRef { +interface KtxDescriptionTableInput extends KtxTableRef { rawDescriptions?: Record; columns?: KtxDescriptionTableColumn[]; } @@ -68,6 +68,7 @@ export interface KtxColumnAnalysisResult { skippedColumns: string[]; } +/** @internal */ export interface KtxColumnDescriptionPromptInput { columnName: string; columnValues: unknown[]; @@ -77,6 +78,7 @@ export interface KtxColumnDescriptionPromptInput { rawDescriptions?: Record; } +/** @internal */ export interface KtxTableDescriptionPromptInput { tableName: string; sampleData?: KtxTableSampleResult; @@ -85,6 +87,7 @@ export interface KtxTableDescriptionPromptInput { rawDescriptions?: Record; } +/** @internal */ export interface KtxDataSourceDescriptionPromptInput { tableSamples: Array<[string, KtxTableSampleResult]>; dataSourceType: string; @@ -247,6 +250,7 @@ function wordLimitLine(maxWords: number): string { return `Please provide a concise description in ${maxWords} words or less.`; } +/** @internal */ export function buildKtxColumnDescriptionPrompt( input: KtxColumnDescriptionPromptInput & { maxWords?: number }, ): KtxDescriptionPrompt { @@ -295,6 +299,7 @@ Example: return { system: systemParts.join('\n\n'), user: user.trim() }; } +/** @internal */ export function buildKtxTableDescriptionPrompt( input: KtxTableDescriptionPromptInput & { maxWords?: number }, ): KtxDescriptionPrompt { @@ -363,6 +368,7 @@ Data source type: ${input.dataSourceType}`; return { system: systemParts.join('\n\n'), user: user.trim() }; } +/** @internal */ export function buildKtxDataSourceDescriptionPrompt( input: KtxDataSourceDescriptionPromptInput & { maxWords?: number }, ): KtxDescriptionPrompt { diff --git a/packages/cli/src/context/scan/embedding-text.ts b/packages/cli/src/context/scan/embedding-text.ts index 927f5779..375e24c2 100644 --- a/packages/cli/src/context/scan/embedding-text.ts +++ b/packages/cli/src/context/scan/embedding-text.ts @@ -1,4 +1,4 @@ -export interface KtxColumnEmbeddingForeignKeys { +interface KtxColumnEmbeddingForeignKeys { outgoing: Array<{ toTable: string; toColumn: string }>; incoming: Array<{ fromTable: string; fromColumn: string }>; } diff --git a/packages/cli/src/context/scan/enrichment-state.ts b/packages/cli/src/context/scan/enrichment-state.ts index 8eb5f421..4b4913e7 100644 --- a/packages/cli/src/context/scan/enrichment-state.ts +++ b/packages/cli/src/context/scan/enrichment-state.ts @@ -1,7 +1,7 @@ import { createHash } from 'node:crypto'; import type { KtxScanEnrichmentStage, KtxScanEnrichmentStateSummary, KtxScanMode, KtxSchemaSnapshot } from './types.js'; -export const KTX_SCAN_ENRICHMENT_STAGES: readonly KtxScanEnrichmentStage[] = [ +const KTX_SCAN_ENRICHMENT_STAGES: readonly KtxScanEnrichmentStage[] = [ 'descriptions', 'embeddings', 'relationships', diff --git a/packages/cli/src/context/scan/enrichment-types.ts b/packages/cli/src/context/scan/enrichment-types.ts index 645951c3..2d19fa5c 100644 --- a/packages/cli/src/context/scan/enrichment-types.ts +++ b/packages/cli/src/context/scan/enrichment-types.ts @@ -1,8 +1,8 @@ import type { KtxSchemaDimensionType, KtxTableRef } from './types.js'; -export type KtxDescriptionSource = 'ai' | 'db' | 'dbt' | 'user' | (string & {}); +type KtxDescriptionSource = 'ai' | 'db' | 'dbt' | 'user' | (string & {}); -export type KtxRelationshipSource = 'formal' | 'inferred' | 'manual'; +type KtxRelationshipSource = 'formal' | 'inferred' | 'manual'; export type KtxRelationshipType = 'many_to_one' | 'one_to_many' | 'one_to_one'; @@ -54,12 +54,14 @@ export interface KtxEnrichedSchema { relationships: KtxEnrichedRelationship[]; } +/** @internal */ export interface KtxStructuralSyncPlan { connectionId: string; snapshotId: string; operations: Array>; } +/** @internal */ export interface KtxDescriptionUpdate { connectionId: string; table: KtxTableRef; @@ -68,14 +70,8 @@ export interface KtxDescriptionUpdate { columnDescriptions?: Record; } -export interface KtxMetadataUpdate { - connectionId: string; - table: KtxTableRef; - source: KtxDescriptionSource; - tableFields?: Record; - columnFields?: Record>; -} +/** @internal */ export interface KtxJoinUpdate { connectionId: string; fromTable: string; @@ -87,6 +83,7 @@ export interface KtxJoinUpdate { authorEmail: string; } +/** @internal */ export interface KtxColumnSampleUpdate { columnId: string; sampleValues: string[] | null; @@ -111,6 +108,7 @@ export interface KtxRelationshipUpdate { skipped: KtxSkippedRelationship[]; } +/** @internal */ export interface KtxScanMetadataStore { loadSchema(connectionId: string): Promise; applyStructuralPlan(plan: KtxStructuralSyncPlan): Promise; diff --git a/packages/cli/src/context/scan/entity-details.test.ts b/packages/cli/src/context/scan/entity-details.test.ts index db81ad11..ddccef87 100644 --- a/packages/cli/src/context/scan/entity-details.test.ts +++ b/packages/cli/src/context/scan/entity-details.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { createKtxEntityDetailsService } from './entity-details.js'; import type { KtxConnectionDriver, KtxScanReport, KtxSchemaTable } from './types.js'; diff --git a/packages/cli/src/context/scan/entity-details.ts b/packages/cli/src/context/scan/entity-details.ts index 6e95690e..37e766b6 100644 --- a/packages/cli/src/context/scan/entity-details.ts +++ b/packages/cli/src/context/scan/entity-details.ts @@ -1,4 +1,4 @@ -import type { KtxLocalProject } from '../project/index.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import { readLocalScanStructuralSnapshot } from './local-structural-artifacts.js'; import type { KtxConnectionDriver, @@ -25,7 +25,7 @@ export interface KtxEntityDetailsSnapshotInfo { scanRunId: string | null; } -export interface KtxEntityDetailsColumn { +interface KtxEntityDetailsColumn { name: string; nativeType: string; normalizedType: string; @@ -35,7 +35,7 @@ export interface KtxEntityDetailsColumn { comment: string | null; } -export interface KtxEntityDetailsRecord { +interface KtxEntityDetailsRecord { ok: true; connectionId: string; tableRef: KtxTableRef; @@ -48,9 +48,9 @@ export interface KtxEntityDetailsRecord { snapshot: KtxEntityDetailsSnapshotInfo; } -export type KtxEntityDetailsErrorCode = 'scan_missing' | 'table_not_found' | 'ambiguous_table' | 'column_not_found'; +type KtxEntityDetailsErrorCode = 'scan_missing' | 'table_not_found' | 'ambiguous_table' | 'column_not_found'; -export interface KtxEntityDetailsErrorResult { +interface KtxEntityDetailsErrorResult { ok: false; connectionId: string; table: KtxEntityDetailsTableInput; diff --git a/packages/cli/src/context/scan/index.ts b/packages/cli/src/context/scan/index.ts deleted file mode 100644 index 6b4cb950..00000000 --- a/packages/cli/src/context/scan/index.ts +++ /dev/null @@ -1,409 +0,0 @@ -export { - REDACTED_KTX_CREDENTIAL_VALUE, - redactKtxCredentialEnvelope, - redactKtxCredentialValue, - redactKtxScanMetadata, - redactKtxScanReport, - redactKtxScanWarning, -} from './credentials.js'; -export type { - KtxDataDictionaryColumnState, - KtxDataDictionarySampleDecision, - KtxDataDictionarySettings, - KtxDataDictionarySkipReason, -} from './data-dictionary.js'; -export { - defaultKtxDataDictionarySettings, - isKtxDataDictionaryCandidate, - shouldKtxSampleColumnForDictionary, -} from './data-dictionary.js'; -export type { - KtxColumnAnalysisResult, - KtxColumnDescriptionPromptInput, - KtxDataSourceDescriptionPromptInput, - KtxDescriptionCachePort, - KtxDescriptionColumn, - KtxDescriptionColumnTable, - KtxDescriptionGenerationSettings, - KtxDescriptionGeneratorOptions, - KtxDescriptionSamplingPort, - KtxDescriptionTableInput, - KtxGenerateColumnDescriptionsInput, - KtxGenerateDataSourceDescriptionInput, - KtxGenerateTableDescriptionInput, - KtxTableDescriptionPromptInput, -} from './description-generation.js'; -export { - buildKtxColumnDescriptionPrompt, - buildKtxDataSourceDescriptionPrompt, - buildKtxTableDescriptionPrompt, - KtxDescriptionGenerator, -} from './description-generation.js'; -export type { KtxColumnEmbeddingForeignKeys, KtxColumnEmbeddingTextInput } from './embedding-text.js'; -export { buildKtxColumnEmbeddingText } from './embedding-text.js'; -export type { - ComputeKtxScanEnrichmentInputHashInput, - KtxScanEnrichmentCompletedStage, - KtxScanEnrichmentFailedStage, - KtxScanEnrichmentStageLookup, - KtxScanEnrichmentStageRecord, - KtxScanEnrichmentStateStore, -} from './enrichment-state.js'; -export { - completedKtxScanEnrichmentStateSummary, - computeKtxScanEnrichmentInputHash, - KTX_SCAN_ENRICHMENT_STAGES, - summarizeKtxScanEnrichmentState, -} from './enrichment-state.js'; -export { - failedKtxScanEnrichmentSummary, - ktxScanErrorMessage, - skippedKtxScanEnrichmentSummary, -} from './enrichment-summary.js'; -export type { - KtxEntityDetailsColumn, - KtxEntityDetailsErrorCode, - KtxEntityDetailsErrorResult, - KtxEntityDetailsInput, - KtxEntityDetailsRecord, - KtxEntityDetailsResponse, - KtxEntityDetailsSnapshotInfo, - KtxEntityDetailsTableInput, -} from './entity-details.js'; -export { createKtxEntityDetailsService } from './entity-details.js'; -export type { - DisplayTargetResolution, - RawSchemaHit, - TableDetail, - WarehouseCatalogServiceDeps, -} from './warehouse-catalog.js'; -export { WarehouseCatalogService } from './warehouse-catalog.js'; -export type { - KtxColumnSampleUpdate, - KtxDescriptionSource, - KtxDescriptionUpdate, - KtxEmbeddingUpdate, - KtxEnrichedColumn, - KtxEnrichedRelationship, - KtxEnrichedSchema, - KtxEnrichedTable, - KtxRelationshipEndpoint, - KtxRelationshipSource, - KtxRelationshipType, - KtxRelationshipUpdate, - KtxScanMetadataStore, - KtxSkippedRelationship, - KtxStructuralSyncPlan, -} from './enrichment-types.js'; -export type { - KtxLocalScanEnrichmentInput, - KtxLocalScanEnrichmentProviders, - KtxLocalScanEnrichmentResult, -} from './local-enrichment.js'; -export { - createDeterministicLocalScanEnrichmentProviders, - runLocalScanEnrichment, - snapshotToKtxEnrichedSchema, -} from './local-enrichment.js'; -export type { - WriteLocalScanEnrichmentArtifactsInput, - WriteLocalScanEnrichmentArtifactsResult, - WriteLocalScanManifestShardsInput, - WriteLocalScanManifestShardsResult, -} from './local-enrichment-artifacts.js'; -export { - writeLocalScanEnrichmentArtifacts, - writeLocalScanManifestShards, -} from './local-enrichment-artifacts.js'; -export type { - LocalScanMcpOptions, - LocalScanRunResult, - LocalScanStatusResponse, - RunLocalScanOptions, -} from './local-scan.js'; -export { filterSnapshotTables, getLocalScanReport, getLocalScanStatus, resolveEnabledTables, runLocalScan } from './local-scan.js'; -export type { ReadLocalScanStructuralSnapshotInput } from './local-structural-artifacts.js'; -export { readLocalScanStructuralSnapshot } from './local-structural-artifacts.js'; -export type { - KtxRelationshipArtifactStatus, - ReadLocalScanRelationshipArtifactsResult, -} from './relationship-artifacts.js'; -export { readLocalScanRelationshipArtifacts } from './relationship-artifacts.js'; -export type { - KtxRelationshipBenchmarkReport, - KtxRelationshipBenchmarkReportCase, - KtxRelationshipBenchmarkReportCaseStatus, -} from './relationship-benchmark-report.js'; -export { - buildKtxRelationshipBenchmarkReport, - formatKtxRelationshipBenchmarkReportMarkdown, -} from './relationship-benchmark-report.js'; -export type { - KtxRelationshipBenchmarkCaseResult, - KtxRelationshipBenchmarkDetectedLink, - KtxRelationshipBenchmarkDetectedPk, - KtxRelationshipBenchmarkDetector, - KtxRelationshipBenchmarkDetectorInput, - KtxRelationshipBenchmarkDetectorResult, - KtxRelationshipBenchmarkExpectedLink, - KtxRelationshipBenchmarkExpectedLinks, - KtxRelationshipBenchmarkExpectedPk, - KtxRelationshipBenchmarkFixture, - KtxRelationshipBenchmarkMetrics, - KtxRelationshipBenchmarkMode, - KtxRelationshipBenchmarkStatus, - KtxRelationshipBenchmarkSuiteResult, - KtxRelationshipBenchmarkTier, -} from './relationship-benchmarks.js'; -export { - currentKtxRelationshipBenchmarkDetector, - ktxRelationshipBenchmarkDetectorWithLlm, - KTX_RELATIONSHIP_BENCHMARK_MODES, - KTX_RELATIONSHIP_BENCHMARK_TIERS, - loadKtxRelationshipBenchmarkFixture, - loadKtxRelationshipBenchmarkFixtures, - maskKtxRelationshipBenchmarkSnapshot, - runKtxRelationshipBenchmarkCase, - runKtxRelationshipBenchmarkSuite, -} from './relationship-benchmarks.js'; -export type { - ApplyKtxRelationshipValidationBudgetInput, - KtxRelationshipBudgetedCandidate, - KtxRelationshipValidationBudget, - KtxRelationshipValidationBudgetResult, -} from './relationship-budget.js'; -export { - applyKtxRelationshipValidationBudget, - defaultKtxRelationshipValidationBudget, -} from './relationship-budget.js'; -export type { - KtxRelationshipDiscoveryCandidate, - KtxRelationshipDiscoveryCandidateEvidence, - KtxRelationshipDiscoveryCandidateOptions, - KtxRelationshipDiscoveryCandidateSource, - KtxRelationshipDiscoveryCandidateStatus, - KtxRelationshipInferredTargetPk, -} from './relationship-candidates.js'; -export { - generateKtxRelationshipDiscoveryCandidates, - inferKtxRelationshipTargetPks, - mergeKtxRelationshipDiscoveryCandidates, -} from './relationship-candidates.js'; -export type { - DiscoverKtxCompositeRelationshipsInput, - DiscoverKtxCompositeRelationshipsResult, - KtxCompositePrimaryKeyCandidate, - KtxCompositeRelationshipCandidate, - KtxCompositeRelationshipStatus, - KtxCompositeRelationshipTupleEndpoint, - KtxCompositeRelationshipValidationEvidence, -} from './relationship-composite-candidates.js'; -export { discoverKtxCompositeRelationships } from './relationship-composite-candidates.js'; -export type { - BuildKtxRelationshipArtifactsInput, - BuildKtxRelationshipDiagnosticsInput, - EmptyKtxRelationshipProfileArtifactInput, - KtxRelationshipArtifact, - KtxRelationshipArtifactEdge, - KtxRelationshipArtifactEndpoint, - KtxRelationshipDiagnosticsArtifact, - KtxRelationshipDiagnosticsSummary, - KtxRelationshipDiagnosticsThresholds, - KtxRelationshipDiagnosticsValidation, -} from './relationship-diagnostics.js'; -export { - buildKtxRelationshipArtifacts, - buildKtxRelationshipDiagnostics, - emptyKtxRelationshipProfileArtifact, -} from './relationship-diagnostics.js'; -export type { - BuildKtxRelationshipFeedbackCalibrationReportInput, - CalibrateLocalRelationshipFeedbackLabelsInput, - KtxRelationshipFeedbackCalibrationBucket, - KtxRelationshipFeedbackCalibrationLabel, - KtxRelationshipFeedbackCalibrationReport, -} from './relationship-feedback-calibration.js'; -export { - buildKtxRelationshipFeedbackCalibrationReport, - calibrateLocalRelationshipFeedbackLabels, - formatKtxRelationshipFeedbackCalibrationMarkdown, -} from './relationship-feedback-calibration.js'; -export type { - ExportLocalRelationshipFeedbackLabelsInput, - ExportLocalRelationshipFeedbackLabelsResult, - KtxRelationshipFeedbackDecisionFilter, - KtxRelationshipFeedbackExportWarning, - KtxRelationshipFeedbackLabel, -} from './relationship-feedback-export.js'; -export { - exportLocalRelationshipFeedbackLabels, - formatKtxRelationshipFeedbackLabelsJsonl, -} from './relationship-feedback-export.js'; -export { - collectKtxFormalMetadataRelationships, - type KtxFormalMetadataRelationshipCollection, -} from './relationship-formal-metadata.js'; -export type { - KtxRelationshipGraphResolutionResult, - KtxRelationshipGraphResolverSettings, - KtxResolvedRelationshipDiscoveryCandidate, - KtxResolvedRelationshipGraphEvidence, - KtxResolvedRelationshipPk, - KtxResolvedRelationshipPkEvidence, - KtxResolvedRelationshipStatus, - ResolveKtxRelationshipGraphInput, -} from './relationship-graph-resolver.js'; -export { resolveKtxRelationshipGraph } from './relationship-graph-resolver.js'; -export type { - KtxRelationshipLlmProposalResult, - KtxRelationshipLlmProposalSettings, - ProposeKtxRelationshipCandidatesWithLlmInput, -} from './relationship-llm-proposal.js'; -export { proposeKtxRelationshipCandidatesWithLlm } from './relationship-llm-proposal.js'; -export type { - KtxRelationshipLocalityCandidateTable, - LocalKtxRelationshipCandidateTablesInput, -} from './relationship-locality.js'; -export { localCandidateTables } from './relationship-locality.js'; -export type { - KtxRelationshipNormalizedName, - KtxRelationshipTokenInput, -} from './relationship-name-similarity.js'; -export { - normalizeKtxRelationshipName, - pluralizeKtxRelationshipToken, - singularizeKtxRelationshipToken, - tokenizeKtxRelationshipName, - tokenSimilarity, -} from './relationship-name-similarity.js'; -export type { - DiscoverKtxRelationshipsInput, - DiscoverKtxRelationshipsResult, -} from './relationship-discovery.js'; -export { discoverKtxRelationships } from './relationship-discovery.js'; -export type { - KtxRelationshipColumnProfile, - KtxRelationshipProfileArtifact, - KtxRelationshipReadOnlyExecutor, - KtxRelationshipTableProfile, - ProfileKtxRelationshipSchemaInput, -} from './relationship-profiling.js'; -export { - formatKtxRelationshipTableRef, - profileKtxRelationshipSchema, - quoteKtxRelationshipIdentifier, -} from './relationship-profiling.js'; -export type { - AppliedRelationshipReviewDecision, - ApplyLocalScanRelationshipReviewDecisionsInput, - ApplyLocalScanRelationshipReviewDecisionsResult, -} from './relationship-review-apply.js'; -export { applyLocalScanRelationshipReviewDecisions } from './relationship-review-apply.js'; -export type { - KtxRelationshipReviewDecisionArtifact, - KtxRelationshipReviewDecisionEntry, - KtxRelationshipReviewDecisionValue, - WriteLocalScanRelationshipReviewDecisionInput, - WriteLocalScanRelationshipReviewDecisionResult, -} from './relationship-review-decisions.js'; -export { writeLocalScanRelationshipReviewDecision } from './relationship-review-decisions.js'; -export type { - KtxRelationshipFixtureOrigin, - KtxRelationshipScoreBreakdown, - KtxRelationshipScoreSignal, - KtxRelationshipScoreWeights, - KtxRelationshipScoringCalibrationObservation, - KtxRelationshipSignalVector, -} from './relationship-scoring.js'; -export { - calibrateWeightsFromSyntheticFixtures, - defaultKtxRelationshipScoreWeights, - KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS, - normalizeKtxRelationshipScoreWeights, - scoreKtxRelationshipCandidate, -} from './relationship-scoring.js'; -export type { - AdviseLocalRelationshipFeedbackThresholdsInput, - BuildKtxRelationshipThresholdAdviceReportInput, - KtxRelationshipThresholdAdviceCandidate, - KtxRelationshipThresholdAdviceReport, - KtxRelationshipThresholdAdviceStatus, -} from './relationship-threshold-advice.js'; -export { - adviseLocalRelationshipFeedbackThresholds, - buildKtxRelationshipThresholdAdviceReport, - formatKtxRelationshipThresholdAdviceMarkdown, -} from './relationship-threshold-advice.js'; -export type { - KtxRelationshipValidationEvidence, - KtxRelationshipValidationSettings, - KtxValidatedRelationshipDiscoveryCandidate, - KtxValidatedRelationshipStatus, - ValidateKtxRelationshipDiscoveryCandidatesInput, -} from './relationship-validation.js'; -export { validateKtxRelationshipDiscoveryCandidates } from './relationship-validation.js'; -export type { SqliteLocalScanEnrichmentStateStoreOptions } from './sqlite-local-enrichment-state-store.js'; -export { SqliteLocalScanEnrichmentStateStore } from './sqlite-local-enrichment-state-store.js'; -export type { KtxColumnTypeMapping } from './type-normalization.js'; -export { - inferKtxDimensionType, - ktxColumnTypeMappingFromNative, - normalizeKtxNativeType, -} from './type-normalization.js'; -export type { - KtxColumnSampleInput, - KtxColumnSampleResult, - KtxColumnStatsInput, - KtxColumnStatsResult, - KtxConnectionDriver, - KtxConnectorCapabilities, - KtxCredentialEnvelope, - KtxCredentialEnvReference, - KtxCredentialFileReference, - KtxEmbeddingPort, - KtxEventPropertyDiscovery, - KtxEventPropertyDiscoveryInput, - KtxEventPropertyValuesInput, - KtxEventPropertyValuesResult, - KtxEventStreamDiscoveryPort, - KtxEventTypeDiscovery, - KtxEventTypeDiscoveryInput, - KtxNetworkEndpoint, - KtxNetworkTunnelPort, - KtxNetworkTunnelRequest, - KtxOptionalConnectorCapabilities, - KtxProgressPort, - KtxProgressUpdateOptions, - KtxQueryResult, - KtxReadOnlyQueryInput, - KtxResolvedCredentialEnvelope, - KtxConnectorTestResult, - KtxScanArtifactPaths, - KtxScanConnector, - KtxScanContext, - KtxScanDiffSummary, - KtxScanEnrichmentStage, - KtxScanEnrichmentStateSummary, - KtxScanEnrichmentSummary, - KtxScanInput, - KtxScanLoggerPort, - KtxScanMode, - KtxScanRelationshipSummary, - KtxScanReport, - KtxScanTrigger, - KtxScanWarning, - KtxScanWarningCode, - KtxSchemaColumn, - KtxSchemaDimensionType, - KtxSchemaForeignKey, - KtxSchemaScope, - KtxSchemaSnapshot, - KtxSchemaTable, - KtxSchemaTableKind, - KtxStructuralSyncStats, - KtxTableListEntry, - KtxTableRef, - KtxTableSampleInput, - KtxTableSampleResult, -} from './types.js'; -export { createKtxConnectorCapabilities } from './types.js'; diff --git a/packages/cli/src/context/scan/local-enrichment-artifacts.test.ts b/packages/cli/src/context/scan/local-enrichment-artifacts.test.ts index 068e6cb1..56994568 100644 --- a/packages/cli/src/context/scan/local-enrichment-artifacts.test.ts +++ b/packages/cli/src/context/scan/local-enrichment-artifacts.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import YAML from 'yaml'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import type { KtxLocalScanEnrichmentResult } from './local-enrichment.js'; import { writeLocalScanEnrichmentArtifacts, writeLocalScanManifestShards } from './local-enrichment-artifacts.js'; import type { KtxSchemaSnapshot } from './types.js'; diff --git a/packages/cli/src/context/scan/local-enrichment-artifacts.ts b/packages/cli/src/context/scan/local-enrichment-artifacts.ts index ed698806..3a2d15f6 100644 --- a/packages/cli/src/context/scan/local-enrichment-artifacts.ts +++ b/packages/cli/src/context/scan/local-enrichment-artifacts.ts @@ -1,15 +1,8 @@ import YAML from 'yaml'; -import { - buildLiveDatabaseManifestShards, - type LiveDatabaseManifestExistingDescriptions, - type LiveDatabaseManifestJoinData, - type LiveDatabaseManifestJoinEntry, - type LiveDatabaseManifestShard, - type LiveDatabaseManifestTableData, - type TableUsageOutput, -} from '../ingest/index.js'; +import { buildLiveDatabaseManifestShards, type LiveDatabaseManifestExistingDescriptions, type LiveDatabaseManifestJoinData, type LiveDatabaseManifestJoinEntry, type LiveDatabaseManifestShard, type LiveDatabaseManifestTableData } from '../../context/ingest/adapters/live-database/manifest.js'; +import type { TableUsageOutput } from '../../context/ingest/adapters/historic-sql/skill-schemas.js'; import type { KtxScanRelationshipConfig } from '../project/config.js'; -import type { KtxLocalProject } from '../project/index.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import type { KtxLocalScanEnrichmentResult } from './local-enrichment.js'; import { buildKtxRelationshipArtifacts, diff --git a/packages/cli/src/context/scan/local-enrichment.ts b/packages/cli/src/context/scan/local-enrichment.ts index 381f3cb0..680f8f60 100644 --- a/packages/cli/src/context/scan/local-enrichment.ts +++ b/packages/cli/src/context/scan/local-enrichment.ts @@ -1,5 +1,5 @@ import pLimit from 'p-limit'; -import type { KtxLlmRuntimePort } from '../llm/index.js'; +import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; import { buildDefaultKtxProjectConfig, type KtxScanRelationshipConfig } from '../project/config.js'; import { type KtxDescriptionColumnTable, KtxDescriptionGenerator } from './description-generation.js'; import { buildKtxColumnEmbeddingText } from './embedding-text.js'; diff --git a/packages/cli/src/context/scan/local-scan.test.ts b/packages/cli/src/context/scan/local-scan.test.ts index d04f9953..081fa055 100644 --- a/packages/cli/src/context/scan/local-scan.test.ts +++ b/packages/cli/src/context/scan/local-scan.test.ts @@ -3,10 +3,11 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import YAML from 'yaml'; -import type { SourceAdapter } from '../ingest/index.js'; -import type { KtxLlmRuntimePort } from '../llm/index.js'; -import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js'; -import { filterSnapshotTables, getLocalScanReport, getLocalScanStatus, resolveEnabledTables, runLocalScan } from './local-scan.js'; +import type { SourceAdapter } from '../../context/ingest/types.js'; +import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; +import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js'; +import { filterSnapshotTables, resolveEnabledTables } from './enabled-tables.js'; +import { getLocalScanReport, getLocalScanStatus, runLocalScan } from './local-scan.js'; import type { KtxQueryResult, KtxReadOnlyQueryInput, KtxSchemaSnapshot, KtxSchemaTable } from './types.js'; function relationshipSqlResult( diff --git a/packages/cli/src/context/scan/local-scan.ts b/packages/cli/src/context/scan/local-scan.ts index 9b925fd2..35333f79 100644 --- a/packages/cli/src/context/scan/local-scan.ts +++ b/packages/cli/src/context/scan/local-scan.ts @@ -1,14 +1,13 @@ -import type { createKtxEmbeddingProvider, createKtxLlmProvider, KtxEmbeddingProvider } from '../../llm/index.js'; -import { - createDefaultLocalIngestAdapters, - getLocalStageOnlyIngestStatus, - type LocalIngestRunRecord, - runLocalStageOnlyIngest, - type SourceAdapter, -} from '../ingest/index.js'; -import { createLocalKtxLlmRuntimeFromConfig, KtxScanEmbeddingPortAdapter } from '../llm/index.js'; +import type { createKtxEmbeddingProvider } from '../../llm/embedding-provider.js'; +import type { createKtxLlmProvider } from '../../llm/model-provider.js'; +import type { KtxEmbeddingProvider } from '../../llm/types.js'; +import { createDefaultLocalIngestAdapters } from '../../context/ingest/local-adapters.js'; +import { getLocalStageOnlyIngestStatus, type LocalIngestRunRecord, runLocalStageOnlyIngest } from '../../context/ingest/local-stage-ingest.js'; +import type { SourceAdapter } from '../../context/ingest/types.js'; +import { createLocalKtxLlmRuntimeFromConfig } from '../../context/llm/local-config.js'; +import { KtxScanEmbeddingPortAdapter } from '../../context/llm/embedding-port.js'; import type { KtxProjectLlmConfig, KtxScanEnrichmentConfig, KtxScanRelationshipConfig } from '../project/config.js'; -import type { KtxLocalProject } from '../project/index.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import { ktxLocalStateDbPath } from '../project/local-state-db.js'; import { redactKtxScanReport } from './credentials.js'; import { filterSnapshotTables, resolveEnabledTables } from './enabled-tables.js'; @@ -96,6 +95,7 @@ export interface LocalScanRunResult { report: KtxScanReport; } +/** @internal */ export interface LocalScanStatusResponse { runId: string; status: LocalIngestRunRecord['status']; @@ -380,7 +380,6 @@ function createFilteredConnector(connector: KtxScanConnector, enabledTables: Set }; } -export { filterSnapshotTables, resolveEnabledTables } from './enabled-tables.js'; function withInternalLiveDatabaseAdapter(project: KtxLocalProject): KtxLocalProject { if (project.config.ingest.adapters.includes(LIVE_DATABASE_ADAPTER)) { @@ -588,6 +587,7 @@ export async function runLocalScan(options: RunLocalScanOptions): Promise { const status = await getLocalStageOnlyIngestStatus(project, runId); if (!status || status.adapter !== LIVE_DATABASE_ADAPTER) { @@ -605,6 +605,7 @@ export async function getLocalScanReport(project: KtxLocalProject, runId: string }; } +/** @internal */ export async function getLocalScanStatus( project: KtxLocalProject, runId: string, diff --git a/packages/cli/src/context/scan/local-structural-artifacts.test.ts b/packages/cli/src/context/scan/local-structural-artifacts.test.ts index e8089158..71019597 100644 --- a/packages/cli/src/context/scan/local-structural-artifacts.test.ts +++ b/packages/cli/src/context/scan/local-structural-artifacts.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { readLocalScanStructuralSnapshot } from './local-structural-artifacts.js'; describe('readLocalScanStructuralSnapshot', () => { diff --git a/packages/cli/src/context/scan/local-structural-artifacts.ts b/packages/cli/src/context/scan/local-structural-artifacts.ts index 5eeeea6d..2c968384 100644 --- a/packages/cli/src/context/scan/local-structural-artifacts.ts +++ b/packages/cli/src/context/scan/local-structural-artifacts.ts @@ -1,4 +1,4 @@ -import type { KtxLocalProject } from '../project/index.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import type { KtxConnectionDriver, KtxSchemaColumn, diff --git a/packages/cli/src/context/scan/relationship-artifacts.test.ts b/packages/cli/src/context/scan/relationship-artifacts.test.ts deleted file mode 100644 index f66de0c2..00000000 --- a/packages/cli/src/context/scan/relationship-artifacts.test.ts +++ /dev/null @@ -1,308 +0,0 @@ -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; -import { runLocalStageOnlyIngest, type SourceAdapter } from '../ingest/index.js'; -import { initKtxProject, loadKtxProject } from '../project/index.js'; -import { describe, expect, it } from 'vitest'; -import { readLocalScanRelationshipArtifacts } from './relationship-artifacts.js'; -import type { KtxRelationshipArtifact, KtxRelationshipDiagnosticsArtifact } from './relationship-diagnostics.js'; -import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js'; -import type { KtxScanReport } from './types.js'; - -async function writeProjectFile(projectDir: string, relativePath: string, content: string): Promise { - const absolutePath = join(projectDir, relativePath); - await mkdir(dirname(absolutePath), { recursive: true }); - await writeFile(absolutePath, content, 'utf-8'); -} - -async function writeWarehouseConfig(projectDir: string): Promise { - await writeFile( - join(projectDir, 'ktx.yaml'), - [ - 'connections:', - ' warehouse:', - ' driver: sqlite', - ' path: warehouse.db', - 'ingest:', - ' adapters:', - ' - live-database', - '', - ].join('\n'), - 'utf-8', - ); -} - -function liveDatabaseAdapter(): SourceAdapter { - return { - source: 'live-database', - skillNames: ['live_database_ingest'], - async fetch(_pullConfig, stagedDir) { - await mkdir(join(stagedDir, 'tables'), { recursive: true }); - await writeFile(join(stagedDir, 'connection.json'), '{"connectionId":"warehouse"}\n', 'utf-8'); - await writeFile(join(stagedDir, 'foreign-keys.json'), '{"foreignKeys":[]}\n', 'utf-8'); - await writeFile( - join(stagedDir, 'tables', 'orders.json'), - '{"name":"orders","db":"public","columns":[{"name":"id","type":"integer","nullable":false,"primaryKey":true}]}\n', - 'utf-8', - ); - }, - async detect(stagedDir) { - await writeFile(join(stagedDir, 'connection.json'), '{"connectionId":"warehouse"}\n', 'utf-8'); - return true; - }, - async chunk() { - return { - workUnits: [ - { - unitKey: 'live-database-public-orders', - rawFiles: ['tables/orders.json'], - dependencyPaths: ['connection.json', 'foreign-keys.json'], - peerFileIndex: [], - }, - ], - }; - }, - }; -} - -async function createLiveDatabaseRun(projectDir: string, runId: string) { - await initKtxProject({ projectDir }); - await writeWarehouseConfig(projectDir); - const project = await loadKtxProject({ projectDir }); - await runLocalStageOnlyIngest({ - project, - adapters: [liveDatabaseAdapter()], - adapter: 'live-database', - connectionId: 'warehouse', - jobId: runId, - now: () => new Date('2026-05-07T10:00:00.000Z'), - }); - return project; -} - -function scanReport(enrichmentArtifacts: string[], syncId = '2026-05-07-100000-scan-run-review'): KtxScanReport { - return { - connectionId: 'warehouse', - driver: 'sqlite', - syncId, - runId: 'scan-run-review', - trigger: 'cli', - mode: 'relationships', - dryRun: false, - artifactPaths: { - rawSourcesDir: `raw-sources/warehouse/live-database/${syncId}`, - reportPath: `raw-sources/warehouse/live-database/${syncId}/scan-report.json`, - manifestShards: [], - enrichmentArtifacts, - }, - diffSummary: { - tablesAdded: 0, - tablesModified: 0, - tablesDeleted: 0, - tablesUnchanged: 2, - columnsAdded: 0, - columnsModified: 0, - columnsDeleted: 0, - }, - manifestShardsWritten: 0, - structuralSyncStats: { - tablesCreated: 0, - tablesUpdated: 0, - tablesDeleted: 0, - columnsCreated: 0, - columnsUpdated: 0, - columnsDeleted: 0, - }, - enrichment: { - dataDictionary: 'skipped', - tableDescriptions: 'skipped', - columnDescriptions: 'skipped', - embeddings: 'skipped', - deterministicRelationships: 'completed', - llmRelationshipValidation: 'skipped', - statisticalValidation: 'skipped', - }, - capabilityGaps: [], - warnings: [], - relationships: { accepted: 0, review: 1, rejected: 1, skipped: 0 }, - enrichmentState: { - resumedStages: [], - completedStages: ['relationships'], - failedStages: [], - }, - createdAt: '2026-05-07T10:00:00.000Z', - }; -} - -const relationshipArtifact: KtxRelationshipArtifact = { - connectionId: 'warehouse', - accepted: [], - review: [ - { - id: 'orders:orders.customer_id->customers:customers.id', - status: 'review', - source: 'deterministic_name', - from: { - tableId: 'orders', - columnIds: ['orders.customer_id'], - table: { catalog: null, db: 'public', name: 'orders' }, - columns: ['customer_id'], - }, - to: { - tableId: 'customers', - columnIds: ['customers.id'], - table: { catalog: null, db: 'public', name: 'customers' }, - columns: ['id'], - }, - relationshipType: 'many_to_one', - confidence: 0.62, - pkScore: 0.91, - fkScore: 0.62, - score: 0.62, - evidence: { sources: ['table_suffix'] }, - validation: { status: 'unavailable' }, - graph: { reasons: ['validation_unavailable_review_only'] }, - reasons: ['validation_unavailable_review_only', 'fk_score_review'], - }, - ], - rejected: [ - { - id: 'orders:orders.note_id->notes:notes.id', - status: 'rejected', - source: 'deterministic_name', - from: { - tableId: 'orders', - columnIds: ['orders.note_id'], - table: { catalog: null, db: 'public', name: 'orders' }, - columns: ['note_id'], - }, - to: { - tableId: 'notes', - columnIds: ['notes.id'], - table: { catalog: null, db: 'public', name: 'notes' }, - columns: ['id'], - }, - relationshipType: 'many_to_one', - confidence: 0.2, - pkScore: 0.4, - fkScore: 0.2, - score: 0.2, - evidence: { sources: ['exact_column_match'] }, - validation: { status: 'failed' }, - graph: { reasons: ['low_source_coverage'] }, - reasons: ['low_source_coverage'], - }, - ], - skipped: [], -}; - -const diagnosticsArtifact: KtxRelationshipDiagnosticsArtifact = { - connectionId: 'warehouse', - generatedAt: '2026-05-07T10:00:00.000Z', - summary: { accepted: 0, review: 1, rejected: 1, skipped: 0 }, - noAcceptedReason: 'relationship candidates require review before manifest writes', - candidateCountsBySource: { deterministic_name: 2 }, - validation: { available: false, sqlAvailable: false, queryCount: 0 }, - thresholds: { acceptThreshold: 0.85, reviewThreshold: 0.55 }, - policy: { - validationRequiredForManifest: true, - maxCandidatesPerColumn: 25, - profileSampleRows: 10000, - validationConcurrency: 4, - }, - warnings: [], - profileWarnings: ['KTX scan connector cannot run read-only SQL relationship validation'], -}; - -const profileArtifact: KtxRelationshipProfileArtifact = { - connectionId: 'warehouse', - driver: 'sqlite', - sqlAvailable: false, - tables: [], - columns: {}, - queryCount: 0, - warnings: ['KTX scan connector cannot run read-only SQL relationship validation'], -}; - -describe('local scan relationship artifact reader', () => { - it('loads relationship, diagnostics, and profile artifacts for a scan run', async () => { - const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-artifacts-')); - try { - const project = await createLiveDatabaseRun(projectDir, 'scan-run-review'); - const syncId = '2026-05-07-100000-scan-run-review'; - const report = scanReport( - [ - `raw-sources/warehouse/live-database/${syncId}/enrichment/relationships.json`, - `raw-sources/warehouse/live-database/${syncId}/enrichment/relationship-profile.json`, - `raw-sources/warehouse/live-database/${syncId}/enrichment/relationship-diagnostics.json`, - ], - syncId, - ); - await writeProjectFile(projectDir, report.artifactPaths.reportPath ?? '', `${JSON.stringify(report, null, 2)}\n`); - await writeProjectFile( - projectDir, - `raw-sources/warehouse/live-database/${syncId}/enrichment/relationships.json`, - `${JSON.stringify(relationshipArtifact, null, 2)}\n`, - ); - await writeProjectFile( - projectDir, - `raw-sources/warehouse/live-database/${syncId}/enrichment/relationship-diagnostics.json`, - `${JSON.stringify(diagnosticsArtifact, null, 2)}\n`, - ); - await writeProjectFile( - projectDir, - `raw-sources/warehouse/live-database/${syncId}/enrichment/relationship-profile.json`, - `${JSON.stringify(profileArtifact, null, 2)}\n`, - ); - - const result = await readLocalScanRelationshipArtifacts(project, 'scan-run-review'); - - expect(result).toMatchObject({ - runId: 'scan-run-review', - connectionId: 'warehouse', - syncId, - paths: { - relationships: `raw-sources/warehouse/live-database/${syncId}/enrichment/relationships.json`, - diagnostics: `raw-sources/warehouse/live-database/${syncId}/enrichment/relationship-diagnostics.json`, - profile: `raw-sources/warehouse/live-database/${syncId}/enrichment/relationship-profile.json`, - }, - }); - expect(result?.relationships.review[0]).toMatchObject({ - id: 'orders:orders.customer_id->customers:customers.id', - status: 'review', - reasons: ['validation_unavailable_review_only', 'fk_score_review'], - }); - expect(result?.diagnostics?.noAcceptedReason).toBe('relationship candidates require review before manifest writes'); - expect(result?.profile?.sqlAvailable).toBe(false); - } finally { - await rm(projectDir, { recursive: true, force: true }); - } - }); - - it('returns null when the scan run has no report', async () => { - const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-artifacts-missing-run-')); - try { - await initKtxProject({ projectDir }); - const project = await loadKtxProject({ projectDir }); - - await expect(readLocalScanRelationshipArtifacts(project, 'missing-run')).resolves.toBeNull(); - } finally { - await rm(projectDir, { recursive: true, force: true }); - } - }); - - it('throws a focused error when a scan report does not reference relationships.json', async () => { - const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-artifacts-missing-artifact-')); - try { - const project = await createLiveDatabaseRun(projectDir, 'scan-run-review'); - const report = scanReport([]); - await writeProjectFile(projectDir, report.artifactPaths.reportPath ?? '', `${JSON.stringify(report, null, 2)}\n`); - - await expect(readLocalScanRelationshipArtifacts(project, 'scan-run-review')).rejects.toThrow( - 'Scan report "scan-run-review" does not reference relationships.json', - ); - } finally { - await rm(projectDir, { recursive: true, force: true }); - } - }); -}); diff --git a/packages/cli/src/context/scan/relationship-artifacts.ts b/packages/cli/src/context/scan/relationship-artifacts.ts deleted file mode 100644 index 99751b71..00000000 --- a/packages/cli/src/context/scan/relationship-artifacts.ts +++ /dev/null @@ -1,75 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import { getLocalScanReport } from './local-scan.js'; -import type { KtxRelationshipArtifact, KtxRelationshipDiagnosticsArtifact } from './relationship-diagnostics.js'; -import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js'; -import type { KtxScanReport } from './types.js'; - -export type KtxRelationshipArtifactStatus = 'accepted' | 'review' | 'rejected' | 'skipped' | 'all'; - -export interface ReadLocalScanRelationshipArtifactsResult { - runId: string; - connectionId: string; - syncId: string; - report: KtxScanReport; - relationships: KtxRelationshipArtifact; - diagnostics: KtxRelationshipDiagnosticsArtifact | null; - profile: KtxRelationshipProfileArtifact | null; - paths: { - relationships: string; - diagnostics: string | null; - profile: string | null; - }; -} - -function findArtifactPath(report: KtxScanReport, fileName: string): string | null { - return report.artifactPaths.enrichmentArtifacts.find((path) => path.endsWith(`/enrichment/${fileName}`)) ?? null; -} - -async function readJsonArtifact(project: KtxLocalProject, path: string): Promise { - const raw = await project.fileStore.readFile(path); - return JSON.parse(raw.content) as T; -} - -async function readOptionalJsonArtifact(project: KtxLocalProject, path: string | null): Promise { - if (!path) { - return null; - } - try { - return await readJsonArtifact(project, path); - } catch { - return null; - } -} - -export async function readLocalScanRelationshipArtifacts( - project: KtxLocalProject, - runId: string, -): Promise { - const report = await getLocalScanReport(project, runId); - if (!report) { - return null; - } - - const relationshipsPath = findArtifactPath(report, 'relationships.json'); - if (!relationshipsPath) { - throw new Error(`Scan report "${runId}" does not reference relationships.json`); - } - - const diagnosticsPath = findArtifactPath(report, 'relationship-diagnostics.json'); - const profilePath = findArtifactPath(report, 'relationship-profile.json'); - - return { - runId, - connectionId: report.connectionId, - syncId: report.syncId, - report, - relationships: await readJsonArtifact(project, relationshipsPath), - diagnostics: await readOptionalJsonArtifact(project, diagnosticsPath), - profile: await readOptionalJsonArtifact(project, profilePath), - paths: { - relationships: relationshipsPath, - diagnostics: diagnosticsPath, - profile: profilePath, - }, - }; -} diff --git a/packages/cli/src/context/scan/relationship-benchmarks.ts b/packages/cli/src/context/scan/relationship-benchmarks.ts index e3c65056..f4367b5a 100644 --- a/packages/cli/src/context/scan/relationship-benchmarks.ts +++ b/packages/cli/src/context/scan/relationship-benchmarks.ts @@ -6,7 +6,7 @@ import { gunzipSync } from 'node:zlib'; import Database from 'better-sqlite3'; import YAML from 'yaml'; import { z } from 'zod'; -import type { KtxLlmRuntimePort } from '../llm/index.js'; +import type { KtxLlmRuntimePort } from '../llm/runtime-port.js'; import type { KtxEnrichedRelationship, KtxEnrichedSchema, KtxRelationshipType } from './enrichment-types.js'; import { snapshotToKtxEnrichedSchema } from './local-enrichment.js'; import type { KtxRelationshipDiscoveryCandidate } from './relationship-candidates.js'; diff --git a/packages/cli/src/context/scan/relationship-budget.ts b/packages/cli/src/context/scan/relationship-budget.ts index 13209301..78d1a348 100644 --- a/packages/cli/src/context/scan/relationship-budget.ts +++ b/packages/cli/src/context/scan/relationship-budget.ts @@ -1,6 +1,6 @@ export type KtxRelationshipValidationBudget = number | 'all' | undefined; -export interface KtxRelationshipBudgetedCandidate { +interface KtxRelationshipBudgetedCandidate { candidate: TCandidate; originalIndex: number; score: number; @@ -19,6 +19,7 @@ export interface ApplyKtxRelationshipValidationBudgetInput { score: (candidate: TCandidate) => number; } +/** @internal */ export function defaultKtxRelationshipValidationBudget(tableCount: number): number { const safeTableCount = Number.isFinite(tableCount) ? Math.max(0, Math.floor(tableCount)) : 0; return Math.min(2 * safeTableCount, 1000); diff --git a/packages/cli/src/context/scan/relationship-candidates.ts b/packages/cli/src/context/scan/relationship-candidates.ts index 9b78d3b7..21b2c040 100644 --- a/packages/cli/src/context/scan/relationship-candidates.ts +++ b/packages/cli/src/context/scan/relationship-candidates.ts @@ -12,7 +12,6 @@ import { pluralizeKtxRelationshipToken, singularizeKtxRelationshipToken, } from './relationship-name-similarity.js'; -; export { normalizeKtxRelationshipName } from './relationship-name-similarity.js'; import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js'; import { @@ -21,7 +20,7 @@ import { type KtxRelationshipSignalVector, } from './relationship-scoring.js'; -export type KtxRelationshipDiscoveryCandidateSource = +type KtxRelationshipDiscoveryCandidateSource = | 'exact_column_match' | 'normalized_table_match' | 'parent_table_name_match' @@ -32,9 +31,9 @@ export type KtxRelationshipDiscoveryCandidateSource = | 'embedding_similarity' | 'llm_proposal'; -export type KtxRelationshipDiscoveryCandidateStatus = 'review'; +type KtxRelationshipDiscoveryCandidateStatus = 'review'; -export interface KtxRelationshipDiscoveryCandidateEvidence { +interface KtxRelationshipDiscoveryCandidateEvidence { sourceColumnBase: string; targetTableBase: string; targetColumnBase: string; @@ -69,6 +68,7 @@ export interface KtxRelationshipDiscoveryCandidateOptions { profiles?: KtxRelationshipProfileArtifact; } +/** @internal */ export interface KtxRelationshipInferredTargetPk { table: string; columns: string[]; @@ -758,6 +758,7 @@ export function mergeKtxRelationshipDiscoveryCandidates( return Array.from(byId.values()).sort((left, right) => candidateSortKey(left).localeCompare(candidateSortKey(right))); } +/** @internal */ export function inferKtxRelationshipTargetPks( candidates: readonly KtxRelationshipDiscoveryCandidate[], ): KtxRelationshipInferredTargetPk[] { diff --git a/packages/cli/src/context/scan/relationship-composite-candidates.ts b/packages/cli/src/context/scan/relationship-composite-candidates.ts index f93d5153..d8ee650d 100644 --- a/packages/cli/src/context/scan/relationship-composite-candidates.ts +++ b/packages/cli/src/context/scan/relationship-composite-candidates.ts @@ -7,9 +7,9 @@ import { } from './relationship-profiling.js'; import type { KtxConnectionDriver, KtxQueryResult, KtxScanContext, KtxTableRef } from './types.js'; -export type KtxCompositeRelationshipStatus = 'accepted' | 'review' | 'rejected'; +type KtxCompositeRelationshipStatus = 'accepted' | 'review' | 'rejected'; -export interface KtxCompositeRelationshipTupleEndpoint { +interface KtxCompositeRelationshipTupleEndpoint { tableId: string; columnIds: string[]; table: KtxTableRef; @@ -33,7 +33,7 @@ export interface KtxCompositePrimaryKeyCandidate { }; } -export interface KtxCompositeRelationshipValidationEvidence { +interface KtxCompositeRelationshipValidationEvidence { targetUniqueness: number; sourceCoverage: number; violationCount: number; diff --git a/packages/cli/src/context/scan/relationship-diagnostics.ts b/packages/cli/src/context/scan/relationship-diagnostics.ts index 69d8886a..e50e892e 100644 --- a/packages/cli/src/context/scan/relationship-diagnostics.ts +++ b/packages/cli/src/context/scan/relationship-diagnostics.ts @@ -12,7 +12,7 @@ import type { KtxCompositeRelationshipCandidate } from './relationship-composite import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js'; import type { KtxConnectionDriver, KtxScanWarning } from './types.js'; -export interface KtxRelationshipArtifactEndpoint { +interface KtxRelationshipArtifactEndpoint { tableId: string; columnIds: string[]; table: { @@ -23,7 +23,7 @@ export interface KtxRelationshipArtifactEndpoint { columns: string[]; } -export interface KtxRelationshipArtifactEdge { +interface KtxRelationshipArtifactEdge { id: string; status: KtxResolvedRelationshipStatus; source: string; @@ -48,20 +48,20 @@ export interface KtxRelationshipArtifact { skipped: KtxRelationshipUpdate['skipped']; } -export interface KtxRelationshipDiagnosticsSummary { +interface KtxRelationshipDiagnosticsSummary { accepted: number; review: number; rejected: number; skipped: number; } -export interface KtxRelationshipDiagnosticsValidation { +interface KtxRelationshipDiagnosticsValidation { available: boolean; sqlAvailable: boolean; queryCount: number; } -export interface KtxRelationshipDiagnosticsThresholds { +interface KtxRelationshipDiagnosticsThresholds { acceptThreshold: number; reviewThreshold: number; } diff --git a/packages/cli/src/context/scan/relationship-discovery.test.ts b/packages/cli/src/context/scan/relationship-discovery.test.ts index 36025ffd..400fae62 100644 --- a/packages/cli/src/context/scan/relationship-discovery.test.ts +++ b/packages/cli/src/context/scan/relationship-discovery.test.ts @@ -1,6 +1,6 @@ import Database from 'better-sqlite3'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import type { KtxLlmRuntimePort } from '../llm/index.js'; +import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; import { buildDefaultKtxProjectConfig } from '../project/config.js'; import { snapshotToKtxEnrichedSchema } from './local-enrichment.js'; import { diff --git a/packages/cli/src/context/scan/relationship-discovery.ts b/packages/cli/src/context/scan/relationship-discovery.ts index ce6dfba9..c1197866 100644 --- a/packages/cli/src/context/scan/relationship-discovery.ts +++ b/packages/cli/src/context/scan/relationship-discovery.ts @@ -1,4 +1,4 @@ -import type { KtxLlmRuntimePort } from '../llm/index.js'; +import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; import type { KtxScanRelationshipConfig } from '../project/config.js'; import type { KtxEnrichedRelationship, KtxEnrichedSchema, KtxRelationshipUpdate } from './enrichment-types.js'; import { diff --git a/packages/cli/src/context/scan/relationship-feedback-calibration.test.ts b/packages/cli/src/context/scan/relationship-feedback-calibration.test.ts deleted file mode 100644 index 43090ad4..00000000 --- a/packages/cli/src/context/scan/relationship-feedback-calibration.test.ts +++ /dev/null @@ -1,211 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import { describe, expect, it, vi } from 'vitest'; -import { - buildKtxRelationshipFeedbackCalibrationReport, - calibrateLocalRelationshipFeedbackLabels, - formatKtxRelationshipFeedbackCalibrationMarkdown, -} from './relationship-feedback-calibration.js'; -import type { - ExportLocalRelationshipFeedbackLabelsResult, - KtxRelationshipFeedbackLabel, -} from './relationship-feedback-export.js'; - -function label( - input: Partial & - Pick, -): KtxRelationshipFeedbackLabel { - return { - schemaVersion: 1, - previousStatus: 'review', - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decidedAt: '2026-05-07T12:00:00.000Z', - reviewer: 'Andrey', - note: null, - relationshipType: 'many_to_one', - source: 'deterministic_name', - confidence: input.score ?? 0, - pkScore: input.pkScore ?? null, - fkScore: input.fkScore ?? input.score, - fromTable: 'public.orders', - fromColumns: ['customer_id'], - toTable: 'public.customers', - toColumns: ['id'], - reasons: [], - artifactPath: 'raw-sources/warehouse/live-database/sync-a/enrichment/relationship-review-decisions.json', - ...input, - }; -} - -function feedback(labels: KtxRelationshipFeedbackLabel[]): ExportLocalRelationshipFeedbackLabelsResult { - return { - generatedAt: '2026-05-07T13:00:00.000Z', - filters: { connectionId: null, decision: 'all' }, - summary: { - total: labels.length, - accepted: labels.filter((item) => item.decision === 'accepted').length, - rejected: labels.filter((item) => item.decision === 'rejected').length, - connections: new Set(labels.map((item) => item.connectionId)).size, - runs: new Set(labels.map((item) => `${item.connectionId}:${item.runId}`)).size, - }, - labels, - warnings: [], - }; -} - -describe('relationship feedback calibration', () => { - it('builds score buckets and threshold-band summary from feedback labels', () => { - const report = buildKtxRelationshipFeedbackCalibrationReport( - feedback([ - label({ - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted', - score: 0.91, - pkScore: 0.97, - fkScore: 0.91, - }), - label({ - candidateId: 'orders:orders.account_id->accounts:accounts.id', - decision: 'accepted', - score: 0.61, - pkScore: 0.88, - fkScore: 0.61, - }), - label({ - candidateId: 'orders:orders.note_id->notes:notes.id', - decision: 'rejected', - score: 0.21, - pkScore: 0.4, - fkScore: 0.21, - }), - label({ - candidateId: 'orders:orders.region_id->regions:regions.id', - decision: 'rejected', - score: 0.88, - pkScore: 0.9, - fkScore: 0.88, - }), - ]), - { - acceptThreshold: 0.85, - reviewThreshold: 0.55, - }, - ); - - expect(report.thresholds).toEqual({ accept: 0.85, review: 0.55 }); - expect(report.summary).toEqual({ - total: 4, - scored: 4, - unscored: 0, - acceptedLabels: 2, - rejectedLabels: 2, - predictedAccepted: 2, - predictedReview: 1, - predictedRejected: 1, - acceptedBandPrecision: 0.5, - rejectedBandPrecision: 1, - reviewBandAcceptedRate: 1, - meanAcceptedScore: 0.76, - meanRejectedScore: 0.545, - }); - expect(report.buckets.map((bucket) => [bucket.label, bucket.total, bucket.accepted, bucket.rejected, bucket.acceptanceRate])).toEqual([ - ['0.00-0.24', 1, 0, 1, 0], - ['0.25-0.49', 0, 0, 0, null], - ['0.50-0.74', 1, 1, 0, 1], - ['0.75-1.00', 2, 1, 1, 0.5], - ]); - expect(report.labels.map((item) => [item.candidateId, item.predictedStatus, item.bucket])).toEqual([ - ['orders:orders.account_id->accounts:accounts.id', 'review', '0.50-0.74'], - ['orders:orders.customer_id->customers:customers.id', 'accepted', '0.75-1.00'], - ['orders:orders.note_id->notes:notes.id', 'rejected', '0.00-0.24'], - ['orders:orders.region_id->regions:regions.id', 'accepted', '0.75-1.00'], - ]); - }); - - it('keeps unscored labels visible without treating them as threshold predictions', () => { - const report = buildKtxRelationshipFeedbackCalibrationReport( - feedback([ - label({ - candidateId: 'orders:orders.note_id->notes:notes.id', - decision: 'rejected', - score: null, - confidence: 0.2, - fkScore: null, - }), - ]), - { - acceptThreshold: 0.85, - reviewThreshold: 0.55, - }, - ); - - expect(report.summary).toMatchObject({ - total: 1, - scored: 0, - unscored: 1, - predictedAccepted: 0, - predictedReview: 0, - predictedRejected: 0, - acceptedBandPrecision: null, - rejectedBandPrecision: null, - reviewBandAcceptedRate: null, - meanAcceptedScore: null, - meanRejectedScore: null, - }); - expect(report.labels[0]).toMatchObject({ - candidateId: 'orders:orders.note_id->notes:notes.id', - predictedStatus: 'unscored', - bucket: 'unscored', - }); - }); - - it('formats a stable markdown summary for human CLI output', () => { - const report = buildKtxRelationshipFeedbackCalibrationReport( - feedback([ - label({ candidateId: 'orders:orders.customer_id->customers:customers.id', decision: 'accepted', score: 0.91 }), - label({ candidateId: 'orders:orders.note_id->notes:notes.id', decision: 'rejected', score: 0.21 }), - ]), - { - acceptThreshold: 0.85, - reviewThreshold: 0.55, - }, - ); - - expect(formatKtxRelationshipFeedbackCalibrationMarkdown(report)).toContain( - 'KTX relationship feedback calibration', - ); - expect(formatKtxRelationshipFeedbackCalibrationMarkdown(report)).toContain('Total labels: 2'); - expect(formatKtxRelationshipFeedbackCalibrationMarkdown(report)).toContain('Accepted-band precision: 1.000'); - expect(formatKtxRelationshipFeedbackCalibrationMarkdown(report)).toContain( - '0.75-1.00: total=1 accepted=1 rejected=0 acceptanceRate=1.000', - ); - }); - - it('wraps the feedback exporter and preserves exporter warnings', async () => { - const project = { projectDir: '/tmp/ktx-project' } as KtxLocalProject; - const exportLocalRelationshipFeedbackLabels = vi.fn(async () => ({ - ...feedback([ - label({ candidateId: 'orders:orders.customer_id->customers:customers.id', decision: 'accepted', score: 0.91 }), - ]), - warnings: [{ path: 'raw-sources/broken/live-database/sync/enrichment/relationship-review-decisions.json', message: 'Unexpected token' }], - })); - - const report = await calibrateLocalRelationshipFeedbackLabels(project, { - connectionId: 'warehouse', - decision: 'all', - acceptThreshold: 0.9, - reviewThreshold: 0.5, - exportLocalRelationshipFeedbackLabels, - }); - - expect(exportLocalRelationshipFeedbackLabels).toHaveBeenCalledWith(project, { - connectionId: 'warehouse', - decision: 'all', - }); - expect(report.thresholds).toEqual({ accept: 0.9, review: 0.5 }); - expect(report.warnings).toEqual([ - { path: 'raw-sources/broken/live-database/sync/enrichment/relationship-review-decisions.json', message: 'Unexpected token' }, - ]); - }); -}); diff --git a/packages/cli/src/context/scan/relationship-feedback-calibration.ts b/packages/cli/src/context/scan/relationship-feedback-calibration.ts deleted file mode 100644 index 5d471907..00000000 --- a/packages/cli/src/context/scan/relationship-feedback-calibration.ts +++ /dev/null @@ -1,300 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import { - exportLocalRelationshipFeedbackLabels, - type ExportLocalRelationshipFeedbackLabelsInput, - type ExportLocalRelationshipFeedbackLabelsResult, - type KtxRelationshipFeedbackExportWarning, - type KtxRelationshipFeedbackLabel, -} from './relationship-feedback-export.js'; -import type { KtxResolvedRelationshipStatus } from './relationship-graph-resolver.js'; -import type { KtxRelationshipReviewDecisionValue } from './relationship-review-decisions.js'; - -const DEFAULT_ACCEPT_THRESHOLD = 0.85; -const DEFAULT_REVIEW_THRESHOLD = 0.55; - -type CalibrationPredictedStatus = KtxResolvedRelationshipStatus | 'unscored'; - -interface Thresholds { - accept: number; - review: number; -} - -export interface BuildKtxRelationshipFeedbackCalibrationReportInput { - acceptThreshold?: number; - reviewThreshold?: number; -} - -export interface CalibrateLocalRelationshipFeedbackLabelsInput - extends ExportLocalRelationshipFeedbackLabelsInput, - BuildKtxRelationshipFeedbackCalibrationReportInput { - exportLocalRelationshipFeedbackLabels?: typeof exportLocalRelationshipFeedbackLabels; -} - -export interface KtxRelationshipFeedbackCalibrationBucket { - label: string; - minInclusive: number; - maxInclusive: number; - total: number; - accepted: number; - rejected: number; - acceptanceRate: number | null; -} - -export interface KtxRelationshipFeedbackCalibrationLabel { - candidateId: string; - decision: KtxRelationshipReviewDecisionValue; - previousStatus: KtxRelationshipFeedbackLabel['previousStatus']; - predictedStatus: CalibrationPredictedStatus; - bucket: string; - score: number | null; - pkScore: number | null; - fkScore: number | null; - connectionId: string; - runId: string; - fromTable: string; - fromColumns: string[]; - toTable: string; - toColumns: string[]; - source: string; - reasons: string[]; -} - -export interface KtxRelationshipFeedbackCalibrationReport { - generatedAt: string; - filters: ExportLocalRelationshipFeedbackLabelsResult['filters']; - thresholds: Thresholds; - summary: { - total: number; - scored: number; - unscored: number; - acceptedLabels: number; - rejectedLabels: number; - predictedAccepted: number; - predictedReview: number; - predictedRejected: number; - acceptedBandPrecision: number | null; - rejectedBandPrecision: number | null; - reviewBandAcceptedRate: number | null; - meanAcceptedScore: number | null; - meanRejectedScore: number | null; - }; - buckets: KtxRelationshipFeedbackCalibrationBucket[]; - labels: KtxRelationshipFeedbackCalibrationLabel[]; - warnings: KtxRelationshipFeedbackExportWarning[]; -} - -const BUCKETS = [ - { label: '0.00-0.24', minInclusive: 0, maxInclusive: 0.249999 }, - { label: '0.25-0.49', minInclusive: 0.25, maxInclusive: 0.499999 }, - { label: '0.50-0.74', minInclusive: 0.5, maxInclusive: 0.749999 }, - { label: '0.75-1.00', minInclusive: 0.75, maxInclusive: 1 }, -] as const; - -function thresholds(input: BuildKtxRelationshipFeedbackCalibrationReportInput): Thresholds { - return { - accept: input.acceptThreshold ?? DEFAULT_ACCEPT_THRESHOLD, - review: input.reviewThreshold ?? DEFAULT_REVIEW_THRESHOLD, - }; -} - -function roundMetric(value: number): number { - return Math.round(value * 1000) / 1000; -} - -function ratio(numerator: number, denominator: number): number | null { - return denominator === 0 ? null : roundMetric(numerator / denominator); -} - -function mean(values: readonly number[]): number | null { - if (values.length === 0) { - return null; - } - return roundMetric(values.reduce((sum, value) => sum + value, 0) / values.length); -} - -function scoreBucket(score: number | null): string { - if (score === null) { - return 'unscored'; - } - return BUCKETS.find((bucket) => score >= bucket.minInclusive && score <= bucket.maxInclusive)?.label ?? 'unscored'; -} - -function predictedStatus(score: number | null, currentThresholds: Thresholds): CalibrationPredictedStatus { - if (score === null) { - return 'unscored'; - } - if (score >= currentThresholds.accept) { - return 'accepted'; - } - if (score >= currentThresholds.review) { - return 'review'; - } - return 'rejected'; -} - -function calibrationLabel( - label: KtxRelationshipFeedbackLabel, - currentThresholds: Thresholds, -): KtxRelationshipFeedbackCalibrationLabel { - return { - candidateId: label.candidateId, - decision: label.decision, - previousStatus: label.previousStatus, - predictedStatus: predictedStatus(label.score, currentThresholds), - bucket: scoreBucket(label.score), - score: label.score, - pkScore: label.pkScore, - fkScore: label.fkScore, - connectionId: label.connectionId, - runId: label.runId, - fromTable: label.fromTable, - fromColumns: [...label.fromColumns], - toTable: label.toTable, - toColumns: [...label.toColumns], - source: label.source, - reasons: [...label.reasons], - }; -} - -function summarize( - labels: readonly KtxRelationshipFeedbackCalibrationLabel[], -): KtxRelationshipFeedbackCalibrationReport['summary'] { - const scored = labels.filter((label) => label.score !== null); - const predictedAccepted = scored.filter((label) => label.predictedStatus === 'accepted'); - const predictedReview = scored.filter((label) => label.predictedStatus === 'review'); - const predictedRejected = scored.filter((label) => label.predictedStatus === 'rejected'); - const acceptedLabels = labels.filter((label) => label.decision === 'accepted'); - const rejectedLabels = labels.filter((label) => label.decision === 'rejected'); - - return { - total: labels.length, - scored: scored.length, - unscored: labels.length - scored.length, - acceptedLabels: acceptedLabels.length, - rejectedLabels: rejectedLabels.length, - predictedAccepted: predictedAccepted.length, - predictedReview: predictedReview.length, - predictedRejected: predictedRejected.length, - acceptedBandPrecision: ratio( - predictedAccepted.filter((label) => label.decision === 'accepted').length, - predictedAccepted.length, - ), - rejectedBandPrecision: ratio( - predictedRejected.filter((label) => label.decision === 'rejected').length, - predictedRejected.length, - ), - reviewBandAcceptedRate: ratio( - predictedReview.filter((label) => label.decision === 'accepted').length, - predictedReview.length, - ), - meanAcceptedScore: mean(acceptedLabels.map((label) => label.score).filter((score): score is number => score !== null)), - meanRejectedScore: mean(rejectedLabels.map((label) => label.score).filter((score): score is number => score !== null)), - }; -} - -function buildBuckets( - labels: readonly KtxRelationshipFeedbackCalibrationLabel[], -): KtxRelationshipFeedbackCalibrationBucket[] { - return BUCKETS.map((bucket) => { - const bucketLabels = labels.filter((label) => label.bucket === bucket.label); - const accepted = bucketLabels.filter((label) => label.decision === 'accepted').length; - const rejected = bucketLabels.filter((label) => label.decision === 'rejected').length; - return { - label: bucket.label, - minInclusive: bucket.minInclusive, - maxInclusive: - bucket.maxInclusive === 0.249999 - ? 0.24 - : bucket.maxInclusive === 0.499999 - ? 0.49 - : bucket.maxInclusive === 0.749999 - ? 0.74 - : 1, - total: bucketLabels.length, - accepted, - rejected, - acceptanceRate: ratio(accepted, bucketLabels.length), - }; - }); -} - -export function buildKtxRelationshipFeedbackCalibrationReport( - feedback: ExportLocalRelationshipFeedbackLabelsResult, - input: BuildKtxRelationshipFeedbackCalibrationReportInput = {}, -): KtxRelationshipFeedbackCalibrationReport { - const currentThresholds = thresholds(input); - const labels = feedback.labels - .map((label) => calibrationLabel(label, currentThresholds)) - .sort( - (left, right) => - left.connectionId.localeCompare(right.connectionId) || - left.runId.localeCompare(right.runId) || - left.candidateId.localeCompare(right.candidateId), - ); - - return { - generatedAt: feedback.generatedAt, - filters: feedback.filters, - thresholds: currentThresholds, - summary: summarize(labels), - buckets: buildBuckets(labels), - labels, - warnings: [...feedback.warnings], - }; -} - -export async function calibrateLocalRelationshipFeedbackLabels( - project: KtxLocalProject, - input: CalibrateLocalRelationshipFeedbackLabelsInput = {}, -): Promise { - const exporter = input.exportLocalRelationshipFeedbackLabels ?? exportLocalRelationshipFeedbackLabels; - const feedback = await exporter(project, { - connectionId: input.connectionId, - decision: input.decision, - }); - return buildKtxRelationshipFeedbackCalibrationReport(feedback, input); -} - -function formatMetric(value: number | null): string { - return value === null ? 'n/a' : value.toFixed(3); -} - -export function formatKtxRelationshipFeedbackCalibrationMarkdown( - report: KtxRelationshipFeedbackCalibrationReport, -): string { - const lines = [ - 'KTX relationship feedback calibration', - `Generated: ${report.generatedAt}`, - `Filter connection: ${report.filters.connectionId ?? 'all'}`, - `Filter decision: ${report.filters.decision}`, - `Thresholds: accept=${report.thresholds.accept.toFixed(2)} review=${report.thresholds.review.toFixed(2)}`, - `Total labels: ${report.summary.total}`, - `Scored labels: ${report.summary.scored}`, - `Unscored labels: ${report.summary.unscored}`, - `Accepted labels: ${report.summary.acceptedLabels}`, - `Rejected labels: ${report.summary.rejectedLabels}`, - `Predicted accepted: ${report.summary.predictedAccepted}`, - `Predicted review: ${report.summary.predictedReview}`, - `Predicted rejected: ${report.summary.predictedRejected}`, - `Accepted-band precision: ${formatMetric(report.summary.acceptedBandPrecision)}`, - `Rejected-band precision: ${formatMetric(report.summary.rejectedBandPrecision)}`, - `Review-band accepted rate: ${formatMetric(report.summary.reviewBandAcceptedRate)}`, - `Mean accepted score: ${formatMetric(report.summary.meanAcceptedScore)}`, - `Mean rejected score: ${formatMetric(report.summary.meanRejectedScore)}`, - '', - 'Score buckets', - ...report.buckets.map( - (bucket) => - ` - ${bucket.label}: total=${bucket.total} accepted=${bucket.accepted} rejected=${bucket.rejected} acceptanceRate=${formatMetric(bucket.acceptanceRate)}`, - ), - ]; - - if (report.warnings.length > 0) { - lines.push('', 'Warnings'); - for (const warning of report.warnings.slice(0, 5)) { - lines.push(` - ${warning.path}: ${warning.message}`); - } - } - - return `${lines.join('\n')}\n`; -} diff --git a/packages/cli/src/context/scan/relationship-feedback-export.test.ts b/packages/cli/src/context/scan/relationship-feedback-export.test.ts deleted file mode 100644 index 512c29ad..00000000 --- a/packages/cli/src/context/scan/relationship-feedback-export.test.ts +++ /dev/null @@ -1,270 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import { describe, expect, it, vi } from 'vitest'; -import { - exportLocalRelationshipFeedbackLabels, - formatKtxRelationshipFeedbackLabelsJsonl, -} from './relationship-feedback-export.js'; -import type { KtxRelationshipReviewDecisionArtifact } from './relationship-review-decisions.js'; - -function projectWithFiles(files: Record): KtxLocalProject { - const contentByPath = new Map( - Object.entries(files).map(([path, value]) => [ - path, - typeof value === 'string' ? value : `${JSON.stringify(value, null, 2)}\n`, - ]), - ); - return { - projectDir: '/tmp/ktx-project', - fileStore: { - async listFiles(path: string) { - return { - files: [...contentByPath.keys()].filter((file) => file.startsWith(`${path}/`)).sort(), - }; - }, - async readFile(path: string) { - const content = contentByPath.get(path); - if (!content) { - throw new Error(`missing file ${path}`); - } - return { content }; - }, - writeFile: vi.fn(), - deleteFile: vi.fn(), - getFileHistory: vi.fn(), - forWorktree: vi.fn(), - }, - } as unknown as KtxLocalProject; -} - -function decisionsArtifact(input: { - connectionId: string; - runId: string; - syncId: string; - decisions: KtxRelationshipReviewDecisionArtifact['decisions']; -}): KtxRelationshipReviewDecisionArtifact { - return { - connectionId: input.connectionId, - runId: input.runId, - syncId: input.syncId, - generatedAt: '2026-05-07T12:00:00.000Z', - decisions: input.decisions, - }; -} - -const acceptedOrderCustomer = { - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted' as const, - previousStatus: 'review' as const, - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decidedAt: '2026-05-07T12:00:00.000Z', - reviewer: 'Andrey', - note: 'Confirmed in warehouse docs', - from: { - tableId: 'orders', - columnIds: ['orders.customer_id'], - table: { catalog: null, db: 'public', name: 'orders' }, - columns: ['customer_id'], - }, - to: { - tableId: 'customers', - columnIds: ['customers.id'], - table: { catalog: null, db: 'public', name: 'customers' }, - columns: ['id'], - }, - relationshipType: 'many_to_one' as const, - source: 'deterministic_name', - score: 0.62, - confidence: 0.62, - pkScore: 0.91, - fkScore: 0.62, - reasons: ['fk_score_review'], -}; - -const rejectedOrderNote = { - candidateId: 'orders:orders.note_id->notes:notes.id', - decision: 'rejected' as const, - previousStatus: 'rejected' as const, - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decidedAt: '2026-05-07T12:05:00.000Z', - reviewer: 'Andrey', - note: null, - from: { - tableId: 'orders', - columnIds: ['orders.note_id'], - table: { catalog: null, db: 'public', name: 'orders' }, - columns: ['note_id'], - }, - to: { - tableId: 'notes', - columnIds: ['notes.id'], - table: { catalog: null, db: 'public', name: 'notes' }, - columns: ['id'], - }, - relationshipType: 'many_to_one' as const, - source: 'deterministic_name', - score: 0.2, - confidence: 0.2, - pkScore: 0.4, - fkScore: 0.2, - reasons: ['low_source_coverage'], -}; - -const acceptedInvoiceAccount = { - candidateId: 'invoices:invoices.account_id->accounts:accounts.id', - decision: 'accepted' as const, - previousStatus: 'accepted' as const, - connectionId: 'billing', - runId: 'scan-run-b', - syncId: 'sync-b', - decidedAt: '2026-05-07T12:10:00.000Z', - reviewer: 'ktx', - note: null, - from: { - tableId: 'invoices', - columnIds: ['invoices.account_id'], - table: { catalog: null, db: 'billing', name: 'invoices' }, - columns: ['account_id'], - }, - to: { - tableId: 'accounts', - columnIds: ['accounts.id'], - table: { catalog: null, db: 'billing', name: 'accounts' }, - columns: ['id'], - }, - relationshipType: 'many_to_one' as const, - source: 'formal_metadata', - score: 1, - confidence: 1, - pkScore: 1, - fkScore: 1, - reasons: ['formal_metadata_relationship'], -}; - -describe('relationship feedback export', () => { - it('exports stable labels from all relationship review decision artifacts', async () => { - const project = projectWithFiles({ - 'raw-sources/warehouse/live-database/sync-a/enrichment/relationship-review-decisions.json': decisionsArtifact({ - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decisions: [rejectedOrderNote, acceptedOrderCustomer], - }), - 'raw-sources/billing/live-database/sync-b/enrichment/relationship-review-decisions.json': decisionsArtifact({ - connectionId: 'billing', - runId: 'scan-run-b', - syncId: 'sync-b', - decisions: [acceptedInvoiceAccount], - }), - 'raw-sources/warehouse/live-database/sync-a/enrichment/relationships.json': { accepted: [], review: [], rejected: [] }, - }); - - const result = await exportLocalRelationshipFeedbackLabels(project, { - now: () => new Date('2026-05-07T13:00:00.000Z'), - }); - - expect(result.summary).toEqual({ - total: 3, - accepted: 2, - rejected: 1, - connections: 2, - runs: 2, - }); - expect(result.labels.map((label) => label.candidateId)).toEqual([ - 'invoices:invoices.account_id->accounts:accounts.id', - 'orders:orders.customer_id->customers:customers.id', - 'orders:orders.note_id->notes:notes.id', - ]); - expect(result.labels[0]).toMatchObject({ - schemaVersion: 1, - decision: 'accepted', - connectionId: 'billing', - source: 'formal_metadata', - fromTable: 'billing.invoices', - fromColumns: ['account_id'], - toTable: 'billing.accounts', - toColumns: ['id'], - artifactPath: 'raw-sources/billing/live-database/sync-b/enrichment/relationship-review-decisions.json', - }); - expect(result.warnings).toEqual([]); - }); - - it('filters labels by connection and decision', async () => { - const project = projectWithFiles({ - 'raw-sources/warehouse/live-database/sync-a/enrichment/relationship-review-decisions.json': decisionsArtifact({ - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decisions: [rejectedOrderNote, acceptedOrderCustomer], - }), - 'raw-sources/billing/live-database/sync-b/enrichment/relationship-review-decisions.json': decisionsArtifact({ - connectionId: 'billing', - runId: 'scan-run-b', - syncId: 'sync-b', - decisions: [acceptedInvoiceAccount], - }), - }); - - const result = await exportLocalRelationshipFeedbackLabels(project, { - connectionId: 'warehouse', - decision: 'rejected', - now: () => new Date('2026-05-07T13:00:00.000Z'), - }); - - expect(result.summary).toMatchObject({ total: 1, accepted: 0, rejected: 1 }); - expect(result.labels).toHaveLength(1); - expect(result.labels[0]?.candidateId).toBe('orders:orders.note_id->notes:notes.id'); - }); - - it('formats JSONL with one stable label object per line', async () => { - const project = projectWithFiles({ - 'raw-sources/warehouse/live-database/sync-a/enrichment/relationship-review-decisions.json': decisionsArtifact({ - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decisions: [acceptedOrderCustomer], - }), - }); - const result = await exportLocalRelationshipFeedbackLabels(project, { - now: () => new Date('2026-05-07T13:00:00.000Z'), - }); - - const lines = formatKtxRelationshipFeedbackLabelsJsonl(result).trim().split('\n').map((line) => JSON.parse(line)); - - expect(lines).toHaveLength(1); - expect(lines[0]).toMatchObject({ - schemaVersion: 1, - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted', - relationshipType: 'many_to_one', - }); - }); - - it('records parse warnings and continues exporting readable decision artifacts', async () => { - const project = projectWithFiles({ - 'raw-sources/warehouse/live-database/sync-a/enrichment/relationship-review-decisions.json': decisionsArtifact({ - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decisions: [acceptedOrderCustomer], - }), - 'raw-sources/broken/live-database/sync-b/enrichment/relationship-review-decisions.json': '{not-json', - }); - - const result = await exportLocalRelationshipFeedbackLabels(project, { - now: () => new Date('2026-05-07T13:00:00.000Z'), - }); - - expect(result.summary.total).toBe(1); - expect(result.warnings).toEqual([ - { - path: 'raw-sources/broken/live-database/sync-b/enrichment/relationship-review-decisions.json', - message: expect.any(String), - }, - ]); - expect(result.warnings[0]?.message.length).toBeGreaterThan(0); - }); -}); diff --git a/packages/cli/src/context/scan/relationship-feedback-export.ts b/packages/cli/src/context/scan/relationship-feedback-export.ts deleted file mode 100644 index d4690cef..00000000 --- a/packages/cli/src/context/scan/relationship-feedback-export.ts +++ /dev/null @@ -1,179 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import type { - KtxRelationshipReviewDecisionArtifact, - KtxRelationshipReviewDecisionEntry, - KtxRelationshipReviewDecisionValue, -} from './relationship-review-decisions.js'; - -const DECISION_ARTIFACT_SUFFIX = '/enrichment/relationship-review-decisions.json'; -const FEEDBACK_SCHEMA_VERSION = 1; - -export type KtxRelationshipFeedbackDecisionFilter = KtxRelationshipReviewDecisionValue | 'all'; - -export interface ExportLocalRelationshipFeedbackLabelsInput { - connectionId?: string | null; - decision?: KtxRelationshipFeedbackDecisionFilter; - now?: () => Date; -} - -export interface KtxRelationshipFeedbackLabel { - schemaVersion: 1; - candidateId: string; - decision: KtxRelationshipReviewDecisionValue; - previousStatus: KtxRelationshipReviewDecisionEntry['previousStatus']; - connectionId: string; - runId: string; - syncId: string; - decidedAt: string; - reviewer: string; - note: string | null; - relationshipType: KtxRelationshipReviewDecisionEntry['relationshipType']; - source: string; - score: number | null; - confidence: number; - pkScore: number | null; - fkScore: number | null; - fromTable: string; - fromColumns: string[]; - toTable: string; - toColumns: string[]; - reasons: string[]; - artifactPath: string; -} - -export interface KtxRelationshipFeedbackExportWarning { - path: string; - message: string; -} - -export interface ExportLocalRelationshipFeedbackLabelsResult { - generatedAt: string; - filters: { - connectionId: string | null; - decision: KtxRelationshipFeedbackDecisionFilter; - }; - summary: { - total: number; - accepted: number; - rejected: number; - connections: number; - runs: number; - }; - labels: KtxRelationshipFeedbackLabel[]; - warnings: KtxRelationshipFeedbackExportWarning[]; -} - -function qualifiedTableName(entry: KtxRelationshipReviewDecisionEntry, side: 'from' | 'to'): string { - const table = entry[side].table; - return [table.catalog, table.db, table.name].filter((part): part is string => Boolean(part)).join('.'); -} - -function labelFromDecision(entry: KtxRelationshipReviewDecisionEntry, artifactPath: string): KtxRelationshipFeedbackLabel { - return { - schemaVersion: FEEDBACK_SCHEMA_VERSION, - candidateId: entry.candidateId, - decision: entry.decision, - previousStatus: entry.previousStatus, - connectionId: entry.connectionId, - runId: entry.runId, - syncId: entry.syncId, - decidedAt: entry.decidedAt, - reviewer: entry.reviewer, - note: entry.note, - relationshipType: entry.relationshipType, - source: entry.source, - score: entry.score, - confidence: entry.confidence, - pkScore: entry.pkScore, - fkScore: entry.fkScore, - fromTable: qualifiedTableName(entry, 'from'), - fromColumns: [...entry.from.columns], - toTable: qualifiedTableName(entry, 'to'), - toColumns: [...entry.to.columns], - reasons: [...entry.reasons], - artifactPath, - }; -} - -function sortLabels(labels: KtxRelationshipFeedbackLabel[]): KtxRelationshipFeedbackLabel[] { - return [...labels].sort((left, right) => { - return ( - left.connectionId.localeCompare(right.connectionId) || - left.runId.localeCompare(right.runId) || - left.candidateId.localeCompare(right.candidateId) || - left.decidedAt.localeCompare(right.decidedAt) - ); - }); -} - -function passesFilters( - label: KtxRelationshipFeedbackLabel, - filters: { connectionId: string | null; decision: KtxRelationshipFeedbackDecisionFilter }, -): boolean { - if (filters.connectionId && label.connectionId !== filters.connectionId) { - return false; - } - return filters.decision === 'all' || label.decision === filters.decision; -} - -function messageFromUnknownError(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} - -async function readDecisionLabels( - project: KtxLocalProject, - artifactPath: string, -): Promise { - const raw = await project.fileStore.readFile(artifactPath); - const parsed = JSON.parse(raw.content) as KtxRelationshipReviewDecisionArtifact; - const decisions = Array.isArray(parsed.decisions) ? parsed.decisions : []; - return decisions.map((entry) => labelFromDecision(entry, artifactPath)); -} - -function summarize(labels: KtxRelationshipFeedbackLabel[]): ExportLocalRelationshipFeedbackLabelsResult['summary'] { - return { - total: labels.length, - accepted: labels.filter((label) => label.decision === 'accepted').length, - rejected: labels.filter((label) => label.decision === 'rejected').length, - connections: new Set(labels.map((label) => label.connectionId)).size, - runs: new Set(labels.map((label) => `${label.connectionId}:${label.runId}`)).size, - }; -} - -export async function exportLocalRelationshipFeedbackLabels( - project: KtxLocalProject, - input: ExportLocalRelationshipFeedbackLabelsInput = {}, -): Promise { - const filters = { - connectionId: input.connectionId ?? null, - decision: input.decision ?? 'all', - }; - const listed = await project.fileStore.listFiles('raw-sources'); - const artifactPaths = listed.files.filter((path) => path.endsWith(DECISION_ARTIFACT_SUFFIX)).sort(); - const labels: KtxRelationshipFeedbackLabel[] = []; - const warnings: KtxRelationshipFeedbackExportWarning[] = []; - - for (const artifactPath of artifactPaths) { - try { - labels.push(...(await readDecisionLabels(project, artifactPath))); - } catch (error) { - warnings.push({ path: artifactPath, message: messageFromUnknownError(error) }); - } - } - - const filtered = sortLabels(labels.filter((label) => passesFilters(label, filters))); - return { - generatedAt: (input.now?.() ?? new Date()).toISOString(), - filters, - summary: summarize(filtered), - labels: filtered, - warnings, - }; -} - -export function formatKtxRelationshipFeedbackLabelsJsonl(result: ExportLocalRelationshipFeedbackLabelsResult): string { - if (result.labels.length === 0) { - return ''; - } - return `${result.labels.map((label) => JSON.stringify(label)).join('\n')}\n`; -} diff --git a/packages/cli/src/context/scan/relationship-graph-resolver.ts b/packages/cli/src/context/scan/relationship-graph-resolver.ts index 36ab44b2..e18d1132 100644 --- a/packages/cli/src/context/scan/relationship-graph-resolver.ts +++ b/packages/cli/src/context/scan/relationship-graph-resolver.ts @@ -11,14 +11,14 @@ import type { KtxValidatedRelationshipDiscoveryCandidate } from './relationship- export type KtxResolvedRelationshipStatus = 'accepted' | 'review' | 'rejected'; -export interface KtxRelationshipGraphResolverSettings { +interface KtxRelationshipGraphResolverSettings { acceptThreshold: number; reviewThreshold: number; minTargetPkScoreForAcceptance: number; validationRequiredForManifest: boolean; } -export interface KtxResolvedRelationshipPkEvidence { +interface KtxResolvedRelationshipPkEvidence { declaredPrimaryKey: boolean; targetUniqueness: number; incomingAcceptedCount: number; @@ -26,7 +26,7 @@ export interface KtxResolvedRelationshipPkEvidence { reasons: string[]; } -export interface KtxResolvedRelationshipPk { +interface KtxResolvedRelationshipPk { table: string; columns: string[]; pkScore: number; @@ -35,7 +35,7 @@ export interface KtxResolvedRelationshipPk { evidence: KtxResolvedRelationshipPkEvidence; } -export interface KtxResolvedRelationshipGraphEvidence { +interface KtxResolvedRelationshipGraphEvidence { targetPkScore: number; incomingCandidateCount: number; conflictRank: number; diff --git a/packages/cli/src/context/scan/relationship-llm-proposal.test.ts b/packages/cli/src/context/scan/relationship-llm-proposal.test.ts index 1aa994de..46e22dcb 100644 --- a/packages/cli/src/context/scan/relationship-llm-proposal.test.ts +++ b/packages/cli/src/context/scan/relationship-llm-proposal.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import type { KtxLlmRuntimePort } from '../llm/index.js'; +import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from './enrichment-types.js'; import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js'; import { proposeKtxRelationshipCandidatesWithLlm } from './relationship-llm-proposal.js'; diff --git a/packages/cli/src/context/scan/relationship-llm-proposal.ts b/packages/cli/src/context/scan/relationship-llm-proposal.ts index 7144035f..04b36abe 100644 --- a/packages/cli/src/context/scan/relationship-llm-proposal.ts +++ b/packages/cli/src/context/scan/relationship-llm-proposal.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import type { KtxLlmRuntimePort } from '../llm/index.js'; +import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js'; import type { KtxEnrichedColumn, KtxEnrichedSchema, KtxEnrichedTable } from './enrichment-types.js'; import { normalizeKtxRelationshipName, @@ -31,7 +31,7 @@ const relationshipLlmProposalSchema = z.object({ type KtxRelationshipLlmProposalOutput = z.infer; -export interface KtxRelationshipLlmProposalSettings { +interface KtxRelationshipLlmProposalSettings { maxTablesPerBatch: number; maxColumnsPerTable: number; maxSampleValuesPerColumn: number; diff --git a/packages/cli/src/context/scan/relationship-name-similarity.ts b/packages/cli/src/context/scan/relationship-name-similarity.ts index 14d53fe3..63378c0c 100644 --- a/packages/cli/src/context/scan/relationship-name-similarity.ts +++ b/packages/cli/src/context/scan/relationship-name-similarity.ts @@ -6,6 +6,7 @@ export interface KtxRelationshipNormalizedName { tokens: string[]; } +/** @internal */ export type KtxRelationshipTokenInput = string | readonly string[] | KtxRelationshipNormalizedName; const WAREHOUSE_LAYER_PREFIXES = new Set(['stg', 'stage', 'staging', 'dim', 'fct', 'fact', 'int', 'mart']); @@ -132,6 +133,7 @@ function roundedScore(value: number): number { return Number(Math.max(0, Math.min(1, value)).toFixed(3)); } +/** @internal */ export function tokenSimilarity(leftInput: KtxRelationshipTokenInput, rightInput: KtxRelationshipTokenInput): number { const left = tokensFromInput(leftInput); const right = tokensFromInput(rightInput); diff --git a/packages/cli/src/context/scan/relationship-profiling.ts b/packages/cli/src/context/scan/relationship-profiling.ts index fa6acfac..2172ac24 100644 --- a/packages/cli/src/context/scan/relationship-profiling.ts +++ b/packages/cli/src/context/scan/relationship-profiling.ts @@ -26,6 +26,7 @@ export interface KtxRelationshipColumnProfile { maxTextLength: number | null; } +/** @internal */ export interface KtxRelationshipTableProfile { table: KtxTableRef; rowCount: number; diff --git a/packages/cli/src/context/scan/relationship-review-apply.test.ts b/packages/cli/src/context/scan/relationship-review-apply.test.ts deleted file mode 100644 index 268c9598..00000000 --- a/packages/cli/src/context/scan/relationship-review-apply.test.ts +++ /dev/null @@ -1,351 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import type { KtxLocalProject } from '../project/index.js'; -import { initKtxProject } from '../project/index.js'; -import { describe, expect, it, vi } from 'vitest'; -import { applyLocalScanRelationshipReviewDecisions } from './relationship-review-apply.js'; -import type { KtxRelationshipReviewDecisionArtifact } from './relationship-review-decisions.js'; -import type { ReadLocalScanRelationshipArtifactsResult } from './relationship-artifacts.js'; -import type { WriteLocalScanManifestShardsResult } from './local-enrichment-artifacts.js'; -import type { KtxSchemaSnapshot } from './types.js'; - -const acceptedDecisionArtifact: KtxRelationshipReviewDecisionArtifact = { - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - generatedAt: '2026-05-07T12:00:00.000Z', - decisions: [ - { - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted', - previousStatus: 'review', - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decidedAt: '2026-05-07T12:01:00.000Z', - reviewer: 'Andrey', - note: 'Customer link is valid.', - from: { - tableId: 'public.orders', - columnIds: ['public.orders.customer_id'], - table: { catalog: null, db: 'public', name: 'orders' }, - columns: ['customer_id'], - }, - to: { - tableId: 'public.customers', - columnIds: ['public.customers.id'], - table: { catalog: null, db: 'public', name: 'customers' }, - columns: ['id'], - }, - relationshipType: 'many_to_one', - source: 'deterministic_name', - score: 0.81, - confidence: 0.81, - pkScore: 0.93, - fkScore: 0.81, - reasons: ['review_threshold'], - }, - { - candidateId: 'orders:orders.note_id->notes:notes.id', - decision: 'rejected', - previousStatus: 'review', - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decidedAt: '2026-05-07T12:02:00.000Z', - reviewer: 'Andrey', - note: null, - from: { - tableId: 'public.orders', - columnIds: ['public.orders.note_id'], - table: { catalog: null, db: 'public', name: 'orders' }, - columns: ['note_id'], - }, - to: { - tableId: 'public.notes', - columnIds: ['public.notes.id'], - table: { catalog: null, db: 'public', name: 'notes' }, - columns: ['id'], - }, - relationshipType: 'many_to_one', - source: 'embedding_similarity', - score: 0.7, - confidence: 0.7, - pkScore: 0.7, - fkScore: 0.7, - reasons: ['review_threshold'], - }, - ], -}; - -const artifacts: ReadLocalScanRelationshipArtifactsResult = { - runId: 'scan-run-a', - connectionId: 'warehouse', - syncId: 'sync-a', - report: { - connectionId: 'warehouse', - driver: 'postgres', - syncId: 'sync-a', - runId: 'scan-run-a', - trigger: 'cli', - mode: 'relationships', - dryRun: false, - artifactPaths: { - rawSourcesDir: 'raw-sources/warehouse/live-database/sync-a', - reportPath: 'raw-sources/warehouse/live-database/sync-a/scan-report.json', - manifestShards: ['semantic-layer/warehouse/_schema/public.yaml'], - enrichmentArtifacts: ['raw-sources/warehouse/live-database/sync-a/enrichment/relationships.json'], - }, - diffSummary: { - tablesAdded: 0, - tablesModified: 0, - tablesDeleted: 0, - tablesUnchanged: 2, - columnsAdded: 0, - columnsModified: 0, - columnsDeleted: 0, - }, - manifestShardsWritten: 1, - structuralSyncStats: { - tablesCreated: 0, - tablesUpdated: 0, - tablesDeleted: 0, - columnsCreated: 0, - columnsUpdated: 0, - columnsDeleted: 0, - }, - enrichment: { - dataDictionary: 'skipped', - tableDescriptions: 'skipped', - columnDescriptions: 'skipped', - embeddings: 'skipped', - deterministicRelationships: 'completed', - llmRelationshipValidation: 'skipped', - statisticalValidation: 'completed', - }, - capabilityGaps: [], - warnings: [], - relationships: { accepted: 0, review: 1, rejected: 1, skipped: 0 }, - enrichmentState: { resumedStages: [], completedStages: ['relationships'], failedStages: [] }, - createdAt: '2026-05-07T12:00:00.000Z', - }, - relationships: { - connectionId: 'warehouse', - accepted: [], - review: [], - rejected: [], - skipped: [], - }, - diagnostics: null, - profile: null, - paths: { - relationships: 'raw-sources/warehouse/live-database/sync-a/enrichment/relationships.json', - diagnostics: null, - profile: null, - }, -}; - -const snapshot: KtxSchemaSnapshot = { - connectionId: 'warehouse', - driver: 'postgres', - extractedAt: '2026-05-07T12:00:00.000Z', - scope: { schemas: ['public'] }, - metadata: {}, - tables: [ - { - catalog: null, - db: 'public', - name: 'customers', - kind: 'table', - comment: null, - estimatedRows: 2, - foreignKeys: [], - columns: [ - { - name: 'id', - nativeType: 'integer', - normalizedType: 'integer', - dimensionType: 'number', - nullable: false, - primaryKey: false, - comment: null, - }, - ], - }, - { - catalog: null, - db: 'public', - name: 'orders', - kind: 'table', - comment: null, - estimatedRows: 2, - foreignKeys: [], - columns: [ - { - name: 'customer_id', - nativeType: 'integer', - normalizedType: 'integer', - dimensionType: 'number', - nullable: false, - primaryKey: false, - comment: null, - }, - ], - }, - ], -}; - -async function projectWithDecisions( - decisions = acceptedDecisionArtifact, -): Promise<{ project: KtxLocalProject; tempDir: string }> { - const tempDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-review-apply-')); - const project = await initKtxProject({ - projectDir: join(tempDir, 'project'), - }); - await project.fileStore.writeFile( - 'raw-sources/warehouse/live-database/sync-a/enrichment/relationship-review-decisions.json', - `${JSON.stringify(decisions)}\n`, - 'ktx', - 'ktx@example.com', - 'Seed relationship review decisions', - ); - return { project, tempDir }; -} - -function manifestResult(): WriteLocalScanManifestShardsResult { - return { - manifestShards: ['semantic-layer/warehouse/_schema/public.yaml'], - manifestShardsWritten: 1, - }; -} - -describe('relationship review apply', () => { - it('previews all accepted decisions without writing manifest shards', async () => { - const { project, tempDir } = await projectWithDecisions(); - const writeLocalScanManifestShards = vi.fn(async () => manifestResult()); - try { - const result = await applyLocalScanRelationshipReviewDecisions(project, { - runId: 'scan-run-a', - applyAllAccepted: true, - dryRun: true, - readLocalScanRelationshipArtifacts: vi.fn(async () => artifacts), - readLocalScanStructuralSnapshot: vi.fn(async () => snapshot), - writeLocalScanManifestShards, - }); - - expect(result).toMatchObject({ - runId: 'scan-run-a', - connectionId: 'warehouse', - syncId: 'sync-a', - dryRun: true, - selectedDecisions: 1, - appliedRelationships: 1, - manifestShards: [], - manifestShardsWritten: 0, - }); - expect(result.relationships[0]).toMatchObject({ - id: 'orders:orders.customer_id->customers:customers.id', - source: 'manual', - relationshipType: 'many_to_one', - confidence: 1, - }); - expect(writeLocalScanManifestShards).not.toHaveBeenCalled(); - } finally { - await rm(tempDir, { recursive: true, force: true }); - } - }); - - it('writes selected accepted decisions as manual manifest relationships', async () => { - const { project, tempDir } = await projectWithDecisions(); - const readLocalScanStructuralSnapshot = vi.fn(async () => snapshot); - const writeLocalScanManifestShards = vi.fn(async () => manifestResult()); - try { - const result = await applyLocalScanRelationshipReviewDecisions(project, { - runId: 'scan-run-a', - candidateIds: ['orders:orders.customer_id->customers:customers.id'], - readLocalScanRelationshipArtifacts: vi.fn(async () => artifacts), - readLocalScanStructuralSnapshot, - writeLocalScanManifestShards, - }); - - expect(readLocalScanStructuralSnapshot).toHaveBeenCalledWith({ - project: expect.any(Object), - connectionId: 'warehouse', - driver: 'postgres', - rawSourcesDir: 'raw-sources/warehouse/live-database/sync-a', - extractedAtFallback: '2026-05-07T12:00:00.000Z', - }); - expect(writeLocalScanManifestShards).toHaveBeenCalledWith({ - project: expect.any(Object), - connectionId: 'warehouse', - syncId: 'sync-a', - driver: 'postgres', - snapshot, - dryRun: false, - relationshipUpdate: { - connectionId: 'warehouse', - accepted: [ - expect.objectContaining({ - id: 'orders:orders.customer_id->customers:customers.id', - source: 'manual', - from: expect.objectContaining({ columns: ['customer_id'] }), - to: expect.objectContaining({ columns: ['id'] }), - }), - ], - rejected: [], - skipped: [], - }, - }); - expect(result.manifestShardsWritten).toBe(1); - } finally { - await rm(tempDir, { recursive: true, force: true }); - } - }); - - it('rejects ambiguous apply selection input', async () => { - const { project, tempDir } = await projectWithDecisions(); - try { - await expect( - applyLocalScanRelationshipReviewDecisions(project, { - runId: 'scan-run-a', - readLocalScanRelationshipArtifacts: vi.fn(async () => artifacts), - }), - ).rejects.toThrow('Pass --all-accepted or at least one --candidate to choose review decisions to apply'); - - await expect( - applyLocalScanRelationshipReviewDecisions(project, { - runId: 'scan-run-a', - applyAllAccepted: true, - candidateIds: ['orders:orders.customer_id->customers:customers.id'], - readLocalScanRelationshipArtifacts: vi.fn(async () => artifacts), - }), - ).rejects.toThrow('Use either --all-accepted or --candidate, not both'); - } finally { - await rm(tempDir, { recursive: true, force: true }); - } - }); - - it('refuses rejected decisions and missing candidate ids', async () => { - const { project, tempDir } = await projectWithDecisions(); - try { - await expect( - applyLocalScanRelationshipReviewDecisions(project, { - runId: 'scan-run-a', - candidateIds: ['orders:orders.note_id->notes:notes.id'], - readLocalScanRelationshipArtifacts: vi.fn(async () => artifacts), - }), - ).rejects.toThrow('Relationship review decision "orders:orders.note_id->notes:notes.id" is rejected, not accepted'); - - await expect( - applyLocalScanRelationshipReviewDecisions(project, { - runId: 'scan-run-a', - candidateIds: ['missing'], - readLocalScanRelationshipArtifacts: vi.fn(async () => artifacts), - }), - ).rejects.toThrow('Relationship review decision "missing" was not found for scan run "scan-run-a"'); - } finally { - await rm(tempDir, { recursive: true, force: true }); - } - }); -}); diff --git a/packages/cli/src/context/scan/relationship-review-apply.ts b/packages/cli/src/context/scan/relationship-review-apply.ts deleted file mode 100644 index 3afa6a7e..00000000 --- a/packages/cli/src/context/scan/relationship-review-apply.ts +++ /dev/null @@ -1,231 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import { - readLocalScanRelationshipArtifacts, - type ReadLocalScanRelationshipArtifactsResult, -} from './relationship-artifacts.js'; -import { - readLocalScanStructuralSnapshot, - type ReadLocalScanStructuralSnapshotInput, -} from './local-structural-artifacts.js'; -import { - writeLocalScanManifestShards, - type WriteLocalScanManifestShardsInput, - type WriteLocalScanManifestShardsResult, -} from './local-enrichment-artifacts.js'; -import type { KtxEnrichedRelationship, KtxRelationshipUpdate } from './enrichment-types.js'; -import type { - KtxRelationshipReviewDecisionArtifact, - KtxRelationshipReviewDecisionEntry, -} from './relationship-review-decisions.js'; - -const DECISIONS_FILE = 'relationship-review-decisions.json'; - -export interface ApplyLocalScanRelationshipReviewDecisionsInput { - runId: string; - applyAllAccepted?: boolean; - candidateIds?: readonly string[]; - dryRun?: boolean; - readLocalScanRelationshipArtifacts?: typeof readLocalScanRelationshipArtifacts; - readLocalScanStructuralSnapshot?: ( - input: ReadLocalScanStructuralSnapshotInput, - ) => Promise; - writeLocalScanManifestShards?: ( - input: WriteLocalScanManifestShardsInput, - ) => Promise; -} - -export interface AppliedRelationshipReviewDecision { - candidateId: string; - decidedAt: string; - reviewer: string; - note: string | null; - relationship: KtxEnrichedRelationship; -} - -export interface ApplyLocalScanRelationshipReviewDecisionsResult { - runId: string; - connectionId: string; - syncId: string; - dryRun: boolean; - decisionsPath: string; - selectedDecisions: number; - appliedRelationships: number; - relationships: KtxEnrichedRelationship[]; - manifestShards: string[]; - manifestShardsWritten: number; -} - -function decisionsPathFromRelationshipsPath(relationshipsPath: string): string { - return relationshipsPath.replace(/relationships\.json$/u, DECISIONS_FILE); -} - -async function readDecisionArtifact( - project: KtxLocalProject, - path: string, - runId: string, -): Promise { - let raw: { content: string }; - try { - raw = await project.fileStore.readFile(path); - } catch { - throw new Error(`Relationship review decisions were not found for scan run "${runId}"`); - } - const parsed = JSON.parse(raw.content) as KtxRelationshipReviewDecisionArtifact; - return { - connectionId: parsed.connectionId, - runId: parsed.runId, - syncId: parsed.syncId, - generatedAt: parsed.generatedAt, - decisions: Array.isArray(parsed.decisions) ? parsed.decisions : [], - }; -} - -function assertSelection(input: ApplyLocalScanRelationshipReviewDecisionsInput): void { - const candidateIds = input.candidateIds ?? []; - if (input.applyAllAccepted === true && candidateIds.length > 0) { - throw new Error('Use either --all-accepted or --candidate, not both'); - } - if (input.applyAllAccepted !== true && candidateIds.length === 0) { - throw new Error('Pass --all-accepted or at least one --candidate to choose review decisions to apply'); - } -} - -function selectAcceptedDecisions( - artifact: KtxRelationshipReviewDecisionArtifact, - input: ApplyLocalScanRelationshipReviewDecisionsInput, -): KtxRelationshipReviewDecisionEntry[] { - assertSelection(input); - if (input.applyAllAccepted === true) { - return artifact.decisions.filter((decision) => decision.decision === 'accepted'); - } - - const decisionsById = new Map(artifact.decisions.map((decision) => [decision.candidateId, decision])); - const selected: KtxRelationshipReviewDecisionEntry[] = []; - for (const candidateId of input.candidateIds ?? []) { - const decision = decisionsById.get(candidateId); - if (!decision) { - throw new Error(`Relationship review decision "${candidateId}" was not found for scan run "${input.runId}"`); - } - if (decision.decision !== 'accepted') { - throw new Error(`Relationship review decision "${candidateId}" is ${decision.decision}, not accepted`); - } - selected.push(decision); - } - return selected; -} - -function tableId(table: KtxRelationshipReviewDecisionEntry['from']['table']): string { - return [table.catalog, table.db, table.name].filter((part): part is string => Boolean(part)).join('.'); -} - -function columnIds(table: KtxRelationshipReviewDecisionEntry['from']['table'], columns: readonly string[]): string[] { - const prefix = tableId(table); - return columns.map((column) => `${prefix}.${column}`); -} - -function relationshipFromDecision(decision: KtxRelationshipReviewDecisionEntry): KtxEnrichedRelationship { - return { - id: decision.candidateId, - source: 'manual', - from: { - tableId: tableId(decision.from.table), - columnIds: columnIds(decision.from.table, decision.from.columns), - table: decision.from.table, - columns: [...decision.from.columns], - }, - to: { - tableId: tableId(decision.to.table), - columnIds: columnIds(decision.to.table, decision.to.columns), - table: decision.to.table, - columns: [...decision.to.columns], - }, - relationshipType: decision.relationshipType, - confidence: 1, - isPrimaryKeyReference: true, - }; -} - -function relationshipUpdate( - connectionId: string, - relationships: readonly KtxEnrichedRelationship[], -): KtxRelationshipUpdate { - return { - connectionId, - accepted: [...relationships], - rejected: [], - skipped: [], - }; -} - -function assertApplyableArtifacts(artifacts: ReadLocalScanRelationshipArtifactsResult): string { - const rawSourcesDir = artifacts.report.artifactPaths.rawSourcesDir; - if (!rawSourcesDir) { - throw new Error(`Scan run "${artifacts.runId}" does not have raw source artifacts for manifest rewriting`); - } - return rawSourcesDir; -} - -export async function applyLocalScanRelationshipReviewDecisions( - project: KtxLocalProject, - input: ApplyLocalScanRelationshipReviewDecisionsInput, -): Promise { - const readArtifacts = input.readLocalScanRelationshipArtifacts ?? readLocalScanRelationshipArtifacts; - const artifacts = await readArtifacts(project, input.runId); - if (!artifacts) { - throw new Error(`Scan run "${input.runId}" was not found`); - } - - const decisionsPath = decisionsPathFromRelationshipsPath(artifacts.paths.relationships); - const decisions = await readDecisionArtifact(project, decisionsPath, input.runId); - const selected = selectAcceptedDecisions(decisions, input); - const relationships = selected.map((decision) => relationshipFromDecision(decision)); - const dryRun = input.dryRun === true; - - if (dryRun || relationships.length === 0) { - return { - runId: artifacts.runId, - connectionId: artifacts.connectionId, - syncId: artifacts.syncId, - dryRun, - decisionsPath, - selectedDecisions: selected.length, - appliedRelationships: relationships.length, - relationships, - manifestShards: [], - manifestShardsWritten: 0, - }; - } - - const rawSourcesDir = assertApplyableArtifacts(artifacts); - const readSnapshot = input.readLocalScanStructuralSnapshot ?? readLocalScanStructuralSnapshot; - const writeManifestShards = input.writeLocalScanManifestShards ?? writeLocalScanManifestShards; - const snapshot = await readSnapshot({ - project, - connectionId: artifacts.connectionId, - driver: artifacts.report.driver, - rawSourcesDir, - extractedAtFallback: artifacts.report.createdAt, - }); - const manifest = await writeManifestShards({ - project, - connectionId: artifacts.connectionId, - syncId: artifacts.syncId, - driver: artifacts.report.driver, - snapshot, - dryRun: false, - relationshipUpdate: relationshipUpdate(artifacts.connectionId, relationships), - }); - - return { - runId: artifacts.runId, - connectionId: artifacts.connectionId, - syncId: artifacts.syncId, - dryRun, - decisionsPath, - selectedDecisions: selected.length, - appliedRelationships: relationships.length, - relationships, - manifestShards: manifest.manifestShards, - manifestShardsWritten: manifest.manifestShardsWritten, - }; -} diff --git a/packages/cli/src/context/scan/relationship-review-decisions.test.ts b/packages/cli/src/context/scan/relationship-review-decisions.test.ts deleted file mode 100644 index 979c863b..00000000 --- a/packages/cli/src/context/scan/relationship-review-decisions.test.ts +++ /dev/null @@ -1,363 +0,0 @@ -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; -import { runLocalStageOnlyIngest, type SourceAdapter } from '../ingest/index.js'; -import { initKtxProject, loadKtxProject } from '../project/index.js'; -import { describe, expect, it } from 'vitest'; -import { writeLocalScanRelationshipReviewDecision } from './relationship-review-decisions.js'; -import type { KtxRelationshipArtifact, KtxRelationshipDiagnosticsArtifact } from './relationship-diagnostics.js'; -import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js'; -import type { KtxScanReport } from './types.js'; - -const RUN_ID = 'scan-run-review'; -const SYNC_ID = '2026-05-07-100000-scan-run-review'; - -async function writeProjectFile(projectDir: string, relativePath: string, content: string): Promise { - const absolutePath = join(projectDir, relativePath); - await mkdir(dirname(absolutePath), { recursive: true }); - await writeFile(absolutePath, content, 'utf-8'); -} - -async function createProject(projectDir: string): Promise { - await initKtxProject({ projectDir }); - await writeFile( - join(projectDir, 'ktx.yaml'), - [ - 'connections:', - ' warehouse:', - ' driver: sqlite', - ' path: warehouse.db', - 'ingest:', - ' adapters:', - ' - live-database', - '', - ].join('\n'), - 'utf-8', - ); -} - -function liveDatabaseAdapter(): SourceAdapter { - return { - source: 'live-database', - skillNames: ['live_database_ingest'], - async fetch(_pullConfig, stagedDir) { - await mkdir(join(stagedDir, 'tables'), { recursive: true }); - await writeFile(join(stagedDir, 'connection.json'), '{"connectionId":"warehouse"}\n', 'utf-8'); - await writeFile(join(stagedDir, 'foreign-keys.json'), '{"foreignKeys":[]}\n', 'utf-8'); - await writeFile( - join(stagedDir, 'tables', 'orders.json'), - '{"name":"orders","db":"public","columns":[{"name":"id","type":"integer","nullable":false,"primaryKey":true}]}\n', - 'utf-8', - ); - }, - async detect(stagedDir) { - await writeFile(join(stagedDir, 'connection.json'), '{"connectionId":"warehouse"}\n', 'utf-8'); - return true; - }, - async chunk() { - return { - workUnits: [ - { - unitKey: 'live-database-public-orders', - rawFiles: ['tables/orders.json'], - dependencyPaths: ['connection.json', 'foreign-keys.json'], - peerFileIndex: [], - }, - ], - }; - }, - }; -} - -async function createLiveDatabaseRun(projectDir: string): Promise { - await createProject(projectDir); - const project = await loadKtxProject({ projectDir }); - await runLocalStageOnlyIngest({ - project, - adapters: [liveDatabaseAdapter()], - adapter: 'live-database', - connectionId: 'warehouse', - jobId: RUN_ID, - now: () => new Date('2026-05-07T10:00:00.000Z'), - }); -} - -function reviewRelationships(): KtxRelationshipArtifact { - return { - connectionId: 'warehouse', - accepted: [], - review: [ - { - id: 'orders:orders.customer_id->customers:customers.id', - status: 'review', - source: 'deterministic_name', - from: { - tableId: 'orders', - columnIds: ['orders.customer_id'], - table: { catalog: null, db: 'public', name: 'orders' }, - columns: ['customer_id'], - }, - to: { - tableId: 'customers', - columnIds: ['customers.id'], - table: { catalog: null, db: 'public', name: 'customers' }, - columns: ['id'], - }, - relationshipType: 'many_to_one', - confidence: 0.62, - pkScore: 0.91, - fkScore: 0.62, - score: 0.62, - evidence: { sources: ['table_suffix'] }, - validation: { status: 'passed' }, - graph: { reasons: ['fk_score_review'] }, - reasons: ['fk_score_review'], - }, - ], - rejected: [], - skipped: [], - }; -} - -function diagnostics(): KtxRelationshipDiagnosticsArtifact { - return { - connectionId: 'warehouse', - generatedAt: '2026-05-07T10:00:00.000Z', - summary: { accepted: 0, review: 1, rejected: 0, skipped: 0 }, - noAcceptedReason: 'relationship candidates require review before manifest writes', - candidateCountsBySource: { deterministic_name: 1 }, - validation: { available: true, sqlAvailable: true, queryCount: 3 }, - thresholds: { acceptThreshold: 0.85, reviewThreshold: 0.55 }, - policy: { - validationRequiredForManifest: true, - maxCandidatesPerColumn: 25, - profileSampleRows: 10000, - validationConcurrency: 4, - }, - warnings: [], - profileWarnings: [], - }; -} - -function profile(): KtxRelationshipProfileArtifact { - return { - connectionId: 'warehouse', - driver: 'sqlite', - sqlAvailable: true, - tables: [], - columns: {}, - queryCount: 3, - warnings: [], - }; -} - -function report(): KtxScanReport { - return { - connectionId: 'warehouse', - driver: 'sqlite', - syncId: SYNC_ID, - runId: RUN_ID, - trigger: 'cli', - mode: 'relationships', - dryRun: false, - artifactPaths: { - rawSourcesDir: `raw-sources/warehouse/live-database/${SYNC_ID}`, - reportPath: `raw-sources/warehouse/live-database/${SYNC_ID}/scan-report.json`, - manifestShards: [], - enrichmentArtifacts: [ - `raw-sources/warehouse/live-database/${SYNC_ID}/enrichment/relationships.json`, - `raw-sources/warehouse/live-database/${SYNC_ID}/enrichment/relationship-diagnostics.json`, - `raw-sources/warehouse/live-database/${SYNC_ID}/enrichment/relationship-profile.json`, - ], - }, - diffSummary: { - tablesAdded: 0, - tablesModified: 0, - tablesDeleted: 0, - tablesUnchanged: 2, - columnsAdded: 0, - columnsModified: 0, - columnsDeleted: 0, - }, - manifestShardsWritten: 0, - structuralSyncStats: { - tablesCreated: 0, - tablesUpdated: 0, - tablesDeleted: 0, - columnsCreated: 0, - columnsUpdated: 0, - columnsDeleted: 0, - }, - enrichment: { - dataDictionary: 'skipped', - tableDescriptions: 'skipped', - columnDescriptions: 'skipped', - embeddings: 'skipped', - deterministicRelationships: 'completed', - llmRelationshipValidation: 'skipped', - statisticalValidation: 'completed', - }, - relationships: { accepted: 0, review: 1, rejected: 0, skipped: 0 }, - enrichmentState: { - resumedStages: [], - completedStages: ['relationships'], - failedStages: [], - }, - warnings: [], - capabilityGaps: [], - createdAt: '2026-05-07T10:00:00.000Z', - }; -} - -async function writeScanArtifacts(projectDir: string): Promise { - await writeProjectFile( - projectDir, - `raw-sources/warehouse/live-database/${SYNC_ID}/scan-report.json`, - JSON.stringify(report(), null, 2), - ); - await writeProjectFile( - projectDir, - `raw-sources/warehouse/live-database/${SYNC_ID}/enrichment/relationships.json`, - JSON.stringify(reviewRelationships(), null, 2), - ); - await writeProjectFile( - projectDir, - `raw-sources/warehouse/live-database/${SYNC_ID}/enrichment/relationship-diagnostics.json`, - JSON.stringify(diagnostics(), null, 2), - ); - await writeProjectFile( - projectDir, - `raw-sources/warehouse/live-database/${SYNC_ID}/enrichment/relationship-profile.json`, - JSON.stringify(profile(), null, 2), - ); -} - -describe('relationship review decisions', () => { - it('writes an accepted decision beside the scan relationship artifacts', async () => { - const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-review-decisions-')); - try { - await createLiveDatabaseRun(projectDir); - await writeScanArtifacts(projectDir); - const project = await loadKtxProject({ projectDir }); - - const result = await writeLocalScanRelationshipReviewDecision(project, { - runId: 'scan-run-review', - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted', - reviewer: 'Andrey', - note: 'Matches the warehouse model', - decidedAt: '2026-05-07T12:00:00.000Z', - }); - - expect(result).not.toBeNull(); - if (!result) { - throw new Error('Expected relationship review decision to be written'); - } - expect(result.path).toBe( - `raw-sources/warehouse/live-database/${SYNC_ID}/enrichment/relationship-review-decisions.json`, - ); - expect(result.artifact.decisions).toHaveLength(1); - expect(result.decision).toMatchObject({ - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted', - previousStatus: 'review', - reviewer: 'Andrey', - note: 'Matches the warehouse model', - source: 'deterministic_name', - relationshipType: 'many_to_one', - score: 0.62, - reasons: ['fk_score_review'], - }); - await expect(project.fileStore.readFile(result.path)).resolves.toMatchObject({ - path: result.path, - content: expect.stringContaining('"decision": "accepted"'), - }); - } finally { - await rm(projectDir, { recursive: true, force: true }); - } - }); - - it('replaces the existing decision for the same candidate id', async () => { - const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-review-replace-')); - try { - await createLiveDatabaseRun(projectDir); - await writeScanArtifacts(projectDir); - const project = await loadKtxProject({ projectDir }); - - await writeLocalScanRelationshipReviewDecision(project, { - runId: 'scan-run-review', - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted', - reviewer: 'Andrey', - note: 'First decision', - decidedAt: '2026-05-07T12:00:00.000Z', - }); - const replacement = await writeLocalScanRelationshipReviewDecision(project, { - runId: 'scan-run-review', - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'rejected', - reviewer: 'Andrey', - note: 'Reviewed against source data and rejected', - decidedAt: '2026-05-07T12:05:00.000Z', - }); - - expect(replacement).not.toBeNull(); - if (!replacement) { - throw new Error('Expected replacement relationship review decision to be written'); - } - expect(replacement.artifact.decisions).toHaveLength(1); - expect(replacement.artifact.decisions[0]).toMatchObject({ - decision: 'rejected', - note: 'Reviewed against source data and rejected', - decidedAt: '2026-05-07T12:05:00.000Z', - }); - } finally { - await rm(projectDir, { recursive: true, force: true }); - } - }); - - it('returns null when the scan run does not exist', async () => { - const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-review-missing-run-')); - try { - await createProject(projectDir); - const project = await loadKtxProject({ projectDir }); - - await expect( - writeLocalScanRelationshipReviewDecision(project, { - runId: 'missing-run', - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted', - reviewer: 'Andrey', - note: null, - decidedAt: '2026-05-07T12:00:00.000Z', - }), - ).resolves.toBeNull(); - } finally { - await rm(projectDir, { recursive: true, force: true }); - } - }); - - it('rejects unknown candidate ids for an existing scan run', async () => { - const projectDir = await mkdtemp(join(tmpdir(), 'ktx-relationship-review-missing-candidate-')); - try { - await createLiveDatabaseRun(projectDir); - await writeScanArtifacts(projectDir); - const project = await loadKtxProject({ projectDir }); - - await expect( - writeLocalScanRelationshipReviewDecision(project, { - runId: 'scan-run-review', - candidateId: 'orders:orders.unknown_id->customers:customers.id', - decision: 'accepted', - reviewer: 'Andrey', - note: null, - decidedAt: '2026-05-07T12:00:00.000Z', - }), - ).rejects.toThrow( - 'Relationship candidate "orders:orders.unknown_id->customers:customers.id" was not found in scan run "scan-run-review"', - ); - } finally { - await rm(projectDir, { recursive: true, force: true }); - } - }); -}); diff --git a/packages/cli/src/context/scan/relationship-review-decisions.ts b/packages/cli/src/context/scan/relationship-review-decisions.ts deleted file mode 100644 index bf459d8a..00000000 --- a/packages/cli/src/context/scan/relationship-review-decisions.ts +++ /dev/null @@ -1,182 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import type { KtxRelationshipType } from './enrichment-types.js'; -import { readLocalScanRelationshipArtifacts } from './relationship-artifacts.js'; -import type { - KtxRelationshipArtifactEdge, - KtxRelationshipArtifactEndpoint, -} from './relationship-diagnostics.js'; -import type { KtxResolvedRelationshipStatus } from './relationship-graph-resolver.js'; - -const LOCAL_AUTHOR = 'ktx'; -const LOCAL_AUTHOR_EMAIL = 'ktx@example.com'; -const DECISIONS_FILE = 'relationship-review-decisions.json'; - -export type KtxRelationshipReviewDecisionValue = 'accepted' | 'rejected'; - -export interface WriteLocalScanRelationshipReviewDecisionInput { - runId: string; - candidateId: string; - decision: KtxRelationshipReviewDecisionValue; - reviewer: string; - note: string | null; - decidedAt?: string; -} - -export interface KtxRelationshipReviewDecisionEntry { - candidateId: string; - decision: KtxRelationshipReviewDecisionValue; - previousStatus: KtxResolvedRelationshipStatus; - connectionId: string; - runId: string; - syncId: string; - decidedAt: string; - reviewer: string; - note: string | null; - from: KtxRelationshipArtifactEndpoint; - to: KtxRelationshipArtifactEndpoint; - relationshipType: KtxRelationshipType; - source: string; - score: number | null; - confidence: number; - pkScore: number | null; - fkScore: number | null; - reasons: string[]; -} - -export interface KtxRelationshipReviewDecisionArtifact { - connectionId: string; - runId: string; - syncId: string; - generatedAt: string; - decisions: KtxRelationshipReviewDecisionEntry[]; -} - -export interface WriteLocalScanRelationshipReviewDecisionResult { - path: string; - decision: KtxRelationshipReviewDecisionEntry; - artifact: KtxRelationshipReviewDecisionArtifact; -} - -function reviewDecisionPath(relationshipsPath: string): string { - return relationshipsPath.replace(/relationships\.json$/u, DECISIONS_FILE); -} - -function allCandidateEdges(result: Awaited>): KtxRelationshipArtifactEdge[] { - if (!result) { - return []; - } - return [...result.relationships.accepted, ...result.relationships.review, ...result.relationships.rejected]; -} - -async function readExistingDecisions( - project: KtxLocalProject, - path: string, - fallback: Omit, -): Promise { - try { - const raw = await project.fileStore.readFile(path); - const parsed = JSON.parse(raw.content) as KtxRelationshipReviewDecisionArtifact; - return { - connectionId: parsed.connectionId, - runId: parsed.runId, - syncId: parsed.syncId, - generatedAt: parsed.generatedAt, - decisions: Array.isArray(parsed.decisions) ? parsed.decisions : [], - }; - } catch { - return { ...fallback, decisions: [] }; - } -} - -function decisionEntry(input: { - candidate: KtxRelationshipArtifactEdge; - connectionId: string; - runId: string; - syncId: string; - decision: KtxRelationshipReviewDecisionValue; - reviewer: string; - note: string | null; - decidedAt: string; -}): KtxRelationshipReviewDecisionEntry { - return { - candidateId: input.candidate.id, - decision: input.decision, - previousStatus: input.candidate.status, - connectionId: input.connectionId, - runId: input.runId, - syncId: input.syncId, - decidedAt: input.decidedAt, - reviewer: input.reviewer, - note: input.note, - from: input.candidate.from, - to: input.candidate.to, - relationshipType: input.candidate.relationshipType, - source: input.candidate.source, - score: input.candidate.score, - confidence: input.candidate.confidence, - pkScore: input.candidate.pkScore, - fkScore: input.candidate.fkScore, - reasons: [...input.candidate.reasons], - }; -} - -function upsertDecision( - existing: readonly KtxRelationshipReviewDecisionEntry[], - next: KtxRelationshipReviewDecisionEntry, -): KtxRelationshipReviewDecisionEntry[] { - return [...existing.filter((item) => item.candidateId !== next.candidateId), next].sort((left, right) => - left.candidateId.localeCompare(right.candidateId), - ); -} - -export async function writeLocalScanRelationshipReviewDecision( - project: KtxLocalProject, - input: WriteLocalScanRelationshipReviewDecisionInput, -): Promise { - const artifacts = await readLocalScanRelationshipArtifacts(project, input.runId); - if (!artifacts) { - return null; - } - - const candidate = allCandidateEdges(artifacts).find((edge) => edge.id === input.candidateId); - if (!candidate) { - throw new Error(`Relationship candidate "${input.candidateId}" was not found in scan run "${input.runId}"`); - } - - const decidedAt = input.decidedAt ?? new Date().toISOString(); - const path = reviewDecisionPath(artifacts.paths.relationships); - const fallback = { - connectionId: artifacts.connectionId, - runId: artifacts.runId, - syncId: artifacts.syncId, - generatedAt: decidedAt, - }; - const existing = await readExistingDecisions(project, path, fallback); - const decision = decisionEntry({ - candidate, - connectionId: artifacts.connectionId, - runId: artifacts.runId, - syncId: artifacts.syncId, - decision: input.decision, - reviewer: input.reviewer, - note: input.note, - decidedAt, - }); - const artifact: KtxRelationshipReviewDecisionArtifact = { - connectionId: artifacts.connectionId, - runId: artifacts.runId, - syncId: artifacts.syncId, - generatedAt: decidedAt, - decisions: upsertDecision(existing.decisions, decision), - }; - - await project.fileStore.writeFile( - path, - `${JSON.stringify(artifact, null, 2)}\n`, - LOCAL_AUTHOR, - LOCAL_AUTHOR_EMAIL, - `scan(live-database): record relationship review decision runId=${input.runId}`, - ); - - return { path, decision, artifact }; -} diff --git a/packages/cli/src/context/scan/relationship-scoring.ts b/packages/cli/src/context/scan/relationship-scoring.ts index ce76f9f4..a6f03e92 100644 --- a/packages/cli/src/context/scan/relationship-scoring.ts +++ b/packages/cli/src/context/scan/relationship-scoring.ts @@ -1,4 +1,4 @@ -export const KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS = [ +const KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS = [ 'nameSimilarity', 'typeCompatibility', 'valueOverlap', @@ -8,7 +8,7 @@ export const KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS = [ 'structuralPrior', ] as const; -export type KtxRelationshipScoreSignal = (typeof KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS)[number]; +type KtxRelationshipScoreSignal = (typeof KTX_RELATIONSHIP_SCORE_SIGNAL_KEYS)[number]; export type KtxRelationshipFixtureOrigin = 'synthetic' | 'public' | 'customer'; @@ -31,6 +31,7 @@ export interface KtxRelationshipScoreBreakdown { contributions: KtxRelationshipScoreWeights; } +/** @internal */ export interface KtxRelationshipScoringCalibrationObservation { fixtureId: string; origin: KtxRelationshipFixtureOrigin; @@ -71,10 +72,12 @@ function sanitizeSignalVector(signals: KtxRelationshipSignalVector): KtxRelation }; } +/** @internal */ export function defaultKtxRelationshipScoreWeights(): KtxRelationshipScoreWeights { return { ...DEFAULT_WEIGHTS }; } +/** @internal */ export function normalizeKtxRelationshipScoreWeights( weights: Partial = DEFAULT_WEIGHTS, ): KtxRelationshipScoreWeights { @@ -123,6 +126,7 @@ function averageSignal( return observations.reduce((sum, observation) => sum + clampScore(observation.signals[key]), 0) / observations.length; } +/** @internal */ export function calibrateWeightsFromSyntheticFixtures( observations: readonly KtxRelationshipScoringCalibrationObservation[], ): KtxRelationshipScoreWeights { diff --git a/packages/cli/src/context/scan/relationship-threshold-advice.test.ts b/packages/cli/src/context/scan/relationship-threshold-advice.test.ts deleted file mode 100644 index aceb19c2..00000000 --- a/packages/cli/src/context/scan/relationship-threshold-advice.test.ts +++ /dev/null @@ -1,241 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import { describe, expect, it, vi } from 'vitest'; -import { - adviseLocalRelationshipFeedbackThresholds, - buildKtxRelationshipThresholdAdviceReport, - formatKtxRelationshipThresholdAdviceMarkdown, -} from './relationship-threshold-advice.js'; -import type { - ExportLocalRelationshipFeedbackLabelsResult, - KtxRelationshipFeedbackLabel, -} from './relationship-feedback-export.js'; - -function label( - input: Partial & Pick, -): KtxRelationshipFeedbackLabel { - return { - schemaVersion: 1, - previousStatus: 'review', - connectionId: 'warehouse', - runId: 'scan-run-a', - syncId: 'sync-a', - decidedAt: '2026-05-07T12:00:00.000Z', - reviewer: 'Andrey', - note: null, - relationshipType: 'many_to_one', - source: 'deterministic_name', - confidence: input.score ?? 0, - pkScore: input.pkScore ?? null, - fkScore: input.fkScore ?? input.score, - fromTable: 'public.orders', - fromColumns: ['customer_id'], - toTable: 'public.customers', - toColumns: ['id'], - reasons: [], - artifactPath: 'raw-sources/warehouse/live-database/sync-a/enrichment/relationship-review-decisions.json', - ...input, - }; -} - -function feedback(labels: KtxRelationshipFeedbackLabel[]): ExportLocalRelationshipFeedbackLabelsResult { - return { - generatedAt: '2026-05-07T13:00:00.000Z', - filters: { connectionId: null, decision: 'all' }, - summary: { - total: labels.length, - accepted: labels.filter((item) => item.decision === 'accepted').length, - rejected: labels.filter((item) => item.decision === 'rejected').length, - connections: new Set(labels.map((item) => item.connectionId)).size, - runs: new Set(labels.map((item) => `${item.connectionId}:${item.runId}`)).size, - }, - labels, - warnings: [], - }; -} - -describe('relationship threshold advice', () => { - it('selects the highest-quality threshold candidate when enough labels exist', () => { - const report = buildKtxRelationshipThresholdAdviceReport( - feedback([ - label({ - candidateId: 'orders:orders.customer_id->customers:customers.id', - decision: 'accepted', - score: 0.91, - pkScore: 0.97, - fkScore: 0.91, - }), - label({ - candidateId: 'orders:orders.account_id->accounts:accounts.id', - decision: 'accepted', - score: 0.61, - pkScore: 0.88, - fkScore: 0.61, - }), - label({ - candidateId: 'orders:orders.note_id->notes:notes.id', - decision: 'rejected', - score: 0.21, - pkScore: 0.4, - fkScore: 0.21, - }), - label({ - candidateId: 'orders:orders.region_id->regions:regions.id', - decision: 'rejected', - score: 0.88, - pkScore: 0.9, - fkScore: 0.88, - }), - ]), - { - acceptThresholds: [0.9, 0.85], - reviewThresholds: [0.55], - minTotalLabels: 4, - minAcceptedLabels: 2, - minRejectedLabels: 2, - minAcceptedBandPrecision: 0.75, - minAcceptedOrReviewRecall: 0.75, - minRejectedBandPrecision: 0.75, - }, - ); - - expect(report.status).toBe('ready'); - expect(report.summary).toMatchObject({ - totalLabels: 4, - scoredLabels: 4, - acceptedLabels: 2, - rejectedLabels: 2, - eligibleCandidates: 1, - }); - expect(report.recommended).toMatchObject({ - acceptThreshold: 0.9, - reviewThreshold: 0.55, - eligible: true, - acceptedBandPrecision: 1, - acceptedRecall: 0.5, - acceptedOrReviewRecall: 1, - rejectedBandPrecision: 1, - rejectedRecall: 1, - falseAcceptedRejectedLabels: 0, - falseRejectedAcceptedLabels: 0, - }); - expect(report.candidates.map((candidate) => [candidate.acceptThreshold, candidate.reviewThreshold, candidate.eligible])).toEqual([ - [0.9, 0.55, true], - [0.85, 0.55, false], - ]); - }); - - it('reports insufficient labels without hiding evaluated candidates', () => { - const report = buildKtxRelationshipThresholdAdviceReport( - feedback([ - label({ candidateId: 'orders:orders.customer_id->customers:customers.id', decision: 'accepted', score: 0.91 }), - label({ candidateId: 'orders:orders.note_id->notes:notes.id', decision: 'rejected', score: 0.21 }), - ]), - { - acceptThresholds: [0.9], - reviewThresholds: [0.55], - minTotalLabels: 10, - minAcceptedLabels: 5, - minRejectedLabels: 5, - }, - ); - - expect(report.status).toBe('insufficient_labels'); - expect(report.recommended).toBeNull(); - expect(report.summary).toMatchObject({ - totalLabels: 2, - scoredLabels: 2, - acceptedLabels: 1, - rejectedLabels: 1, - eligibleCandidates: 1, - }); - expect(report.reasons).toEqual([ - 'Need at least 10 scored labels; found 2.', - 'Need at least 5 accepted labels; found 1.', - 'Need at least 5 rejected labels; found 1.', - ]); - expect(report.candidates).toHaveLength(1); - }); - - it('reports no eligible thresholds when label counts pass but quality gates fail', () => { - const report = buildKtxRelationshipThresholdAdviceReport( - feedback([ - label({ candidateId: 'a', decision: 'accepted', score: 0.92 }), - label({ candidateId: 'b', decision: 'accepted', score: 0.58 }), - label({ candidateId: 'c', decision: 'rejected', score: 0.91 }), - label({ candidateId: 'd', decision: 'rejected', score: 0.2 }), - ]), - { - acceptThresholds: [0.9], - reviewThresholds: [0.55], - minTotalLabels: 4, - minAcceptedLabels: 2, - minRejectedLabels: 2, - minAcceptedBandPrecision: 0.9, - }, - ); - - expect(report.status).toBe('no_eligible_thresholds'); - expect(report.recommended).toBeNull(); - expect(report.reasons).toEqual(['No threshold candidate met the precision and recall gates.']); - expect(report.candidates[0]).toMatchObject({ - acceptThreshold: 0.9, - reviewThreshold: 0.55, - eligible: false, - acceptedBandPrecision: 0.5, - }); - }); - - it('wraps the feedback exporter and preserves warnings', async () => { - const project = { projectDir: '/tmp/ktx-project' } as KtxLocalProject; - const exportLocalRelationshipFeedbackLabels = vi.fn(async () => ({ - ...feedback([]), - warnings: [ - { - path: 'raw-sources/broken/live-database/sync/enrichment/relationship-review-decisions.json', - message: 'Unexpected token', - }, - ], - })); - - const report = await adviseLocalRelationshipFeedbackThresholds(project, { - connectionId: 'warehouse', - exportLocalRelationshipFeedbackLabels, - minTotalLabels: 1, - }); - - expect(exportLocalRelationshipFeedbackLabels).toHaveBeenCalledWith(project, { - connectionId: 'warehouse', - decision: 'all', - }); - expect(report.warnings).toEqual([ - { - path: 'raw-sources/broken/live-database/sync/enrichment/relationship-review-decisions.json', - message: 'Unexpected token', - }, - ]); - }); - - it('formats a stable human-readable report', () => { - const report = buildKtxRelationshipThresholdAdviceReport( - feedback([ - label({ candidateId: 'orders:orders.customer_id->customers:customers.id', decision: 'accepted', score: 0.91 }), - label({ candidateId: 'orders:orders.account_id->accounts:accounts.id', decision: 'accepted', score: 0.61 }), - label({ candidateId: 'orders:orders.note_id->notes:notes.id', decision: 'rejected', score: 0.21 }), - label({ candidateId: 'orders:orders.region_id->regions:regions.id', decision: 'rejected', score: 0.88 }), - ]), - { - acceptThresholds: [0.9], - reviewThresholds: [0.55], - minTotalLabels: 4, - minAcceptedLabels: 2, - minRejectedLabels: 2, - minAcceptedBandPrecision: 0.75, - }, - ); - - expect(formatKtxRelationshipThresholdAdviceMarkdown(report)).toContain('KTX relationship threshold advice'); - expect(formatKtxRelationshipThresholdAdviceMarkdown(report)).toContain('Status: ready'); - expect(formatKtxRelationshipThresholdAdviceMarkdown(report)).toContain('Recommended: accept=0.90 review=0.55'); - expect(formatKtxRelationshipThresholdAdviceMarkdown(report)).toContain('acceptedPrecision=1.000'); - }); -}); diff --git a/packages/cli/src/context/scan/relationship-threshold-advice.ts b/packages/cli/src/context/scan/relationship-threshold-advice.ts deleted file mode 100644 index d94abc2e..00000000 --- a/packages/cli/src/context/scan/relationship-threshold-advice.ts +++ /dev/null @@ -1,335 +0,0 @@ -import type { KtxLocalProject } from '../project/index.js'; -import { - exportLocalRelationshipFeedbackLabels, - type ExportLocalRelationshipFeedbackLabelsInput, - type ExportLocalRelationshipFeedbackLabelsResult, - type KtxRelationshipFeedbackExportWarning, - type KtxRelationshipFeedbackLabel, -} from './relationship-feedback-export.js'; -import type { KtxResolvedRelationshipStatus } from './relationship-graph-resolver.js'; - -const DEFAULT_ACCEPT_THRESHOLDS = [0.95, 0.9, 0.85, 0.8, 0.75] as const; -const DEFAULT_REVIEW_THRESHOLDS = [0.65, 0.6, 0.55, 0.5, 0.45] as const; - -type AdvicePredictedStatus = KtxResolvedRelationshipStatus; -export type KtxRelationshipThresholdAdviceStatus = 'ready' | 'insufficient_labels' | 'no_eligible_thresholds'; - -export interface BuildKtxRelationshipThresholdAdviceReportInput { - acceptThresholds?: readonly number[]; - reviewThresholds?: readonly number[]; - minTotalLabels?: number; - minAcceptedLabels?: number; - minRejectedLabels?: number; - minAcceptedBandPrecision?: number; - minAcceptedOrReviewRecall?: number; - minRejectedBandPrecision?: number; -} - -export interface AdviseLocalRelationshipFeedbackThresholdsInput - extends Omit, - BuildKtxRelationshipThresholdAdviceReportInput { - exportLocalRelationshipFeedbackLabels?: typeof exportLocalRelationshipFeedbackLabels; -} - -export interface KtxRelationshipThresholdAdviceCandidate { - acceptThreshold: number; - reviewThreshold: number; - eligible: boolean; - predictedAccepted: number; - predictedReview: number; - predictedRejected: number; - acceptedBandPrecision: number | null; - acceptedRecall: number | null; - acceptedOrReviewRecall: number | null; - rejectedBandPrecision: number | null; - rejectedRecall: number | null; - falseAcceptedRejectedLabels: number; - falseRejectedAcceptedLabels: number; -} - -export interface KtxRelationshipThresholdAdviceReport { - generatedAt: string; - filters: ExportLocalRelationshipFeedbackLabelsResult['filters']; - status: KtxRelationshipThresholdAdviceStatus; - gates: { - minTotalLabels: number; - minAcceptedLabels: number; - minRejectedLabels: number; - minAcceptedBandPrecision: number; - minAcceptedOrReviewRecall: number; - minRejectedBandPrecision: number; - }; - summary: { - totalLabels: number; - scoredLabels: number; - unscoredLabels: number; - acceptedLabels: number; - rejectedLabels: number; - evaluatedCandidates: number; - eligibleCandidates: number; - }; - recommended: KtxRelationshipThresholdAdviceCandidate | null; - candidates: KtxRelationshipThresholdAdviceCandidate[]; - reasons: string[]; - warnings: KtxRelationshipFeedbackExportWarning[]; -} - -interface ResolvedAdviceInput { - acceptThresholds: number[]; - reviewThresholds: number[]; - minTotalLabels: number; - minAcceptedLabels: number; - minRejectedLabels: number; - minAcceptedBandPrecision: number; - minAcceptedOrReviewRecall: number; - minRejectedBandPrecision: number; -} - -function resolveInput(input: BuildKtxRelationshipThresholdAdviceReportInput): ResolvedAdviceInput { - return { - acceptThresholds: [...(input.acceptThresholds ?? DEFAULT_ACCEPT_THRESHOLDS)].sort((left, right) => right - left), - reviewThresholds: [...(input.reviewThresholds ?? DEFAULT_REVIEW_THRESHOLDS)].sort((left, right) => right - left), - minTotalLabels: input.minTotalLabels ?? 20, - minAcceptedLabels: input.minAcceptedLabels ?? 5, - minRejectedLabels: input.minRejectedLabels ?? 5, - minAcceptedBandPrecision: input.minAcceptedBandPrecision ?? 0.9, - minAcceptedOrReviewRecall: input.minAcceptedOrReviewRecall ?? 0.8, - minRejectedBandPrecision: input.minRejectedBandPrecision ?? 0.8, - }; -} - -function roundMetric(value: number): number { - return Math.round(value * 1000) / 1000; -} - -function ratio(numerator: number, denominator: number): number | null { - return denominator === 0 ? null : roundMetric(numerator / denominator); -} - -function prediction(score: number, acceptThreshold: number, reviewThreshold: number): AdvicePredictedStatus { - if (score >= acceptThreshold) { - return 'accepted'; - } - if (score >= reviewThreshold) { - return 'review'; - } - return 'rejected'; -} - -function isMetricAtLeast(value: number | null, minimum: number): boolean { - return value !== null && value >= minimum; -} - -function thresholdCandidate( - labels: readonly KtxRelationshipFeedbackLabel[], - acceptThreshold: number, - reviewThreshold: number, - gates: ResolvedAdviceInput, -): KtxRelationshipThresholdAdviceCandidate { - const scored = labels.filter((label): label is KtxRelationshipFeedbackLabel & { score: number } => label.score !== null); - const acceptedLabels = scored.filter((label) => label.decision === 'accepted'); - const rejectedLabels = scored.filter((label) => label.decision === 'rejected'); - const predictions = scored.map((label) => ({ - label, - predictedStatus: prediction(label.score, acceptThreshold, reviewThreshold), - })); - const predictedAccepted = predictions.filter((item) => item.predictedStatus === 'accepted'); - const predictedReview = predictions.filter((item) => item.predictedStatus === 'review'); - const predictedRejected = predictions.filter((item) => item.predictedStatus === 'rejected'); - const acceptedBandPrecision = ratio( - predictedAccepted.filter((item) => item.label.decision === 'accepted').length, - predictedAccepted.length, - ); - const acceptedOrReviewRecall = ratio( - predictions.filter((item) => item.label.decision === 'accepted' && item.predictedStatus !== 'rejected').length, - acceptedLabels.length, - ); - const rejectedBandPrecision = ratio( - predictedRejected.filter((item) => item.label.decision === 'rejected').length, - predictedRejected.length, - ); - - return { - acceptThreshold, - reviewThreshold, - eligible: - predictedAccepted.length > 0 && - predictedRejected.length > 0 && - isMetricAtLeast(acceptedBandPrecision, gates.minAcceptedBandPrecision) && - isMetricAtLeast(acceptedOrReviewRecall, gates.minAcceptedOrReviewRecall) && - isMetricAtLeast(rejectedBandPrecision, gates.minRejectedBandPrecision), - predictedAccepted: predictedAccepted.length, - predictedReview: predictedReview.length, - predictedRejected: predictedRejected.length, - acceptedBandPrecision, - acceptedRecall: ratio( - predictedAccepted.filter((item) => item.label.decision === 'accepted').length, - acceptedLabels.length, - ), - acceptedOrReviewRecall, - rejectedBandPrecision, - rejectedRecall: ratio( - predictions.filter((item) => item.label.decision === 'rejected' && item.predictedStatus !== 'accepted').length, - rejectedLabels.length, - ), - falseAcceptedRejectedLabels: predictedAccepted.filter((item) => item.label.decision === 'rejected').length, - falseRejectedAcceptedLabels: predictedRejected.filter((item) => item.label.decision === 'accepted').length, - }; -} - -function metricRank(value: number | null): number { - return value ?? -1; -} - -function sortCandidates( - candidates: readonly KtxRelationshipThresholdAdviceCandidate[], -): KtxRelationshipThresholdAdviceCandidate[] { - return [...candidates].sort( - (left, right) => - Number(right.eligible) - Number(left.eligible) || - metricRank(right.acceptedBandPrecision) - metricRank(left.acceptedBandPrecision) || - metricRank(right.acceptedOrReviewRecall) - metricRank(left.acceptedOrReviewRecall) || - metricRank(right.rejectedBandPrecision) - metricRank(left.rejectedBandPrecision) || - right.acceptThreshold - left.acceptThreshold || - right.reviewThreshold - left.reviewThreshold, - ); -} - -function labelGateReasons(labels: readonly KtxRelationshipFeedbackLabel[], gates: ResolvedAdviceInput): string[] { - const scored = labels.filter((label) => label.score !== null); - const accepted = scored.filter((label) => label.decision === 'accepted'); - const rejected = scored.filter((label) => label.decision === 'rejected'); - const reasons: string[] = []; - if (scored.length < gates.minTotalLabels) { - reasons.push(`Need at least ${gates.minTotalLabels} scored labels; found ${scored.length}.`); - } - if (accepted.length < gates.minAcceptedLabels) { - reasons.push(`Need at least ${gates.minAcceptedLabels} accepted labels; found ${accepted.length}.`); - } - if (rejected.length < gates.minRejectedLabels) { - reasons.push(`Need at least ${gates.minRejectedLabels} rejected labels; found ${rejected.length}.`); - } - return reasons; -} - -export function buildKtxRelationshipThresholdAdviceReport( - feedback: ExportLocalRelationshipFeedbackLabelsResult, - input: BuildKtxRelationshipThresholdAdviceReportInput = {}, -): KtxRelationshipThresholdAdviceReport { - const gates = resolveInput(input); - const scored = feedback.labels.filter((label) => label.score !== null); - const acceptedLabels = scored.filter((label) => label.decision === 'accepted'); - const rejectedLabels = scored.filter((label) => label.decision === 'rejected'); - const candidates = sortCandidates( - gates.acceptThresholds.flatMap((acceptThreshold) => - gates.reviewThresholds.flatMap((reviewThreshold) => - acceptThreshold > reviewThreshold - ? [thresholdCandidate(feedback.labels, acceptThreshold, reviewThreshold, gates)] - : [], - ), - ), - ); - const labelReasons = labelGateReasons(feedback.labels, gates); - const eligibleCandidates = candidates.filter((candidate) => candidate.eligible); - const status: KtxRelationshipThresholdAdviceStatus = - labelReasons.length > 0 ? 'insufficient_labels' : eligibleCandidates.length > 0 ? 'ready' : 'no_eligible_thresholds'; - const reasons = - status === 'insufficient_labels' - ? labelReasons - : status === 'no_eligible_thresholds' - ? ['No threshold candidate met the precision and recall gates.'] - : []; - - return { - generatedAt: feedback.generatedAt, - filters: feedback.filters, - status, - gates: { - minTotalLabels: gates.minTotalLabels, - minAcceptedLabels: gates.minAcceptedLabels, - minRejectedLabels: gates.minRejectedLabels, - minAcceptedBandPrecision: gates.minAcceptedBandPrecision, - minAcceptedOrReviewRecall: gates.minAcceptedOrReviewRecall, - minRejectedBandPrecision: gates.minRejectedBandPrecision, - }, - summary: { - totalLabels: feedback.labels.length, - scoredLabels: scored.length, - unscoredLabels: feedback.labels.length - scored.length, - acceptedLabels: acceptedLabels.length, - rejectedLabels: rejectedLabels.length, - evaluatedCandidates: candidates.length, - eligibleCandidates: eligibleCandidates.length, - }, - recommended: status === 'ready' ? eligibleCandidates[0] ?? null : null, - candidates, - reasons, - warnings: [...feedback.warnings], - }; -} - -export async function adviseLocalRelationshipFeedbackThresholds( - project: KtxLocalProject, - input: AdviseLocalRelationshipFeedbackThresholdsInput = {}, -): Promise { - const exporter = input.exportLocalRelationshipFeedbackLabels ?? exportLocalRelationshipFeedbackLabels; - const feedback = await exporter(project, { - connectionId: input.connectionId, - decision: 'all', - }); - return buildKtxRelationshipThresholdAdviceReport(feedback, input); -} - -function formatMetric(value: number | null): string { - return value === null ? 'n/a' : value.toFixed(3); -} - -function candidateLine(candidate: KtxRelationshipThresholdAdviceCandidate): string { - return [ - `accept=${candidate.acceptThreshold.toFixed(2)}`, - `review=${candidate.reviewThreshold.toFixed(2)}`, - `eligible=${candidate.eligible ? 'yes' : 'no'}`, - `acceptedPrecision=${formatMetric(candidate.acceptedBandPrecision)}`, - `acceptedRecall=${formatMetric(candidate.acceptedRecall)}`, - `acceptedOrReviewRecall=${formatMetric(candidate.acceptedOrReviewRecall)}`, - `rejectedPrecision=${formatMetric(candidate.rejectedBandPrecision)}`, - `rejectedRecall=${formatMetric(candidate.rejectedRecall)}`, - `falseAcceptedRejected=${candidate.falseAcceptedRejectedLabels}`, - `falseRejectedAccepted=${candidate.falseRejectedAcceptedLabels}`, - ].join(' '); -} - -export function formatKtxRelationshipThresholdAdviceMarkdown(report: KtxRelationshipThresholdAdviceReport): string { - const lines = [ - 'KTX relationship threshold advice', - `Generated: ${report.generatedAt}`, - `Filter connection: ${report.filters.connectionId ?? 'all'}`, - `Status: ${report.status}`, - `Labels: total=${report.summary.totalLabels} scored=${report.summary.scoredLabels} accepted=${report.summary.acceptedLabels} rejected=${report.summary.rejectedLabels}`, - `Gates: minTotal=${report.gates.minTotalLabels} minAccepted=${report.gates.minAcceptedLabels} minRejected=${report.gates.minRejectedLabels} acceptedPrecision=${report.gates.minAcceptedBandPrecision.toFixed(3)} acceptedOrReviewRecall=${report.gates.minAcceptedOrReviewRecall.toFixed(3)} rejectedPrecision=${report.gates.minRejectedBandPrecision.toFixed(3)}`, - `Evaluated candidates: ${report.summary.evaluatedCandidates}`, - `Eligible candidates: ${report.summary.eligibleCandidates}`, - `Recommended: ${ - report.recommended - ? `accept=${report.recommended.acceptThreshold.toFixed(2)} review=${report.recommended.reviewThreshold.toFixed(2)}` - : 'none' - }`, - ]; - - if (report.reasons.length > 0) { - lines.push('', 'Reasons', ...report.reasons.map((reason) => ` - ${reason}`)); - } - - if (report.candidates.length > 0) { - lines.push('', 'Top candidates', ...report.candidates.slice(0, 5).map((candidate) => ` - ${candidateLine(candidate)}`)); - } - - if (report.warnings.length > 0) { - lines.push('', 'Warnings'); - for (const warning of report.warnings.slice(0, 5)) { - lines.push(` - ${warning.path}: ${warning.message}`); - } - } - - return `${lines.join('\n')}\n`; -} diff --git a/packages/cli/src/context/scan/relationship-validation.ts b/packages/cli/src/context/scan/relationship-validation.ts index b735afb2..63d7328a 100644 --- a/packages/cli/src/context/scan/relationship-validation.ts +++ b/packages/cli/src/context/scan/relationship-validation.ts @@ -9,9 +9,9 @@ import { } from './relationship-profiling.js'; import type { KtxConnectionDriver, KtxQueryResult, KtxScanContext } from './types.js'; -export type KtxValidatedRelationshipStatus = 'accepted' | 'review' | 'rejected'; +type KtxValidatedRelationshipStatus = 'accepted' | 'review' | 'rejected'; -export interface KtxRelationshipValidationSettings { +interface KtxRelationshipValidationSettings { acceptThreshold: number; reviewThreshold: number; minTargetUniqueness: number; @@ -22,7 +22,7 @@ export interface KtxRelationshipValidationSettings { validationBudget?: KtxRelationshipValidationBudget; } -export interface KtxRelationshipValidationEvidence { +interface KtxRelationshipValidationEvidence { targetUniqueness: number; sourceCoverage: number; violationCount: number; diff --git a/packages/cli/src/context/scan/type-normalization.ts b/packages/cli/src/context/scan/type-normalization.ts index 5c696339..38d15ebc 100644 --- a/packages/cli/src/context/scan/type-normalization.ts +++ b/packages/cli/src/context/scan/type-normalization.ts @@ -1,5 +1,6 @@ import type { KtxSchemaDimensionType } from './types.js'; +/** @internal */ export interface KtxColumnTypeMapping { normalizedType: string; dimensionType: KtxSchemaDimensionType; @@ -24,6 +25,7 @@ export function inferKtxDimensionType(nativeType: string): KtxSchemaDimensionTyp return 'string'; } +/** @internal */ export function ktxColumnTypeMappingFromNative(nativeType: string): KtxColumnTypeMapping { return { normalizedType: normalizeKtxNativeType(nativeType), diff --git a/packages/cli/src/context/scan/types.ts b/packages/cli/src/context/scan/types.ts index bc8959f5..f4299e86 100644 --- a/packages/cli/src/context/scan/types.ts +++ b/packages/cli/src/context/scan/types.ts @@ -42,13 +42,13 @@ export function createKtxConnectorCapabilities( }; } -export interface KtxSchemaScope { +interface KtxSchemaScope { catalogs?: string[]; schemas?: string[]; datasets?: string[]; } -export type KtxSchemaTableKind = 'table' | 'view' | 'external' | 'event_stream'; +type KtxSchemaTableKind = 'table' | 'view' | 'external' | 'event_stream'; export type KtxSchemaDimensionType = 'time' | 'string' | 'number' | 'boolean'; @@ -91,17 +91,17 @@ export interface KtxSchemaSnapshot { metadata: Record; } -export interface KtxCredentialEnvReference { +interface KtxCredentialEnvReference { kind: 'env'; name: string; } -export interface KtxCredentialFileReference { +interface KtxCredentialFileReference { kind: 'file'; path: string; } -export interface KtxResolvedCredentialEnvelope { +interface KtxResolvedCredentialEnvelope { kind: 'resolved'; source: 'standalone' | 'host'; values: Record; @@ -113,13 +113,14 @@ export type KtxCredentialEnvelope = | KtxCredentialFileReference | KtxResolvedCredentialEnvelope; +/** @internal */ export interface KtxNetworkEndpoint { host: string; port: number; close?: () => Promise; } -export interface KtxNetworkTunnelRequest> { +interface KtxNetworkTunnelRequest> { connectionId: string; driver: KtxConnectionDriver; host: string; @@ -127,6 +128,7 @@ export interface KtxNetworkTunnelRequest> connection: TConnection; } +/** @internal */ export interface KtxNetworkTunnelPort> { resolveEndpoint(input: KtxNetworkTunnelRequest): Promise; } @@ -211,6 +213,7 @@ export interface KtxColumnStatsResult { distinctCount: number | null; } +/** @internal */ export interface KtxEventTypeDiscoveryInput { connectionId: string; table: KtxTableRef; @@ -220,11 +223,13 @@ export interface KtxEventTypeDiscoveryInput { lookbackDays?: number; } +/** @internal */ export interface KtxEventTypeDiscovery { value: string; count: number; } +/** @internal */ export interface KtxEventPropertyDiscoveryInput { connectionId: string; table: KtxTableRef; @@ -234,11 +239,13 @@ export interface KtxEventPropertyDiscoveryInput { lookbackDays?: number; } +/** @internal */ export interface KtxEventPropertyDiscovery { key: string; count: number; } +/** @internal */ export interface KtxEventPropertyValuesInput { connectionId: string; table: KtxTableRef; @@ -249,11 +256,13 @@ export interface KtxEventPropertyValuesInput { lookbackDays?: number; } +/** @internal */ export interface KtxEventPropertyValuesResult { values: string[]; cardinality: number; } +/** @internal */ export interface KtxEventStreamDiscoveryPort { listEventTypes(input: KtxEventTypeDiscoveryInput, ctx: KtxScanContext): Promise; listPropertyKeys(input: KtxEventPropertyDiscoveryInput, ctx: KtxScanContext): Promise; @@ -283,7 +292,7 @@ export interface KtxTableListEntry { kind: 'table' | 'view'; } -export interface KtxConnectorTestResult { +interface KtxConnectorTestResult { success: boolean; error?: string; } @@ -308,7 +317,7 @@ export interface KtxEmbeddingPort { embedBatch(texts: string[]): Promise; } -export interface KtxStructuralSyncStats { +interface KtxStructuralSyncStats { tablesCreated: number; tablesUpdated: number; tablesDeleted: number; @@ -317,7 +326,7 @@ export interface KtxStructuralSyncStats { columnsDeleted: number; } -export interface KtxScanDiffSummary { +interface KtxScanDiffSummary { tablesAdded: number; tablesModified: number; tablesDeleted: number; @@ -327,14 +336,14 @@ export interface KtxScanDiffSummary { columnsDeleted: number; } -export interface KtxScanArtifactPaths { +interface KtxScanArtifactPaths { rawSourcesDir: string | null; reportPath: string | null; manifestShards: string[]; enrichmentArtifacts: string[]; } -export type KtxScanWarningCode = +type KtxScanWarningCode = | 'connector_capability_missing' | 'sampling_failed' | 'statistics_failed' diff --git a/packages/cli/src/context/scan/warehouse-catalog.test.ts b/packages/cli/src/context/scan/warehouse-catalog.test.ts index 7cabc1df..6ef1f03a 100644 --- a/packages/cli/src/context/scan/warehouse-catalog.test.ts +++ b/packages/cli/src/context/scan/warehouse-catalog.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { WarehouseCatalogService } from './warehouse-catalog.js'; describe('WarehouseCatalogService', () => { diff --git a/packages/cli/src/context/scan/warehouse-catalog.ts b/packages/cli/src/context/scan/warehouse-catalog.ts index 8cbe324d..2f360eeb 100644 --- a/packages/cli/src/context/scan/warehouse-catalog.ts +++ b/packages/cli/src/context/scan/warehouse-catalog.ts @@ -1,5 +1,5 @@ -import { getDialectForDriver } from '../connections/index.js'; -import type { KtxFileStorePort } from '../core/index.js'; +import { getDialectForDriver } from '../../context/connections/dialects.js'; +import type { KtxFileStorePort } from '../../context/core/file-store.js'; import type { KtxConnectionDriver, KtxSchemaColumn, diff --git a/packages/cli/src/context/search/backend-conformance.test.ts b/packages/cli/src/context/search/backend-conformance.test-utils.test.ts similarity index 99% rename from packages/cli/src/context/search/backend-conformance.test.ts rename to packages/cli/src/context/search/backend-conformance.test-utils.test.ts index 31519c8b..c9ecebb7 100644 --- a/packages/cli/src/context/search/backend-conformance.test.ts +++ b/packages/cli/src/context/search/backend-conformance.test-utils.test.ts @@ -2,9 +2,9 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, it } from 'vitest'; -import { SqliteContextEvidenceStore } from '../ingest/context-evidence/index.js'; +import { SqliteContextEvidenceStore } from '../ingest/context-evidence/sqlite-context-evidence-store.js'; import type { JsonValue } from '../ingest/ports.js'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../project/project.js'; import { type LocalSlSourceSearchResult, searchLocalSlSources, writeLocalSlSource } from '../sl/local-sl.js'; import type { ContextEvidenceSearchResult } from '../tools/context-evidence-tool-store.js'; import { @@ -16,7 +16,7 @@ import { assertSearchBackendCapabilities, assertSearchBackendConformanceCase, type SearchBackendConformanceResult, -} from './backend-conformance.js'; +} from './backend-conformance.test-utils.js'; import type { SearchBackendCapabilities } from './types.js'; const SQLITE_SEARCH_CAPABILITIES = { diff --git a/packages/cli/src/context/search/backend-conformance.ts b/packages/cli/src/context/search/backend-conformance.test-utils.ts similarity index 100% rename from packages/cli/src/context/search/backend-conformance.ts rename to packages/cli/src/context/search/backend-conformance.test-utils.ts diff --git a/packages/cli/src/context/search/discover.test.ts b/packages/cli/src/context/search/discover.test.ts index 7f9df413..931de2be 100644 --- a/packages/cli/src/context/search/discover.test.ts +++ b/packages/cli/src/context/search/discover.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { writeLocalKnowledgePage } from '../wiki/local-knowledge.js'; import { createKtxDiscoverDataService } from './discover.js'; diff --git a/packages/cli/src/context/search/discover.ts b/packages/cli/src/context/search/discover.ts index 53694f6a..b3456459 100644 --- a/packages/cli/src/context/search/discover.ts +++ b/packages/cli/src/context/search/discover.ts @@ -1,12 +1,14 @@ -import type { KtxEmbeddingPort } from '../core/index.js'; -import type { KtxLocalProject } from '../project/index.js'; -import type { KtxScanReport, KtxSchemaColumn, KtxSchemaTable, KtxTableRef } from '../scan/index.js'; -import { DEFAULT_PRIORITY, loadLocalSlSourceRecords, resolveDescription } from '../sl/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import type { KtxScanReport, KtxSchemaColumn, KtxSchemaTable, KtxTableRef } from '../../context/scan/types.js'; +import { DEFAULT_PRIORITY, resolveDescription } from '../../context/sl/descriptions.js'; +import { loadLocalSlSourceRecords } from '../../context/sl/local-sl.js'; import { readLocalKnowledgePage, searchLocalKnowledgePages } from '../wiki/local-knowledge.js'; -import { HybridSearchCore, type FusedSearchCandidate, type SearchCandidateGenerator } from './index.js'; +import { HybridSearchCore } from '../../context/search/hybrid-search-core.js'; +import type { FusedSearchCandidate, SearchCandidateGenerator } from '../../context/search/types.js'; -export type KtxDiscoverDataKind = 'wiki' | 'sl_source' | 'sl_measure' | 'sl_dimension' | 'table' | 'column'; -export type KtxDiscoverDataMatchedOn = 'name' | 'display' | 'description' | 'comment' | 'expr' | 'sample_value' | 'body'; +type KtxDiscoverDataKind = 'wiki' | 'sl_source' | 'sl_measure' | 'sl_dimension' | 'table' | 'column'; +type KtxDiscoverDataMatchedOn = 'name' | 'display' | 'description' | 'comment' | 'expr' | 'sample_value' | 'body'; export interface KtxDiscoverDataInput { query: string; @@ -15,7 +17,7 @@ export interface KtxDiscoverDataInput { limit?: number; } -export interface KtxDiscoverDataRef { +interface KtxDiscoverDataRef { kind: KtxDiscoverDataKind; id: string; score: number; diff --git a/packages/cli/src/context/search/index.ts b/packages/cli/src/context/search/index.ts deleted file mode 100644 index 9cec3602..00000000 --- a/packages/cli/src/context/search/index.ts +++ /dev/null @@ -1,44 +0,0 @@ -export type { - AssertSearchBackendCapabilitiesInput, - AssertSearchBackendConformanceCaseInput, - ExpectedSearchBackendConformanceLane, - SearchBackendConformanceDictionaryMatch, - SearchBackendConformanceLane, - SearchBackendConformanceResult, -} from './backend-conformance.js'; -export { - assertSearchBackendCapabilities, - assertSearchBackendConformanceCase, -} from './backend-conformance.js'; -export { createKtxDiscoverDataService } from './discover.js'; -export type { - KtxDiscoverDataInput, - KtxDiscoverDataKind, - KtxDiscoverDataMatchedOn, - KtxDiscoverDataRef, - KtxDiscoverDataResponse, - KtxDiscoverDataServiceOptions, -} from './discover.js'; -export { HybridSearchCore } from './hybrid-search-core.js'; -export { defaultLaneCandidatePoolLimit, normalizeSearchQuery } from './query.js'; -export { - compareFusedSearchCandidates, - DEFAULT_RRF_K, - DEFAULT_SEARCH_LANE_WEIGHTS, - rrfContribution, -} from './rrf.js'; -export type { - FusedSearchCandidate, - HybridSearchOptions, - HybridSearchResult, - NormalizedSearchQuery, - SearchBackendCapabilities, - SearchCandidate, - SearchCandidateGenerator, - SearchCandidateGeneratorArgs, - SearchLaneBreakdown, - SearchLaneName, - SearchLaneResult, - SearchLaneStatus, - SearchResultHydrator, -} from './types.js'; diff --git a/packages/cli/src/context/search/pglite-owner-process.test.ts b/packages/cli/src/context/search/pglite-owner-process.test.ts index 22da646a..3a15eea9 100644 --- a/packages/cli/src/context/search/pglite-owner-process.test.ts +++ b/packages/cli/src/context/search/pglite-owner-process.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { Client } from 'pg'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { assertSearchBackendConformanceCase } from './index.js'; +import { assertSearchBackendConformanceCase } from '../../context/search/backend-conformance.test-utils.js'; import { KtxPGliteOwnerProcess } from './pglite-owner-process.js'; async function allocatePort(): Promise { diff --git a/packages/cli/src/context/search/pglite-runtime-boundary.test.ts b/packages/cli/src/context/search/pglite-runtime-boundary.test.ts index 8b0d123b..feb7443d 100644 --- a/packages/cli/src/context/search/pglite-runtime-boundary.test.ts +++ b/packages/cli/src/context/search/pglite-runtime-boundary.test.ts @@ -35,16 +35,6 @@ describe('PGlite hybrid search runtime boundary', () => { expect(packageExportKeys.filter((key) => key.toLowerCase().includes('pglite'))).toEqual([]); - const publicExportFiles = [ - 'packages/cli/src/context/index.ts', - 'packages/cli/src/context/search/index.ts', - 'packages/cli/src/context/sl/index.ts', - ]; - - for (const relativePath of publicExportFiles) { - expect(readKtxFile(relativePath), relativePath).not.toMatch(/pglite/i); - } - const productionRoutingFiles = [ 'packages/cli/src/sl.ts', 'packages/cli/src/knowledge.ts', diff --git a/packages/cli/src/context/search/pglite-spike.test.ts b/packages/cli/src/context/search/pglite-spike.test.ts index e8dff54e..470000da 100644 --- a/packages/cli/src/context/search/pglite-spike.test.ts +++ b/packages/cli/src/context/search/pglite-spike.test.ts @@ -5,11 +5,8 @@ import { PGlite, type PGliteInterface } from '@electric-sql/pglite'; import { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm'; import { vector } from '@electric-sql/pglite/vector'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { - assertSearchBackendCapabilities, - assertSearchBackendConformanceCase, - type SearchBackendCapabilities, -} from './index.js'; +import { assertSearchBackendCapabilities, assertSearchBackendConformanceCase } from '../../context/search/backend-conformance.test-utils.js'; +import type { SearchBackendCapabilities } from '../../context/search/types.js'; type PGliteDb = PGliteInterface; diff --git a/packages/cli/src/context/search/types.ts b/packages/cli/src/context/search/types.ts index 658961f0..8080ddfa 100644 --- a/packages/cli/src/context/search/types.ts +++ b/packages/cli/src/context/search/types.ts @@ -1,5 +1,6 @@ export type SearchLaneName = 'lexical' | 'semantic' | 'dictionary' | 'token' | string; +/** @internal */ export type SearchLaneStatus = 'available' | 'skipped' | 'failed'; export interface NormalizedSearchQuery { @@ -16,7 +17,7 @@ export interface SearchCandidate { evidence?: unknown; } -export interface SearchCandidateGeneratorArgs { +interface SearchCandidateGeneratorArgs { queryText: string; normalizedQuery: NormalizedSearchQuery; finalLimit: number; @@ -64,9 +65,6 @@ export interface FusedSearchCandidate { evidenceByLane: Record; } -export interface SearchResultHydrator { - hydrate(candidates: FusedSearchCandidate[]): Promise; -} export interface HybridSearchResult { query: NormalizedSearchQuery; @@ -76,6 +74,7 @@ export interface HybridSearchResult { lanes: SearchLaneBreakdown[]; } +/** @internal */ export interface SearchBackendCapabilities { fts: boolean; vector: boolean; diff --git a/packages/cli/src/context/skills/index.ts b/packages/cli/src/context/skills/index.ts deleted file mode 100644 index 13f06853..00000000 --- a/packages/cli/src/context/skills/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export type { FrontmatterFields, SkillCaller, SkillMetadata, SkillsRegistryServiceOptions } from './skills-registry.service.js'; -export { SkillsRegistryService } from './skills-registry.service.js'; diff --git a/packages/cli/src/context/skills/skills-registry.service.ts b/packages/cli/src/context/skills/skills-registry.service.ts index cd33e6d8..4e8bb5cb 100644 --- a/packages/cli/src/context/skills/skills-registry.service.ts +++ b/packages/cli/src/context/skills/skills-registry.service.ts @@ -1,6 +1,6 @@ import { readFile, readdir, stat } from 'node:fs/promises'; import { join } from 'node:path'; -import { noopLogger, type KtxLogger } from '../core/index.js'; +import { noopLogger, type KtxLogger } from '../../context/core/config.js'; export type SkillCaller = 'research' | 'memory_agent'; diff --git a/packages/cli/src/context/sl/dictionary-search.test.ts b/packages/cli/src/context/sl/dictionary-search.test.ts index 7c3e2d1f..1838f0d9 100644 --- a/packages/cli/src/context/sl/dictionary-search.test.ts +++ b/packages/cli/src/context/sl/dictionary-search.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { createKtxDictionarySearchService } from './dictionary-search.js'; describe('createKtxDictionarySearchService', () => { diff --git a/packages/cli/src/context/sl/dictionary-search.ts b/packages/cli/src/context/sl/dictionary-search.ts index 041b828d..14899482 100644 --- a/packages/cli/src/context/sl/dictionary-search.ts +++ b/packages/cli/src/context/sl/dictionary-search.ts @@ -1,15 +1,15 @@ -import type { KtxLocalProject } from '../project/index.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import { loadLatestSlDictionaryEntries, type SlDictionaryEntry } from './sl-dictionary-profile.js'; -export type KtxDictionarySearchStatus = 'ready' | 'no_profile_artifact' | 'no_candidate_columns'; -export type KtxDictionarySearchMissReason = 'no_profile_artifact' | 'no_candidate_columns' | 'value_not_in_sample'; +type KtxDictionarySearchStatus = 'ready' | 'no_profile_artifact' | 'no_candidate_columns'; +type KtxDictionarySearchMissReason = 'no_profile_artifact' | 'no_candidate_columns' | 'value_not_in_sample'; export interface KtxDictionarySearchInput { values: string[]; connectionId?: string; } -export interface KtxDictionarySearchCoverage { +interface KtxDictionarySearchCoverage { sampledRows: number | null; valuesPerColumn: number | null; profiledColumns: number; @@ -17,13 +17,13 @@ export interface KtxDictionarySearchCoverage { profiledAt: string | null; } -export interface KtxDictionarySearchSearchedConnection { +interface KtxDictionarySearchSearchedConnection { connectionId: string; coverage: KtxDictionarySearchCoverage; status: KtxDictionarySearchStatus; } -export interface KtxDictionarySearchMatch { +interface KtxDictionarySearchMatch { connectionId: string; sourceName: string; columnName: string; @@ -31,12 +31,12 @@ export interface KtxDictionarySearchMatch { cardinality: number | null; } -export interface KtxDictionarySearchMiss { +interface KtxDictionarySearchMiss { connectionId: string; reason: KtxDictionarySearchMissReason; } -export interface KtxDictionarySearchValueResult { +interface KtxDictionarySearchValueResult { value: string; matches: KtxDictionarySearchMatch[]; misses: KtxDictionarySearchMiss[]; diff --git a/packages/cli/src/context/sl/index.ts b/packages/cli/src/context/sl/index.ts deleted file mode 100644 index 600a5a93..00000000 --- a/packages/cli/src/context/sl/index.ts +++ /dev/null @@ -1,44 +0,0 @@ -export type { SlValidationResult, SlValidatorPort } from './sl-validator.port.js'; -export type { - SemanticLayerQueryExecutionResult, - SemanticLayerQueryInput, - SemanticLayerSource, - SlDictionaryMatch, - SlSearchLaneSummary, - SlSearchMatchReason, - SlSearchMetadata, -} from './types.js'; -export type { - KtxConnectionInfo, - KtxQueryResult, - SlConnectionCatalogPort, - SlPythonPort, - SlSourcesIndexPort, -} from './ports.js'; -export { DEFAULT_PRIORITY, resolveDescription } from './descriptions.js'; -export { isOverlaySource, sourceDefinitionSchema, sourceOverlaySchema } from './schemas.js'; -export { - composeOverlay, - enrichColumnsFromManifest, - findDanglingSegmentRefs, - SemanticLayerService, -} from './semantic-layer.service.js'; -export { loadLatestSlDictionaryEntries } from './sl-dictionary-profile.js'; -export type { SlDictionaryEntry } from './sl-dictionary-profile.js'; -export { createKtxDictionarySearchService } from './dictionary-search.js'; -export type { - KtxDictionarySearchCoverage, - KtxDictionarySearchInput, - KtxDictionarySearchMatch, - KtxDictionarySearchMiss, - KtxDictionarySearchMissReason, - KtxDictionarySearchResponse, - KtxDictionarySearchSearchedConnection, - KtxDictionarySearchStatus, - KtxDictionarySearchValueResult, -} from './dictionary-search.js'; -export { buildSemanticLayerSourceSearchText, SlSearchService } from './sl-search.service.js'; -export { SqliteSlSourcesIndex, type SqliteSlSourcesIndexOptions } from './sqlite-sl-sources-index.js'; -export * from './local-sl.js'; -export * from './local-query.js'; -export * from './tools/index.js'; diff --git a/packages/cli/src/context/sl/local-query.test.ts b/packages/cli/src/context/sl/local-query.test.ts index b4703fe6..800bdb95 100644 --- a/packages/cli/src/context/sl/local-query.test.ts +++ b/packages/cli/src/context/sl/local-query.test.ts @@ -2,8 +2,8 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { KtxSemanticLayerComputePort } from '../daemon/index.js'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { compileLocalSlQuery } from './local-query.js'; describe('compileLocalSlQuery', () => { diff --git a/packages/cli/src/context/sl/local-query.ts b/packages/cli/src/context/sl/local-query.ts index 05a264fc..4d71504e 100644 --- a/packages/cli/src/context/sl/local-query.ts +++ b/packages/cli/src/context/sl/local-query.ts @@ -1,7 +1,7 @@ -import type { KtxSqlQueryExecutorPort } from '../connections/index.js'; -import type { KtxSemanticLayerComputePort } from '../daemon/index.js'; +import type { KtxSqlQueryExecutorPort } from '../../context/connections/query-executor.js'; +import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js'; import type { KtxMcpProgressCallback } from '../mcp/types.js'; -import type { KtxLocalProject } from '../project/index.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; import { loadLocalSlSourceRecords } from './local-sl.js'; import { toResolvedWire } from './semantic-layer.service.js'; import type { SemanticLayerQueryExecutionResult, SemanticLayerQueryInput } from './types.js'; diff --git a/packages/cli/src/context/sl/local-sl.test.ts b/packages/cli/src/context/sl/local-sl.test.ts index 00c00874..18cc7392 100644 --- a/packages/cli/src/context/sl/local-sl.test.ts +++ b/packages/cli/src/context/sl/local-sl.test.ts @@ -2,7 +2,7 @@ import { access, mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { listLocalSlSources, readLocalSlSource, diff --git a/packages/cli/src/context/sl/local-sl.ts b/packages/cli/src/context/sl/local-sl.ts index f4844b7b..18ec8417 100644 --- a/packages/cli/src/context/sl/local-sl.ts +++ b/packages/cli/src/context/sl/local-sl.ts @@ -1,9 +1,11 @@ import { join } from 'node:path'; import YAML from 'yaml'; import { z } from 'zod'; -import type { KtxEmbeddingPort, KtxFileWriteResult } from '../core/index.js'; -import type { KtxLocalProject } from '../project/index.js'; -import { HybridSearchCore, type SearchCandidateGenerator } from '../search/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxFileWriteResult } from '../../context/core/file-store.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { HybridSearchCore } from '../../context/search/hybrid-search-core.js'; +import type { SearchCandidateGenerator } from '../../context/search/types.js'; import { DEFAULT_PRIORITY, resolveDescription } from './descriptions.js'; import { normalizeSemanticLayerDescriptions } from './description-normalization.js'; import { sourceDefinitionSchema, sourceOverlaySchema } from './schemas.js'; @@ -301,6 +303,7 @@ export async function validateLocalSlSource( } } +/** @internal */ export async function writeLocalSlSource( project: KtxLocalProject, input: { connectionId: string; sourceName: string; yaml: string }, diff --git a/packages/cli/src/context/sl/pglite-sl-search-prototype.test.ts b/packages/cli/src/context/sl/pglite-sl-search-prototype.test.ts index 29c81062..372f8668 100644 --- a/packages/cli/src/context/sl/pglite-sl-search-prototype.test.ts +++ b/packages/cli/src/context/sl/pglite-sl-search-prototype.test.ts @@ -3,8 +3,8 @@ import { createServer } from 'node:net'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; -import { assertSearchBackendConformanceCase } from '../search/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; +import { assertSearchBackendConformanceCase } from '../../context/search/backend-conformance.test-utils.js'; import { searchLocalSlSources, writeLocalSlSource, type LocalSlSourceSearchResult } from './local-sl.js'; import { searchLocalSlSourcesWithPglitePrototype } from './pglite-sl-search-prototype.js'; diff --git a/packages/cli/src/context/sl/pglite-sl-search-prototype.ts b/packages/cli/src/context/sl/pglite-sl-search-prototype.ts index 4a521437..95d07505 100644 --- a/packages/cli/src/context/sl/pglite-sl-search-prototype.ts +++ b/packages/cli/src/context/sl/pglite-sl-search-prototype.ts @@ -1,8 +1,9 @@ import { mkdir } from 'node:fs/promises'; import { join } from 'node:path'; -import type { KtxEmbeddingPort } from '../core/index.js'; -import type { KtxLocalProject } from '../project/index.js'; -import { HybridSearchCore, type SearchCandidateGenerator } from '../search/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { HybridSearchCore } from '../../context/search/hybrid-search-core.js'; +import type { SearchCandidateGenerator } from '../../context/search/types.js'; import { KtxPGliteOwnerProcess } from '../search/pglite-owner-process.js'; import { listLocalSlSources, diff --git a/packages/cli/src/context/sl/semantic-layer.service.ts b/packages/cli/src/context/sl/semantic-layer.service.ts index 3c9194ac..7a780af2 100644 --- a/packages/cli/src/context/sl/semantic-layer.service.ts +++ b/packages/cli/src/context/sl/semantic-layer.service.ts @@ -1,6 +1,7 @@ import YAML from 'yaml'; -import type { KtxFileStorePort, KtxLogger } from '../core/index.js'; -import { noopLogger } from '../core/index.js'; +import type { KtxFileStorePort } from '../../context/core/file-store.js'; +import type { KtxLogger } from '../../context/core/config.js'; +import { noopLogger } from '../../context/core/config.js'; import type { TableUsageOutput } from '../ingest/adapters/historic-sql/skill-schemas.js'; import type { SlConnectionCatalogPort, SlPythonPort } from './ports.js'; import { normalizeSemanticLayerDescriptions } from './description-normalization.js'; @@ -1404,6 +1405,7 @@ function parseJoinColumns( * Returns one message per measure-level segment reference that doesn't resolve to * a segment defined on the source. Array is empty when every reference checks out. */ +/** @internal */ export function findDanglingSegmentRefs(source: Record): string[] { const segmentDefs = (source.segments as Array<{ name: string }> | undefined) ?? []; const segmentNames = new Set(segmentDefs.map((s) => s.name)); @@ -1571,6 +1573,7 @@ function parseJoinOn( * matching manifest column (by name). Local values always win. Columns absent from * the manifest pass through unchanged. Returns a new source; does not mutate input. */ +/** @internal */ export function enrichColumnsFromManifest( source: SemanticLayerSource, manifestEntry: SemanticLayerSource | null | undefined, diff --git a/packages/cli/src/context/sl/sl-dictionary-profile.test.ts b/packages/cli/src/context/sl/sl-dictionary-profile.test.ts index 21400a71..f7aa3854 100644 --- a/packages/cli/src/context/sl/sl-dictionary-profile.test.ts +++ b/packages/cli/src/context/sl/sl-dictionary-profile.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { loadLatestSlDictionaryEntries } from './sl-dictionary-profile.js'; describe('loadLatestSlDictionaryEntries', () => { diff --git a/packages/cli/src/context/sl/sl-dictionary-profile.ts b/packages/cli/src/context/sl/sl-dictionary-profile.ts index 141d71ce..53798318 100644 --- a/packages/cli/src/context/sl/sl-dictionary-profile.ts +++ b/packages/cli/src/context/sl/sl-dictionary-profile.ts @@ -1,5 +1,5 @@ -import type { KtxLocalProject } from '../project/index.js'; -import { defaultKtxDataDictionarySettings, isKtxDataDictionaryCandidate } from '../scan/index.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { defaultKtxDataDictionarySettings, isKtxDataDictionaryCandidate } from '../../context/scan/data-dictionary.js'; export interface SlDictionaryEntry { connectionId: string; diff --git a/packages/cli/src/context/sl/sl-search.service.ts b/packages/cli/src/context/sl/sl-search.service.ts index 0a7ecfb5..414f6011 100644 --- a/packages/cli/src/context/sl/sl-search.service.ts +++ b/packages/cli/src/context/sl/sl-search.service.ts @@ -1,5 +1,6 @@ -import type { KtxEmbeddingPort, KtxLogger } from '../core/index.js'; -import { noopLogger } from '../core/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxLogger } from '../../context/core/config.js'; +import { noopLogger } from '../../context/core/config.js'; import type { ReindexWorkResult } from '../index-sync/types.js'; import { DEFAULT_PRIORITY, resolveDescription } from './descriptions.js'; import { normalizeSemanticLayerDescriptions } from './description-normalization.js'; diff --git a/packages/cli/src/context/sl/sl-validator.port.ts b/packages/cli/src/context/sl/sl-validator.port.ts index 83a29968..0b9ecf37 100644 --- a/packages/cli/src/context/sl/sl-validator.port.ts +++ b/packages/cli/src/context/sl/sl-validator.port.ts @@ -1,4 +1,4 @@ -export interface SlValidationResult { +interface SlValidationResult { errors: string[]; warnings: string[]; } diff --git a/packages/cli/src/context/sl/tools/base-semantic-layer.tool.ts b/packages/cli/src/context/sl/tools/base-semantic-layer.tool.ts index 2ec6891f..22822368 100644 --- a/packages/cli/src/context/sl/tools/base-semantic-layer.tool.ts +++ b/packages/cli/src/context/sl/tools/base-semantic-layer.tool.ts @@ -1,14 +1,13 @@ import type { ZodType } from 'zod'; -import type { GitAuthorResolverPort, ToolContext, ToolOutput } from '../../tools/index.js'; -import { BaseTool } from '../../tools/index.js'; +import type { GitAuthorResolverPort } from '../../../context/tools/authors.js'; +import type { ToolContext, ToolOutput } from '../../../context/tools/base-tool.js'; +import { BaseTool } from '../../../context/tools/base-tool.js'; import { sourceDefinitionSchema } from '../schemas.js'; import { SemanticLayerService } from '../semantic-layer.service.js'; import { SlSearchService } from '../sl-search.service.js'; export { sourceDefinitionSchema }; - // ── Shared output types ── - export interface SemanticLayerStructured { success: boolean; sourceName: string; diff --git a/packages/cli/src/context/sl/tools/index.ts b/packages/cli/src/context/sl/tools/index.ts deleted file mode 100644 index 915f91ad..00000000 --- a/packages/cli/src/context/sl/tools/index.ts +++ /dev/null @@ -1,11 +0,0 @@ -export type { BaseSemanticLayerToolDeps, SemanticLayerStructured } from './base-semantic-layer.tool.js'; -export { BaseSemanticLayerTool, sourceDefinitionSchema } from './base-semantic-layer.tool.js'; -export type { SlDiscoverySettings } from './sl-discover.tool.js'; -export { SlDiscoverTool } from './sl-discover.tool.js'; -export { SlEditSourceTool } from './sl-edit-source.tool.js'; -export { SlReadSourceTool } from './sl-read-source.tool.js'; -export { SlRollbackTool } from './sl-rollback.tool.js'; -export { SlValidateTool, validateSemanticLayerEndpoint } from './sl-validate.tool.js'; -export { SlWriteSourceTool } from './sl-write-source.tool.js'; -export type { SlValidationDeps, SourceValidationResult } from './sl-warehouse-validation.js'; -export { revertSourceToPreHead, validateSingleSource } from './sl-warehouse-validation.js'; diff --git a/packages/cli/src/context/sl/tools/sl-discover.tool.test.ts b/packages/cli/src/context/sl/tools/sl-discover.tool.test.ts index 1b961141..6dc30478 100644 --- a/packages/cli/src/context/sl/tools/sl-discover.tool.test.ts +++ b/packages/cli/src/context/sl/tools/sl-discover.tool.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolContext, ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources } from '../../tools/index.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; import type { SemanticLayerSource } from '../types.js'; import { SlDiscoverTool } from './sl-discover.tool.js'; diff --git a/packages/cli/src/context/sl/tools/sl-discover.tool.ts b/packages/cli/src/context/sl/tools/sl-discover.tool.ts index fb55175d..a8b67699 100644 --- a/packages/cli/src/context/sl/tools/sl-discover.tool.ts +++ b/packages/cli/src/context/sl/tools/sl-discover.tool.ts @@ -2,7 +2,7 @@ import { z } from 'zod'; import { DEFAULT_PRIORITY, resolveDescription } from '../descriptions.js'; import type { SemanticLayerService } from '../semantic-layer.service.js'; import type { SemanticLayerSource } from '../types.js'; -import type { ToolContext, ToolOutput } from '../../tools/index.js'; +import type { ToolContext, ToolOutput } from '../../../context/tools/base-tool.js'; import { BaseSemanticLayerTool, type BaseSemanticLayerToolDeps } from './base-semantic-layer.tool.js'; import { slToolConnectionIdSchema } from './connection-id-schema.js'; diff --git a/packages/cli/src/context/sl/tools/sl-edit-source.tool.test.ts b/packages/cli/src/context/sl/tools/sl-edit-source.tool.test.ts index 75c753ef..cf66baf8 100644 --- a/packages/cli/src/context/sl/tools/sl-edit-source.tool.test.ts +++ b/packages/cli/src/context/sl/tools/sl-edit-source.tool.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources, hasTouchedSlSource, type ToolContext } from '../../tools/index.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources, hasTouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import { SlEditSourceTool } from './sl-edit-source.tool.js'; function makeTool(overrides: any = {}) { diff --git a/packages/cli/src/context/sl/tools/sl-edit-source.tool.ts b/packages/cli/src/context/sl/tools/sl-edit-source.tool.ts index f6669120..94abbb36 100644 --- a/packages/cli/src/context/sl/tools/sl-edit-source.tool.ts +++ b/packages/cli/src/context/sl/tools/sl-edit-source.tool.ts @@ -1,12 +1,9 @@ import YAML from 'yaml'; import { z } from 'zod'; -import { - addTouchedSlSource, - type ToolContext, - type ToolOutput, - validateActionRawPaths, - validateActionTargetConnection, -} from '../../tools/index.js'; +import { addTouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext, ToolOutput } from '../../../context/tools/base-tool.js'; +import { validateActionRawPaths } from '../../../context/tools/action-raw-paths.js'; +import { validateActionTargetConnection } from '../../../context/tools/action-target-connection.js'; import { applySqlEdits } from '../../tools/sql-edit-replacer.js'; import { normalizeSemanticLayerDescriptions } from '../description-normalization.js'; import type { SemanticLayerSource } from '../types.js'; diff --git a/packages/cli/src/context/sl/tools/sl-read-source.tool.session.test.ts b/packages/cli/src/context/sl/tools/sl-read-source.tool.session.test.ts index dcb2a919..481c4cfe 100644 --- a/packages/cli/src/context/sl/tools/sl-read-source.tool.session.test.ts +++ b/packages/cli/src/context/sl/tools/sl-read-source.tool.session.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources, type ToolContext } from '../../tools/index.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import { SlReadSourceTool } from './sl-read-source.tool.js'; function makeTool(overrides: Partial> = {}) { diff --git a/packages/cli/src/context/sl/tools/sl-read-source.tool.ts b/packages/cli/src/context/sl/tools/sl-read-source.tool.ts index e4602f31..d037f4f1 100644 --- a/packages/cli/src/context/sl/tools/sl-read-source.tool.ts +++ b/packages/cli/src/context/sl/tools/sl-read-source.tool.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import type { ToolContext, ToolOutput } from '../../tools/index.js'; +import type { ToolContext, ToolOutput } from '../../../context/tools/base-tool.js'; import { BaseSemanticLayerTool, type BaseSemanticLayerToolDeps } from './base-semantic-layer.tool.js'; import { slToolConnectionIdSchema } from './connection-id-schema.js'; diff --git a/packages/cli/src/context/sl/tools/sl-rollback.tool.test.ts b/packages/cli/src/context/sl/tools/sl-rollback.tool.test.ts index 73461e87..5a1927a4 100644 --- a/packages/cli/src/context/sl/tools/sl-rollback.tool.test.ts +++ b/packages/cli/src/context/sl/tools/sl-rollback.tool.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources, hasTouchedSlSource, type ToolContext } from '../../tools/index.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources, hasTouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import { SlRollbackTool } from './sl-rollback.tool.js'; function makeSession(overrides: Partial = {}): ToolSession { diff --git a/packages/cli/src/context/sl/tools/sl-rollback.tool.ts b/packages/cli/src/context/sl/tools/sl-rollback.tool.ts index f3354ac1..bbafa380 100644 --- a/packages/cli/src/context/sl/tools/sl-rollback.tool.ts +++ b/packages/cli/src/context/sl/tools/sl-rollback.tool.ts @@ -1,5 +1,6 @@ import { z } from 'zod'; -import { BaseTool, deleteTouchedSlSource, type ToolContext, type ToolOutput } from '../../tools/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../context/tools/base-tool.js'; +import { deleteTouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; import type { SlConnectionCatalogPort, SlSourcesIndexPort } from '../ports.js'; import { revertSourceToPreHead } from './sl-warehouse-validation.js'; diff --git a/packages/cli/src/context/sl/tools/sl-validate.tool.test.ts b/packages/cli/src/context/sl/tools/sl-validate.tool.test.ts index b6725c08..f0c18eac 100644 --- a/packages/cli/src/context/sl/tools/sl-validate.tool.test.ts +++ b/packages/cli/src/context/sl/tools/sl-validate.tool.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources, type ToolContext } from '../../tools/index.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import type { SemanticLayerService } from '../semantic-layer.service.js'; import type { SemanticLayerSource } from '../types.js'; import { SlValidateTool, validateSemanticLayerEndpoint } from './sl-validate.tool.js'; diff --git a/packages/cli/src/context/sl/tools/sl-validate.tool.ts b/packages/cli/src/context/sl/tools/sl-validate.tool.ts index 8117fbcf..c3690c6f 100644 --- a/packages/cli/src/context/sl/tools/sl-validate.tool.ts +++ b/packages/cli/src/context/sl/tools/sl-validate.tool.ts @@ -1,5 +1,6 @@ import { z } from 'zod'; -import { type ToolContext, type ToolOutput, touchedSlSourceNamesForConnection } from '../../tools/index.js'; +import type { ToolContext, ToolOutput } from '../../../context/tools/base-tool.js'; +import { touchedSlSourceNamesForConnection } from '../../../context/tools/touched-sl-sources.js'; import { SemanticLayerService } from '../semantic-layer.service.js'; import { BaseSemanticLayerTool, @@ -19,6 +20,7 @@ type ValidationReport = { warnings: string[]; }; +/** @internal */ export async function validateSemanticLayerEndpoint( connectionId: string, semanticLayerService: SemanticLayerService, diff --git a/packages/cli/src/context/sl/tools/sl-warehouse-validation.ts b/packages/cli/src/context/sl/tools/sl-warehouse-validation.ts index 36a97647..742d855a 100644 --- a/packages/cli/src/context/sl/tools/sl-warehouse-validation.ts +++ b/packages/cli/src/context/sl/tools/sl-warehouse-validation.ts @@ -1,6 +1,7 @@ import YAML from 'yaml'; -import type { GitService, KtxFileStorePort } from '../../core/index.js'; -import { SYSTEM_GIT_AUTHOR } from '../../tools/index.js'; +import type { GitService } from '../../../context/core/git.service.js'; +import type { KtxFileStorePort } from '../../../context/core/file-store.js'; +import { SYSTEM_GIT_AUTHOR } from '../../../context/tools/authors.js'; import type { SlConnectionCatalogPort, SlSourcesIndexPort } from '../ports.js'; import { sourceOverlaySchema } from '../schemas.js'; import { SemanticLayerService } from '../semantic-layer.service.js'; @@ -16,6 +17,7 @@ export interface SlValidationDeps { probeRowCount: number; } +/** @internal */ export interface SourceValidationResult { errors: string[]; warnings: string[]; @@ -52,6 +54,7 @@ function wrapWithSingleRowQuery(sql: string, dialect: string): string { * Returns errors and hint-style warnings. An empty errors array means the YAML is * structurally valid AND the warehouse can execute a probe against its embedded sql. */ +/** @internal */ export async function validateSingleSource( deps: SlValidationDeps, connectionId: string, diff --git a/packages/cli/src/context/sl/tools/sl-write-source.tool.test.ts b/packages/cli/src/context/sl/tools/sl-write-source.tool.test.ts index 186028b8..f168095c 100644 --- a/packages/cli/src/context/sl/tools/sl-write-source.tool.test.ts +++ b/packages/cli/src/context/sl/tools/sl-write-source.tool.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources, hasTouchedSlSource, type ToolContext } from '../../tools/index.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources, hasTouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import { SlWriteSourceTool } from './sl-write-source.tool.js'; function makeTool(overrides: Partial> = {}) { diff --git a/packages/cli/src/context/sl/tools/sl-write-source.tool.ts b/packages/cli/src/context/sl/tools/sl-write-source.tool.ts index b9a79e6b..1ff0f7ee 100644 --- a/packages/cli/src/context/sl/tools/sl-write-source.tool.ts +++ b/packages/cli/src/context/sl/tools/sl-write-source.tool.ts @@ -1,12 +1,9 @@ import YAML from 'yaml'; import { z } from 'zod'; -import { - addTouchedSlSource, - type ToolContext, - type ToolOutput, - validateActionRawPaths, - validateActionTargetConnection, -} from '../../tools/index.js'; +import { addTouchedSlSource } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext, ToolOutput } from '../../../context/tools/base-tool.js'; +import { validateActionRawPaths } from '../../../context/tools/action-raw-paths.js'; +import { validateActionTargetConnection } from '../../../context/tools/action-target-connection.js'; import { sourceOverlaySchema } from '../schemas.js'; import type { SemanticLayerService } from '../semantic-layer.service.js'; import type { SemanticLayerSource } from '../types.js'; diff --git a/packages/cli/src/context/sl/types.ts b/packages/cli/src/context/sl/types.ts index da1ea9aa..f58b57c5 100644 --- a/packages/cli/src/context/sl/types.ts +++ b/packages/cli/src/context/sl/types.ts @@ -109,9 +109,3 @@ export interface SlSearchLaneSummary { reason?: string; } -export interface SlSearchMetadata { - score: number; - matchReasons: SlSearchMatchReason[]; - dictionaryMatches?: SlDictionaryMatch[]; - lanes?: SlSearchLaneSummary[]; -} diff --git a/packages/cli/src/context/sql-analysis/index.ts b/packages/cli/src/context/sql-analysis/index.ts deleted file mode 100644 index c01a8aaa..00000000 --- a/packages/cli/src/context/sql-analysis/index.ts +++ /dev/null @@ -1,13 +0,0 @@ -export { createHttpSqlAnalysisPort } from './http-sql-analysis-port.js'; -export type { HttpSqlAnalysisPortOptions, KtxSqlAnalysisHttpJsonRunner } from './http-sql-analysis-port.js'; -export type { - SqlAnalysisBatchItem, - SqlAnalysisBatchResult, - SqlAnalysisClause, - SqlAnalysisDialect, - SqlAnalysisFingerprintResult, - SqlAnalysisLiteralSlot, - SqlAnalysisLiteralSlotType, - SqlAnalysisPort, - SqlReadOnlyValidationResult, -} from './ports.js'; diff --git a/packages/cli/src/context/sql-analysis/ports.ts b/packages/cli/src/context/sql-analysis/ports.ts index 891515b7..887be605 100644 --- a/packages/cli/src/context/sql-analysis/ports.ts +++ b/packages/cli/src/context/sql-analysis/ports.ts @@ -25,7 +25,7 @@ export interface SqlAnalysisFingerprintResult { error?: string | null; } -export type SqlAnalysisClause = 'select' | 'where' | 'join' | 'groupBy' | 'having' | 'orderBy' | (string & {}); +type SqlAnalysisClause = 'select' | 'where' | 'join' | 'groupBy' | 'having' | 'orderBy' | (string & {}); export interface SqlAnalysisBatchItem { id: string; diff --git a/packages/cli/src/context/tools/base-tool.ts b/packages/cli/src/context/tools/base-tool.ts index faf27e59..d002ac42 100644 --- a/packages/cli/src/context/tools/base-tool.ts +++ b/packages/cli/src/context/tools/base-tool.ts @@ -1,6 +1,6 @@ import { tool } from 'ai'; import { z, type ZodType } from 'zod'; -import { noopLogger, type KtxLogger } from '../core/index.js'; +import { noopLogger, type KtxLogger } from '../../context/core/config.js'; import type { KtxRuntimeToolDescriptor } from '../llm/runtime-port.js'; import { normalizeKtxRuntimeToolOutput } from '../llm/runtime-tools.js'; import type { IngestToolMetadata, ToolSession } from './tool-session.js'; @@ -10,12 +10,12 @@ export interface ToolOutput { structured: T; } -export interface ToolTimingTrackerPort { +interface ToolTimingTrackerPort { recordToolExecutionStart(messageId: string, toolName: string, toolCallId: string): void; recordToolExecutionEnd(messageId: string, toolName: string, toolCallId: string, state: string): void; } -export interface ToolProgressRelayPort { +interface ToolProgressRelayPort { emit(event: unknown): void; } @@ -62,7 +62,7 @@ export interface ToolContext { }; } -export interface MethodologyEntry { +interface MethodologyEntry { key: string; toolName: string; label: string; diff --git a/packages/cli/src/context/tools/context-candidate-write.tool.ts b/packages/cli/src/context/tools/context-candidate-write.tool.ts index 16a28707..8c2b0ea3 100644 --- a/packages/cli/src/context/tools/context-candidate-write.tool.ts +++ b/packages/cli/src/context/tools/context-candidate-write.tool.ts @@ -1,7 +1,7 @@ import { createHash } from 'node:crypto'; import { z } from 'zod'; -import type { KtxEmbeddingPort } from '../core/index.js'; -import { buildContextCandidateEmbeddingText } from '../ingest/context-candidates/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import { buildContextCandidateEmbeddingText } from '../../context/ingest/context-candidates/embedding-text.js'; import { BaseTool, type ToolContext, type ToolOutput } from './base-tool.js'; import { chunkIdSchema } from './context-evidence-ids.js'; import type { ContextEvidenceToolStorePort } from './context-evidence-tool-store.js'; diff --git a/packages/cli/src/context/tools/context-evidence-search.tool.ts b/packages/cli/src/context/tools/context-evidence-search.tool.ts index cdd32734..51534f7a 100644 --- a/packages/cli/src/context/tools/context-evidence-search.tool.ts +++ b/packages/cli/src/context/tools/context-evidence-search.tool.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import type { KtxEmbeddingPort } from '../core/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; import { BaseTool, type ToolContext, type ToolOutput } from './base-tool.js'; import type { ContextEvidenceToolStorePort } from './context-evidence-tool-store.js'; import { ingestMetadataRequired, resolveIngestMetadata, type ToolFailure } from './context-ingest-metadata.js'; diff --git a/packages/cli/src/context/tools/context-evidence-tool-store.ts b/packages/cli/src/context/tools/context-evidence-tool-store.ts index b6fac3d4..713a224a 100644 --- a/packages/cli/src/context/tools/context-evidence-tool-store.ts +++ b/packages/cli/src/context/tools/context-evidence-tool-store.ts @@ -1,4 +1,4 @@ -import type { InsertContextCandidateInput } from '../ingest/context-candidates/index.js'; +import type { InsertContextCandidateInput } from '../../context/ingest/context-candidates/types.js'; import type { JsonValue } from '../ingest/ports.js'; export interface ContextEvidenceSearchArgs { @@ -40,7 +40,7 @@ export interface ContextEvidenceSearchResult { lanes?: ContextEvidenceSearchLaneSummary[]; } -export interface ContextEvidenceDocumentForRead { +interface ContextEvidenceDocumentForRead { id: string; title: string; path: string; @@ -48,7 +48,7 @@ export interface ContextEvidenceDocumentForRead { url: string | null; } -export interface ContextEvidenceChunkForRead { +interface ContextEvidenceChunkForRead { id: string; content: string; citation?: JsonValue; @@ -89,7 +89,7 @@ export interface ContextEvidenceChunkForCandidate { lastEditedAt: Date | null; } -export interface ContextCandidateInsertResult { +interface ContextCandidateInsertResult { id: string; candidate_key: string; promotion_score: number; diff --git a/packages/cli/src/context/tools/context-evidence-tools.test.ts b/packages/cli/src/context/tools/context-evidence-tools.test.ts index 2a76f924..08a8654f 100644 --- a/packages/cli/src/context/tools/context-evidence-tools.test.ts +++ b/packages/cli/src/context/tools/context-evidence-tools.test.ts @@ -3,7 +3,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { KtxEmbeddingPort } from '../core/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; import { SqliteContextEvidenceStore } from '../ingest/context-evidence/sqlite-context-evidence-store.js'; import { ContextCandidateMarkTool } from './context-candidate-mark.tool.js'; import { ContextCandidateWriteTool } from './context-candidate-write.tool.js'; @@ -11,7 +11,9 @@ import { ContextEvidenceNeighborsTool } from './context-evidence-neighbors.tool. import { ContextEvidenceReadTool } from './context-evidence-read.tool.js'; import { ContextEvidenceSearchTool } from './context-evidence-search.tool.js'; import type { ContextEvidenceToolStorePort } from './context-evidence-tool-store.js'; -import { createTouchedSlSources, type ToolContext, type ToolSession } from './index.js'; +import { createTouchedSlSources } from '../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../context/tools/base-tool.js'; +import type { ToolSession } from '../../context/tools/tool-session.js'; const ingestContext = (): ToolContext => ({ sourceId: 'ingest', diff --git a/packages/cli/src/context/tools/index.ts b/packages/cli/src/context/tools/index.ts deleted file mode 100644 index c6a334d5..00000000 --- a/packages/cli/src/context/tools/index.ts +++ /dev/null @@ -1,45 +0,0 @@ -export type { GitAuthor, GitAuthorResolverPort } from './authors.js'; -export { SYSTEM_GIT_AUTHOR } from './authors.js'; -export type { - MethodologyEntry, - ToolContext, - ToolOutput, - ToolProgressRelayPort, - ToolTimingTrackerPort, -} from './base-tool.js'; -export { BaseTool } from './base-tool.js'; -export { ContextCandidateMarkTool } from './context-candidate-mark.tool.js'; -export { ContextCandidateWriteTool } from './context-candidate-write.tool.js'; -export { ContextEvidenceNeighborsTool } from './context-evidence-neighbors.tool.js'; -export { ContextEvidenceReadTool } from './context-evidence-read.tool.js'; -export { ContextEvidenceSearchTool } from './context-evidence-search.tool.js'; -export type { - ContextCandidateInsertResult, - ContextCandidateStatusResult, - ContextEvidenceChunkForCandidate, - ContextEvidenceChunkForRead, - ContextEvidenceChunkReadResult, - ContextEvidenceDocumentForRead, - ContextEvidenceNeighborResult, - ContextEvidenceReadResult, - ContextEvidenceSearchArgs, - ContextEvidenceSearchResult, - ContextEvidenceToolStorePort, -} from './context-evidence-tool-store.js'; -export type { ToolFailure } from './context-ingest-metadata.js'; -export { ingestMetadataRequired, resolveIngestMetadata } from './context-ingest-metadata.js'; -export type { SqlEdit } from './sql-edit-replacer.js'; -export { applySqlEdits } from './sql-edit-replacer.js'; -export type { IngestToolMetadata, MemoryAction, ToolSession } from './tool-session.js'; -export { validateActionRawPaths } from './action-raw-paths.js'; -export { validateActionTargetConnection } from './action-target-connection.js'; -export type { TouchedSlSource, TouchedSlSourceSet } from './touched-sl-sources.js'; -export { - addTouchedSlSource, - createTouchedSlSources, - deleteTouchedSlSource, - hasTouchedSlSource, - listTouchedSlSources, - touchedSlSourceCount, - touchedSlSourceNamesForConnection, -} from './touched-sl-sources.js'; diff --git a/packages/cli/src/context/tools/tool-session.ts b/packages/cli/src/context/tools/tool-session.ts index 05da85d9..a1c95a89 100644 --- a/packages/cli/src/context/tools/tool-session.ts +++ b/packages/cli/src/context/tools/tool-session.ts @@ -1,6 +1,7 @@ -import type { GitService, KtxFileStorePort } from '../core/index.js'; -import type { SemanticLayerService } from '../sl/index.js'; -import type { KnowledgeWikiService } from '../wiki/index.js'; +import type { GitService } from '../../context/core/git.service.js'; +import type { KtxFileStorePort } from '../../context/core/file-store.js'; +import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; +import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; import type { TouchedSlSourceSet } from './touched-sl-sources.js'; export interface IngestToolMetadata { diff --git a/packages/cli/src/context/tools/touched-sl-sources.ts b/packages/cli/src/context/tools/touched-sl-sources.ts index 44c3a834..75f080ae 100644 --- a/packages/cli/src/context/tools/touched-sl-sources.ts +++ b/packages/cli/src/context/tools/touched-sl-sources.ts @@ -30,6 +30,7 @@ export function deleteTouchedSlSource(touched: TouchedSlSourceSet, connectionId: } } +/** @internal */ export function hasTouchedSlSource(touched: TouchedSlSourceSet, connectionId: string, sourceName: string): boolean { return touched.get(connectionId)?.has(sourceName) ?? false; } diff --git a/packages/cli/src/context/wiki/index.ts b/packages/cli/src/context/wiki/index.ts deleted file mode 100644 index 17d37399..00000000 --- a/packages/cli/src/context/wiki/index.ts +++ /dev/null @@ -1,37 +0,0 @@ -export { buildKnowledgeSearchText } from './knowledge-search-text.js'; -export { - assertFlatWikiKey, - invalidFlatWikiKeyMessage, - isFlatWikiKey, - suggestFlatWikiKey, - validateFlatWikiKey, -} from './keys.js'; -export { KnowledgeWikiService } from './knowledge-wiki.service.js'; -export * from './local-knowledge.js'; -export type { - KnowledgeEventPort, - KnowledgeGitDiffPort, - KnowledgeIndexPort, - KnowledgeIndexPageListing, - UpsertPageParams, - WikiFileStorePort, -} from './ports.js'; -export type { - ExistingKnowledgeIndexPage, - SqliteKnowledgeIndexOptions, - SqliteKnowledgeIndexPage, - SqliteKnowledgeIndexSearchResult, - WikiSqliteLaneCandidate, -} from './sqlite-knowledge-index.js'; -export { SqliteKnowledgeIndex } from './sqlite-knowledge-index.js'; -export * from './tools/index.js'; -export type { - HistoricSqlWikiUsageFrontmatter, - WikiFrontmatter, - WikiPage, - WikiPageWithScope, - WikiScope, - WikiSearchLaneSummary, - WikiSearchMatchReason, - WikiSearchMetadata, -} from './types.js'; diff --git a/packages/cli/src/context/wiki/keys.ts b/packages/cli/src/context/wiki/keys.ts index 4bffae2e..8af66373 100644 --- a/packages/cli/src/context/wiki/keys.ts +++ b/packages/cli/src/context/wiki/keys.ts @@ -10,7 +10,7 @@ export function suggestFlatWikiKey(key: string): string { return suggested.length > 0 ? suggested : 'page-key'; } -export function invalidFlatWikiKeyMessage(key: string): string { +function invalidFlatWikiKeyMessage(key: string): string { return `Invalid wiki key "${key}". Wiki keys must be flat; use "${suggestFlatWikiKey(key)}".`; } diff --git a/packages/cli/src/context/wiki/knowledge-wiki.service.ts b/packages/cli/src/context/wiki/knowledge-wiki.service.ts index 88447c14..1383e4d4 100644 --- a/packages/cli/src/context/wiki/knowledge-wiki.service.ts +++ b/packages/cli/src/context/wiki/knowledge-wiki.service.ts @@ -1,7 +1,9 @@ import { createHash } from 'node:crypto'; import YAML from 'yaml'; -import type { KtxEmbeddingPort, KtxFileStorePort, KtxLogger } from '../core/index.js'; -import { noopLogger } from '../core/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxFileStorePort } from '../../context/core/file-store.js'; +import type { KtxLogger } from '../../context/core/config.js'; +import { noopLogger } from '../../context/core/config.js'; import type { ReindexWorkResult } from '../index-sync/types.js'; import { assertFlatWikiKey, isFlatWikiKey } from './keys.js'; import { buildKnowledgeSearchText } from './knowledge-search-text.js'; @@ -11,10 +13,8 @@ import type { WikiFrontmatter, WikiPage, WikiPageWithScope } from './types.js'; const WIKI_PREFIX = 'wiki'; export type { WikiFrontmatter }; - export class KnowledgeWikiService { private isWorktreeScoped = false; - constructor( private readonly configService: KtxFileStorePort, private readonly embeddingService: KtxEmbeddingPort | null, diff --git a/packages/cli/src/context/wiki/local-knowledge.test.ts b/packages/cli/src/context/wiki/local-knowledge.test.ts index 2a166fc5..8229d5e7 100644 --- a/packages/cli/src/context/wiki/local-knowledge.test.ts +++ b/packages/cli/src/context/wiki/local-knowledge.test.ts @@ -2,7 +2,7 @@ import { access, mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { initKtxProject, type KtxLocalProject } from '../project/index.js'; +import { initKtxProject, type KtxLocalProject } from '../../context/project/project.js'; import { listLocalKnowledgePages, readLocalKnowledgePage, diff --git a/packages/cli/src/context/wiki/local-knowledge.ts b/packages/cli/src/context/wiki/local-knowledge.ts index b228cfd4..b7132b50 100644 --- a/packages/cli/src/context/wiki/local-knowledge.ts +++ b/packages/cli/src/context/wiki/local-knowledge.ts @@ -1,8 +1,10 @@ import { join } from 'node:path'; import YAML from 'yaml'; -import type { KtxEmbeddingPort, KtxFileWriteResult } from '../core/index.js'; -import type { KtxLocalProject } from '../project/index.js'; -import { HybridSearchCore, type SearchCandidateGenerator } from '../search/index.js'; +import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; +import type { KtxFileWriteResult } from '../../context/core/file-store.js'; +import type { KtxLocalProject } from '../../context/project/project.js'; +import { HybridSearchCore } from '../../context/search/hybrid-search-core.js'; +import type { SearchCandidateGenerator } from '../../context/search/types.js'; import { buildKnowledgeSearchText } from './knowledge-search-text.js'; import { assertFlatWikiKey, isFlatWikiKey } from './keys.js'; import { SqliteKnowledgeIndex, type SqliteKnowledgeIndexPage } from './sqlite-knowledge-index.js'; @@ -34,6 +36,7 @@ export interface LocalKnowledgeSearchResult extends LocalKnowledgeSummary { lanes?: WikiSearchLaneSummary[]; } +/** @internal */ export interface WriteLocalKnowledgePageInput { key: string; scope: LocalKnowledgeScope; @@ -148,6 +151,7 @@ async function readPageAtPath( } } +/** @internal */ export async function writeLocalKnowledgePage( project: KtxLocalProject, input: WriteLocalKnowledgePageInput, diff --git a/packages/cli/src/context/wiki/ports.ts b/packages/cli/src/context/wiki/ports.ts index 6bcb2f17..6b026966 100644 --- a/packages/cli/src/context/wiki/ports.ts +++ b/packages/cli/src/context/wiki/ports.ts @@ -1,4 +1,3 @@ -import type { KtxFileStorePort } from '../core/file-store.js'; export interface UpsertPageParams { scope: string; @@ -72,4 +71,3 @@ export interface KnowledgeGitDiffPort { getFileAtCommit(path: string, sha: string): Promise; } -export type WikiFileStorePort = KtxFileStorePort; diff --git a/packages/cli/src/context/wiki/tools/index.ts b/packages/cli/src/context/wiki/tools/index.ts deleted file mode 100644 index 8cc5ac9c..00000000 --- a/packages/cli/src/context/wiki/tools/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export { WikiListTagsTool } from './wiki-list-tags.tool.js'; -export { WikiReadTool } from './wiki-read.tool.js'; -export { WikiRemoveTool } from './wiki-remove.tool.js'; -export { WikiSearchTool } from './wiki-search.tool.js'; -export { WikiWriteTool } from './wiki-write.tool.js'; diff --git a/packages/cli/src/context/wiki/tools/wiki-list-tags.tool.test.ts b/packages/cli/src/context/wiki/tools/wiki-list-tags.tool.test.ts index e4b5b7f3..49605c4f 100644 --- a/packages/cli/src/context/wiki/tools/wiki-list-tags.tool.test.ts +++ b/packages/cli/src/context/wiki/tools/wiki-list-tags.tool.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolContext } from '../../tools/index.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import { WikiListTagsTool } from './wiki-list-tags.tool.js'; describe('WikiListTagsTool', () => { diff --git a/packages/cli/src/context/wiki/tools/wiki-list-tags.tool.ts b/packages/cli/src/context/wiki/tools/wiki-list-tags.tool.ts index 3a31ee41..e7b358e8 100644 --- a/packages/cli/src/context/wiki/tools/wiki-list-tags.tool.ts +++ b/packages/cli/src/context/wiki/tools/wiki-list-tags.tool.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; import type { KnowledgeIndexPort } from '../ports.js'; -import { BaseTool, type ToolContext, type ToolOutput } from '../../tools/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../context/tools/base-tool.js'; const wikiListTagsInputSchema = z.object({}); diff --git a/packages/cli/src/context/wiki/tools/wiki-read.tool.test.ts b/packages/cli/src/context/wiki/tools/wiki-read.tool.test.ts index 1457702c..ac75b174 100644 --- a/packages/cli/src/context/wiki/tools/wiki-read.tool.test.ts +++ b/packages/cli/src/context/wiki/tools/wiki-read.tool.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources, type ToolContext } from '../../tools/index.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import { WikiReadTool } from './wiki-read.tool.js'; describe('WikiReadTool', () => { diff --git a/packages/cli/src/context/wiki/tools/wiki-read.tool.ts b/packages/cli/src/context/wiki/tools/wiki-read.tool.ts index 858e717e..2600d0ad 100644 --- a/packages/cli/src/context/wiki/tools/wiki-read.tool.ts +++ b/packages/cli/src/context/wiki/tools/wiki-read.tool.ts @@ -1,8 +1,8 @@ import { z } from 'zod'; import type { KnowledgeIndexPort } from '../ports.js'; -import { KnowledgeWikiService } from '../index.js'; +import { KnowledgeWikiService } from '../../../context/wiki/knowledge-wiki.service.js'; import { validateFlatWikiKey } from '../keys.js'; -import { BaseTool, type ToolContext, type ToolOutput } from '../../tools/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../context/tools/base-tool.js'; const WikiReadInputSchema = z.object({ key: z diff --git a/packages/cli/src/context/wiki/tools/wiki-remove.tool.test.ts b/packages/cli/src/context/wiki/tools/wiki-remove.tool.test.ts index 2cdb87e0..8130613c 100644 --- a/packages/cli/src/context/wiki/tools/wiki-remove.tool.test.ts +++ b/packages/cli/src/context/wiki/tools/wiki-remove.tool.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources, type ToolContext } from '../../tools/index.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import { WikiRemoveTool } from './wiki-remove.tool.js'; describe('WikiRemoveTool', () => { diff --git a/packages/cli/src/context/wiki/tools/wiki-remove.tool.ts b/packages/cli/src/context/wiki/tools/wiki-remove.tool.ts index 4d4c1333..7f56fcb6 100644 --- a/packages/cli/src/context/wiki/tools/wiki-remove.tool.ts +++ b/packages/cli/src/context/wiki/tools/wiki-remove.tool.ts @@ -2,9 +2,10 @@ import { z } from 'zod'; import type { KnowledgeIndexPort } from '../ports.js'; import type { KnowledgeEventPort } from '../ports.js'; type BlockScope = 'GLOBAL' | 'USER'; -import { KnowledgeWikiService } from '../index.js'; +import { KnowledgeWikiService } from '../../../context/wiki/knowledge-wiki.service.js'; import { validateFlatWikiKey } from '../keys.js'; -import { BaseTool, type ToolContext, type ToolOutput, validateActionRawPaths } from '../../tools/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../context/tools/base-tool.js'; +import { validateActionRawPaths } from '../../../context/tools/action-raw-paths.js'; const SYSTEM_AUTHOR = 'System User'; const SYSTEM_EMAIL = 'system@example.com'; diff --git a/packages/cli/src/context/wiki/tools/wiki-search.tool.ts b/packages/cli/src/context/wiki/tools/wiki-search.tool.ts index 88081a06..fd3f8c35 100644 --- a/packages/cli/src/context/wiki/tools/wiki-search.tool.ts +++ b/packages/cli/src/context/wiki/tools/wiki-search.tool.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { BaseTool, type ToolContext, type ToolOutput } from '../../tools/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../context/tools/base-tool.js'; import type { WikiSearchLaneSummary, WikiSearchMatchReason } from '../types.js'; const WikiSearchInputSchema = z.object({ diff --git a/packages/cli/src/context/wiki/tools/wiki-write.tool.test.ts b/packages/cli/src/context/wiki/tools/wiki-write.tool.test.ts index 71d9ca6e..ad2bc54b 100644 --- a/packages/cli/src/context/wiki/tools/wiki-write.tool.test.ts +++ b/packages/cli/src/context/wiki/tools/wiki-write.tool.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import type { ToolSession } from '../../tools/index.js'; -import { createTouchedSlSources, type ToolContext } from '../../tools/index.js'; +import type { ToolSession } from '../../../context/tools/tool-session.js'; +import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js'; +import type { ToolContext } from '../../../context/tools/base-tool.js'; import { WikiWriteTool } from './wiki-write.tool.js'; function makeTool(overrides: any = {}) { diff --git a/packages/cli/src/context/wiki/tools/wiki-write.tool.ts b/packages/cli/src/context/wiki/tools/wiki-write.tool.ts index 5dd82578..4b0f1b39 100644 --- a/packages/cli/src/context/wiki/tools/wiki-write.tool.ts +++ b/packages/cli/src/context/wiki/tools/wiki-write.tool.ts @@ -2,11 +2,13 @@ import { z } from 'zod'; import type { KnowledgeIndexPort } from '../ports.js'; import type { KnowledgeEventPort } from '../ports.js'; type BlockScope = 'GLOBAL' | 'USER'; -import { KnowledgeWikiService, type WikiFrontmatter } from '../index.js'; +import { KnowledgeWikiService } from '../../../context/wiki/knowledge-wiki.service.js'; +import type { WikiFrontmatter } from '../../../context/wiki/types.js'; import { validateFlatWikiKey } from '../keys.js'; import { findMissingWikiRefs } from '../wiki-ref-validation.js'; import { applySqlEdits } from '../../tools/sql-edit-replacer.js'; -import { BaseTool, type ToolContext, type ToolOutput, validateActionRawPaths } from '../../tools/index.js'; +import { BaseTool, type ToolContext, type ToolOutput } from '../../../context/tools/base-tool.js'; +import { validateActionRawPaths } from '../../../context/tools/action-raw-paths.js'; const MAX_USER_BLOCKS = 100; const SYSTEM_AUTHOR = 'System User'; diff --git a/packages/cli/src/context/wiki/types.ts b/packages/cli/src/context/wiki/types.ts index bff57aa5..e5ff3312 100644 --- a/packages/cli/src/context/wiki/types.ts +++ b/packages/cli/src/context/wiki/types.ts @@ -50,8 +50,3 @@ export interface WikiSearchLaneSummary { reason?: string; } -export interface WikiSearchMetadata { - score: number; - matchReasons: WikiSearchMatchReason[]; - lanes?: WikiSearchLaneSummary[]; -} diff --git a/packages/cli/src/context/wiki/wiki-ref-validation.ts b/packages/cli/src/context/wiki/wiki-ref-validation.ts index 5a3ae8c2..28e67f2e 100644 --- a/packages/cli/src/context/wiki/wiki-ref-validation.ts +++ b/packages/cli/src/context/wiki/wiki-ref-validation.ts @@ -1,4 +1,4 @@ -import type { MemoryAction } from '../tools/index.js'; +import type { MemoryAction } from '../../context/tools/tool-session.js'; import { isFlatWikiKey } from './keys.js'; import type { KnowledgeWikiService } from './knowledge-wiki.service.js'; import type { WikiScope } from './types.js'; diff --git a/packages/cli/src/database-tree-picker.ts b/packages/cli/src/database-tree-picker.ts index d7ab1b20..aea3c092 100644 --- a/packages/cli/src/database-tree-picker.ts +++ b/packages/cli/src/database-tree-picker.ts @@ -1,4 +1,4 @@ -import type { KtxTableListEntry } from './context/scan/index.js'; +import type { KtxTableListEntry } from './context/scan/types.js'; import type { KtxCliIo } from './cli-runtime.js'; import { profileMark } from './startup-profile.js'; import { diff --git a/packages/cli/src/demo-metrics.test.ts b/packages/cli/src/demo-metrics.test.ts index ba2ff6a9..9e40be36 100644 --- a/packages/cli/src/demo-metrics.test.ts +++ b/packages/cli/src/demo-metrics.test.ts @@ -1,4 +1,4 @@ -import type { MemoryFlowEvent, MemoryFlowReplayInput } from './context/ingest/memory-flow/index.js'; +import type { MemoryFlowEvent, MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; import { describe, expect, it } from 'vitest'; import { buildDemoMetrics, diff --git a/packages/cli/src/demo-metrics.ts b/packages/cli/src/demo-metrics.ts index 280a869f..d6f9c207 100644 --- a/packages/cli/src/demo-metrics.ts +++ b/packages/cli/src/demo-metrics.ts @@ -1,4 +1,4 @@ -import type { MemoryFlowEvent, MemoryFlowReplayInput } from './context/ingest/memory-flow/index.js'; +import type { MemoryFlowEvent, MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; const DEFAULT_INPUT_TOKENS_PER_STEP = 4500; const DEFAULT_OUTPUT_TOKENS_PER_STEP = 700; diff --git a/packages/cli/src/doctor.ts b/packages/cli/src/doctor.ts index 9a743160..ffde396d 100644 --- a/packages/cli/src/doctor.ts +++ b/packages/cli/src/doctor.ts @@ -4,7 +4,7 @@ import { access, readFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; -import type { KtxConfigIssue } from './context/project/index.js'; +import type { KtxConfigIssue } from './context/project/config.js'; import { KTX_NEXT_STEP_DIRECT_COMMANDS } from './next-steps.js'; import type { BuildProjectStatusOptions } from './status-project.js'; @@ -593,7 +593,7 @@ export async function runKtxDoctor( renderMissingProjectMessage(args.projectDir, args.outputMode, io); return 1; } - const { validateKtxProjectConfig } = await import('./context/project/index.js'); + const { validateKtxProjectConfig } = await import('./context/project/config.js');; const rawConfig = await readFile(configPath, 'utf-8'); const validation = validateKtxProjectConfig(rawConfig); if (!validation.ok) { @@ -610,7 +610,8 @@ export async function runKtxDoctor( renderMissingProjectMessage(args.projectDir, args.outputMode, io); return 1; } - const { loadKtxProject, validateKtxProjectConfig } = await import('./context/project/index.js'); + const { loadKtxProject } = await import('./context/project/project.js'); + const { validateKtxProjectConfig } = await import('./context/project/config.js');; const { buildProjectStatus, renderProjectStatus } = await import('./status-project.js'); const rawConfig = await readFile(configPath, 'utf-8'); const validation = validateKtxProjectConfig(rawConfig); diff --git a/packages/cli/src/embedding-resolution.test.ts b/packages/cli/src/embedding-resolution.test.ts index 7163d88a..40c71538 100644 --- a/packages/cli/src/embedding-resolution.test.ts +++ b/packages/cli/src/embedding-resolution.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { buildDefaultKtxProjectConfig, type KtxLocalProject, type KtxProjectConfig } from './context/project/index.js'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; +import type { KtxLocalProject } from './context/project/project.js'; import { resolveProjectEmbeddingProvider } from './embedding-resolution.js'; import type { ManagedLocalEmbeddingsDaemon } from './managed-local-embeddings.js'; diff --git a/packages/cli/src/embedding-resolution.ts b/packages/cli/src/embedding-resolution.ts index cb9b396c..0243943c 100644 --- a/packages/cli/src/embedding-resolution.ts +++ b/packages/cli/src/embedding-resolution.ts @@ -1,9 +1,8 @@ -import { - type KtxEmbeddingProvider, - createKtxEmbeddingProvider as defaultCreateKtxEmbeddingProvider, -} from './llm/index.js'; -import type { KtxLocalProject, KtxProjectEmbeddingConfig } from './context/project/index.js'; -import { resolveLocalKtxEmbeddingConfig } from './context/index.js'; +import type { KtxEmbeddingProvider } from './llm/types.js'; +import { createKtxEmbeddingProvider as defaultCreateKtxEmbeddingProvider } from './llm/embedding-provider.js'; +import type { KtxLocalProject } from './context/project/project.js'; +import type { KtxProjectEmbeddingConfig } from './context/project/config.js'; +import { resolveLocalKtxEmbeddingConfig } from './context/llm/local-config.js'; import type { KtxCliIo } from './cli-runtime.js'; import { ensureManagedLocalEmbeddingsDaemon as defaultEnsureManagedDaemon, diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index 843a3010..473b5002 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject } from './context/project/index.js'; +import { initKtxProject } from './context/project/project.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { diff --git a/packages/cli/src/ingest-depth.ts b/packages/cli/src/ingest-depth.ts index 940cf6bc..489c44e8 100644 --- a/packages/cli/src/ingest-depth.ts +++ b/packages/cli/src/ingest-depth.ts @@ -1,4 +1,4 @@ -import type { KtxProjectConfig, KtxProjectConnectionConfig } from './context/project/index.js'; +import type { KtxProjectConfig, KtxProjectConnectionConfig } from './context/project/config.js'; export type KtxDatabaseContextDepth = 'fast' | 'deep'; diff --git a/packages/cli/src/ingest-query-executor.test.ts b/packages/cli/src/ingest-query-executor.test.ts index c1bd4b09..14b714d9 100644 --- a/packages/cli/src/ingest-query-executor.test.ts +++ b/packages/cli/src/ingest-query-executor.test.ts @@ -1,5 +1,5 @@ -import type { KtxLocalProject } from './context/project/index.js'; -import { createKtxConnectorCapabilities, type KtxScanConnector } from './context/scan/index.js'; +import type { KtxLocalProject } from './context/project/project.js'; +import { createKtxConnectorCapabilities, type KtxScanConnector } from './context/scan/types.js'; import { describe, expect, it, vi } from 'vitest'; import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js'; diff --git a/packages/cli/src/ingest-query-executor.ts b/packages/cli/src/ingest-query-executor.ts index a8871806..f8b6880d 100644 --- a/packages/cli/src/ingest-query-executor.ts +++ b/packages/cli/src/ingest-query-executor.ts @@ -1,6 +1,6 @@ -import type { KtxSqlQueryExecutionInput, KtxSqlQueryExecutorPort } from './context/connections/index.js'; -import type { KtxLocalProject } from './context/project/index.js'; -import type { KtxScanConnector, KtxScanContext } from './context/scan/index.js'; +import type { KtxSqlQueryExecutionInput, KtxSqlQueryExecutorPort } from './context/connections/query-executor.js'; +import type { KtxLocalProject } from './context/project/project.js'; +import type { KtxScanConnector, KtxScanContext } from './context/scan/types.js'; import { createKtxCliScanConnector } from './local-scan-connectors.js'; type CreateConnector = typeof createKtxCliScanConnector; diff --git a/packages/cli/src/ingest-report-file.ts b/packages/cli/src/ingest-report-file.ts index 38162062..7d545429 100644 --- a/packages/cli/src/ingest-report-file.ts +++ b/packages/cli/src/ingest-report-file.ts @@ -1,5 +1,6 @@ import { readFile } from 'node:fs/promises'; -import { parseIngestReportSnapshot, type IngestReportSnapshot } from './context/ingest/index.js'; +import { parseIngestReportSnapshot } from './context/ingest/report-snapshot.js'; +import type { IngestReportSnapshot } from './context/ingest/reports.js'; export async function readIngestReportSnapshotFile(reportFile: string): Promise { const raw = await readFile(reportFile, 'utf-8'); diff --git a/packages/cli/src/ingest-viz.test.ts b/packages/cli/src/ingest-viz.test.ts index 457b02e8..17b35f75 100644 --- a/packages/cli/src/ingest-viz.test.ts +++ b/packages/cli/src/ingest-viz.test.ts @@ -1,11 +1,8 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { - type LocalIngestResult, - type MemoryFlowReplayInput, - type RunLocalIngestOptions, -} from './context/ingest/index.js'; +import type { LocalIngestResult, RunLocalIngestOptions } from './context/ingest/local-ingest.js'; +import type { MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runKtxIngest } from './ingest.js'; import { diff --git a/packages/cli/src/ingest.test-utils.ts b/packages/cli/src/ingest.test-utils.ts index 949fcb67..9b3f16fa 100644 --- a/packages/cli/src/ingest.test-utils.ts +++ b/packages/cli/src/ingest.test-utils.ts @@ -1,29 +1,19 @@ import { EventEmitter } from 'node:events'; import { mkdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; -import type { AgentRunnerPort, RunLoopParams } from './context/index.js'; -import { - KtxYamlMetabaseSourceStateReader, - LocalMetabaseDiscoveryCache, - MetabaseSourceAdapter, - getLocalIngestStatus, - type ChunkResult, - type FetchContext, - type IngestReportSnapshot, - type LocalIngestResult, - type LookerMappingClient, - type LookerRuntimeClient, - type LookerTableIdentifierParser, - type MemoryFlowEventSink, - type MetabaseCard, - type MetabaseCardSummary, - type MetabaseClientFactory, - type MetabaseRuntimeClient, - type RunLocalIngestOptions, - type SourceAdapter, - type SqliteBundleIngestStore, -} from './context/ingest/index.js'; -import { ktxLocalStateDbPath, loadKtxProject } from './context/project/index.js'; +import type { AgentRunnerPort, RunLoopParams } from './context/llm/runtime-port.js'; +import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './context/ingest/adapters/metabase/local-source-state-store.js'; +import { MetabaseSourceAdapter } from './context/ingest/adapters/metabase/metabase.adapter.js'; +import { getLocalIngestStatus, type LocalIngestResult, type RunLocalIngestOptions } from './context/ingest/local-ingest.js'; +import type { ChunkResult, FetchContext, SourceAdapter } from './context/ingest/types.js'; +import type { IngestReportSnapshot } from './context/ingest/reports.js'; +import type { LookerMappingClient, LookerTableIdentifierParser } from './context/ingest/adapters/looker/mapping.js'; +import type { LookerRuntimeClient } from './context/ingest/adapters/looker/fetch.js'; +import type { MemoryFlowEventSink } from './context/ingest/memory-flow/types.js'; +import type { MetabaseCard, MetabaseCardSummary, MetabaseClientFactory, MetabaseRuntimeClient } from './context/ingest/adapters/metabase/client-port.js'; +import type { SqliteBundleIngestStore } from './context/ingest/sqlite-bundle-ingest-store.js'; +import { ktxLocalStateDbPath } from './context/project/local-state-db.js'; +import { loadKtxProject } from './context/project/project.js'; import { expect, vi } from 'vitest'; import { runKtxIngest } from './ingest.js'; @@ -685,7 +675,7 @@ export function localFakeBundleReport( } export async function localBundleStore(projectDir: string, ids: [string, string]): Promise { - const { SqliteBundleIngestStore } = await import('./context/ingest/index.js'); + const { SqliteBundleIngestStore } = await import('./context/ingest/sqlite-bundle-ingest-store.js');; const project = await loadKtxProject({ projectDir }); return new SqliteBundleIngestStore({ dbPath: ktxLocalStateDbPath(project), diff --git a/packages/cli/src/ingest.test.ts b/packages/cli/src/ingest.test.ts index 9cf07459..d5b1689d 100644 --- a/packages/cli/src/ingest.test.ts +++ b/packages/cli/src/ingest.test.ts @@ -1,15 +1,12 @@ import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { - LocalLookerRuntimeStore, - LocalMetabaseDiscoveryCache, - type LocalIngestResult, - type LocalMetabaseFanoutProgress, - type RunLocalIngestOptions, - type SourceAdapter, -} from './context/ingest/index.js'; -import { initKtxProject, ktxLocalStateDbPath, loadKtxProject } from './context/project/index.js'; +import { LocalLookerRuntimeStore } from './context/ingest/adapters/looker/local-runtime-store.js'; +import { LocalMetabaseDiscoveryCache } from './context/ingest/adapters/metabase/local-source-state-store.js'; +import type { LocalIngestResult, LocalMetabaseFanoutProgress, RunLocalIngestOptions } from './context/ingest/local-ingest.js'; +import type { SourceAdapter } from './context/ingest/types.js'; +import { initKtxProject, loadKtxProject } from './context/project/project.js'; +import { ktxLocalStateDbPath } from './context/project/local-state-db.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type KtxIngestArgs, type KtxIngestDeps, runKtxIngest } from './ingest.js'; import type { KtxCliLocalIngestAdaptersOptions } from './local-adapters.js'; diff --git a/packages/cli/src/ingest.ts b/packages/cli/src/ingest.ts index 07417bfc..c821bf45 100644 --- a/packages/cli/src/ingest.ts +++ b/packages/cli/src/ingest.ts @@ -1,24 +1,13 @@ -import { - buildMemoryFlowViewModel, - createMemoryFlowLiveBuffer, - formatMemoryFlowFinalSummary, - getLatestLocalIngestStatus, - getLocalIngestStatus, - type IngestReportSnapshot, - ingestReportToMemoryFlowReplay, - type LocalMetabaseFanoutResult, - type LocalMetabaseFanoutProgress, - type MemoryFlowEvent, - type MemoryFlowReplayInput, - type RunLocalIngestOptions, - renderMemoryFlowReplay, - runLocalIngest, - runLocalMetabaseIngest, - savedMemoryCountsForReport, - sanitizeMemoryFlowError, -} from './context/ingest/index.js'; -import type { KtxSqlQueryExecutorPort } from './context/connections/index.js'; -import { loadKtxProject, type KtxLocalProject } from './context/project/index.js'; +import { buildMemoryFlowViewModel } from './context/ingest/memory-flow/view-model.js'; +import { createMemoryFlowLiveBuffer, sanitizeMemoryFlowError } from './context/ingest/memory-flow/live-buffer.js'; +import { formatMemoryFlowFinalSummary } from './context/ingest/memory-flow/summary.js'; +import { getLatestLocalIngestStatus, getLocalIngestStatus, type LocalMetabaseFanoutResult, type LocalMetabaseFanoutProgress, type RunLocalIngestOptions, runLocalIngest, runLocalMetabaseIngest } from './context/ingest/local-ingest.js'; +import { type IngestReportSnapshot, savedMemoryCountsForReport } from './context/ingest/reports.js'; +import { ingestReportToMemoryFlowReplay } from './context/ingest/memory-flow/events.js'; +import type { MemoryFlowEvent, MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; +import { renderMemoryFlowReplay } from './context/ingest/memory-flow/render.js'; +import type { KtxSqlQueryExecutorPort } from './context/connections/query-executor.js'; +import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; import { resolveProjectEmbeddingProvider } from './embedding-resolution.js'; import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js'; import { readIngestReportSnapshotFile } from './ingest-report-file.js'; diff --git a/packages/cli/src/knowledge.test.ts b/packages/cli/src/knowledge.test.ts index 1b19e82b..9fc92a31 100644 --- a/packages/cli/src/knowledge.test.ts +++ b/packages/cli/src/knowledge.test.ts @@ -2,9 +2,9 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { stripVTControlCharacters } from 'node:util'; -import { initKtxProject, loadKtxProject } from './context/project/index.js'; -import type { KtxEmbeddingPort } from './context/index.js'; -import { writeLocalKnowledgePage } from './context/wiki/index.js'; +import { initKtxProject, loadKtxProject } from './context/project/project.js'; +import type { KtxEmbeddingPort } from './context/core/embedding.js'; +import { writeLocalKnowledgePage } from './context/wiki/local-knowledge.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runKtxKnowledge } from './knowledge.js'; diff --git a/packages/cli/src/knowledge.ts b/packages/cli/src/knowledge.ts index b8f5c65a..9eb35b12 100644 --- a/packages/cli/src/knowledge.ts +++ b/packages/cli/src/knowledge.ts @@ -1,11 +1,7 @@ -import { KtxIngestEmbeddingPortAdapter, type KtxEmbeddingPort } from './context/index.js'; -import { loadKtxProject } from './context/project/index.js'; -import { - type LocalKnowledgeSearchResult, - type LocalKnowledgeSummary, - listLocalKnowledgePages, - searchLocalKnowledgePages as defaultSearchLocalKnowledgePages, -} from './context/wiki/index.js'; +import { KtxIngestEmbeddingPortAdapter } from './context/llm/embedding-port.js'; +import type { KtxEmbeddingPort } from './context/core/embedding.js'; +import { loadKtxProject } from './context/project/project.js'; +import { type LocalKnowledgeSearchResult, type LocalKnowledgeSummary, listLocalKnowledgePages, searchLocalKnowledgePages as defaultSearchLocalKnowledgePages } from './context/wiki/local-knowledge.js'; import { resolveProjectEmbeddingProvider, type EmbeddingProviderResolution, diff --git a/packages/cli/src/llm/index.ts b/packages/cli/src/llm/index.ts deleted file mode 100644 index ab2ad341..00000000 --- a/packages/cli/src/llm/index.ts +++ /dev/null @@ -1,31 +0,0 @@ -export { createKtxEmbeddingProvider } from './embedding-provider.js'; -export { runKtxEmbeddingHealthCheck } from './embedding-health.js'; -export { KtxMessageBuilder, splitKtxSystemMessages } from './message-builder.js'; -export type { KtxSplitSystemMessagesResult } from './message-builder.js'; -export type { KtxEmbeddingHealthCheckOptions, KtxEmbeddingHealthCheckResult } from './embedding-health.js'; -export type { KtxEmbeddingProviderDeps } from './embedding-provider.js'; -export type { KtxLlmHealthCheckDeps, KtxLlmHealthCheckOptions, KtxLlmHealthCheckResult } from './model-health.js'; -export { runKtxLlmHealthCheck } from './model-health.js'; -export { - createKtxLlmProvider, - isAnthropicProtocolModel, - modelIdFromLanguageModel, - type KtxLlmProviderFactoryDeps, -} from './model-provider.js'; -export type { - KtxEmbeddingBackend, - KtxEmbeddingConfig, - KtxEmbeddingProvider, - KtxEmbeddingTokenUsageEvent, - KtxJsonValue, - KtxLlmBackend, - KtxLlmConfig, - KtxLlmProvider, - KtxModelRole, - KtxPromptCacheTtl, - KtxPromptCachingConfig, - KtxPromptParts, - KtxProviderOptions, - KtxTokenUsageEvent, -} from './types.js'; -export { KTX_MODEL_ROLES } from './types.js'; diff --git a/packages/cli/src/llm/model-health.ts b/packages/cli/src/llm/model-health.ts index abbc2735..91920478 100644 --- a/packages/cli/src/llm/model-health.ts +++ b/packages/cli/src/llm/model-health.ts @@ -4,7 +4,7 @@ import type { KtxLlmConfig } from './types.js'; export type KtxLlmHealthCheckResult = { ok: true } | { ok: false; message: string }; -export interface KtxLlmHealthCheckDeps extends Omit { +interface KtxLlmHealthCheckDeps extends Omit { generateText?: (options: Parameters[0]) => Promise; } diff --git a/packages/cli/src/llm/model-provider.ts b/packages/cli/src/llm/model-provider.ts index 86b9270b..207faa93 100644 --- a/packages/cli/src/llm/model-provider.ts +++ b/packages/cli/src/llm/model-provider.ts @@ -55,7 +55,7 @@ function resolveDevtoolsEnabled(override: boolean | undefined): boolean { return value === 'true' || value === '1' || value === 'yes'; } -export function modelIdFromLanguageModel(model: LanguageModel | string): string { +function modelIdFromLanguageModel(model: LanguageModel | string): string { return typeof model === 'string' ? model : ((model as { modelId?: string }).modelId ?? ''); } diff --git a/packages/cli/src/llm/types.ts b/packages/cli/src/llm/types.ts index bc928e08..3f7f67e2 100644 --- a/packages/cli/src/llm/types.ts +++ b/packages/cli/src/llm/types.ts @@ -3,10 +3,10 @@ import type { LanguageModel, TelemetrySettings, ToolCallRepairFunction, ToolSet export const KTX_MODEL_ROLES = ['default', 'triage', 'candidateExtraction', 'curator', 'reconcile', 'repair'] as const; export type KtxModelRole = (typeof KTX_MODEL_ROLES)[number]; -export type KtxLlmBackend = 'anthropic' | 'vertex' | 'gateway' | 'claude-code'; +type KtxLlmBackend = 'anthropic' | 'vertex' | 'gateway' | 'claude-code'; export type KtxPromptCacheTtl = '5m' | '1h'; -export type KtxJsonValue = +type KtxJsonValue = | null | string | number @@ -27,7 +27,7 @@ export interface KtxPromptCachingConfig { vertexFallbackTo5m: boolean; } -export interface KtxTokenUsageEvent { +interface KtxTokenUsageEvent { source?: string; modelId?: string; inputTokens?: number; @@ -62,9 +62,9 @@ export interface KtxLlmProvider { activeBackend(): KtxLlmBackend; } -export type KtxEmbeddingBackend = 'openai' | 'sentence-transformers'; +type KtxEmbeddingBackend = 'openai' | 'sentence-transformers'; -export interface KtxEmbeddingTokenUsageEvent { +interface KtxEmbeddingTokenUsageEvent { backend: KtxEmbeddingBackend; model: string; inputCount: number; diff --git a/packages/cli/src/local-adapters.test.ts b/packages/cli/src/local-adapters.test.ts index 91f5d212..ac8c3c41 100644 --- a/packages/cli/src/local-adapters.test.ts +++ b/packages/cli/src/local-adapters.test.ts @@ -1,7 +1,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { loadKtxProject } from './context/project/index.js'; +import { loadKtxProject } from './context/project/project.js'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { createKtxCliLocalIngestAdapters } from './local-adapters.js'; diff --git a/packages/cli/src/local-adapters.ts b/packages/cli/src/local-adapters.ts index cfcf5caa..d5b7817e 100644 --- a/packages/cli/src/local-adapters.ts +++ b/packages/cli/src/local-adapters.ts @@ -1,33 +1,28 @@ -import { - createBigQueryLiveDatabaseIntrospection, - isKtxBigQueryConnectionConfig, - KtxBigQueryScanConnector, - type KtxBigQueryConnectionConfig, -} from './connectors/bigquery/index.js'; -import { createClickHouseLiveDatabaseIntrospection, isKtxClickHouseConnectionConfig } from './connectors/clickhouse/index.js'; -import { createMysqlLiveDatabaseIntrospection, isKtxMysqlConnectionConfig } from './connectors/mysql/index.js'; -import { - createPostgresLiveDatabaseIntrospection, - isKtxPostgresConnectionConfig, - type KtxPostgresConnectionConfig, - KtxPostgresHistoricSqlQueryClient, -} from './connectors/postgres/index.js'; -import { createSqliteLiveDatabaseIntrospection, isKtxSqliteConnectionConfig } from './connectors/sqlite/index.js'; -import { createSqlServerLiveDatabaseIntrospection, isKtxSqlServerConnectionConfig } from './connectors/sqlserver/index.js'; -import { - BigQueryHistoricSqlQueryHistoryReader, - createDaemonLiveDatabaseIntrospection, - createDefaultLocalIngestAdapters, - type DefaultLocalIngestAdaptersOptions, - type HistoricSqlReader, - type LiveDatabaseIntrospectionPort, - LiveDatabaseSourceAdapter, - PostgresPgssReader, - SnowflakeHistoricSqlQueryHistoryReader, - type SourceAdapter, -} from './context/ingest/index.js'; -import type { KtxLocalProject } from './context/project/index.js'; -import { createHttpSqlAnalysisPort, type SqlAnalysisPort } from './context/sql-analysis/index.js'; +import { createBigQueryLiveDatabaseIntrospection } from './connectors/bigquery/live-database-introspection.js'; +import { isKtxBigQueryConnectionConfig, KtxBigQueryScanConnector, type KtxBigQueryConnectionConfig } from './connectors/bigquery/connector.js'; +import { createClickHouseLiveDatabaseIntrospection } from './connectors/clickhouse/live-database-introspection.js'; +import { isKtxClickHouseConnectionConfig } from './connectors/clickhouse/connector.js'; +import { createMysqlLiveDatabaseIntrospection } from './connectors/mysql/live-database-introspection.js'; +import { isKtxMysqlConnectionConfig } from './connectors/mysql/connector.js'; +import { createPostgresLiveDatabaseIntrospection } from './connectors/postgres/live-database-introspection.js'; +import { isKtxPostgresConnectionConfig, type KtxPostgresConnectionConfig } from './connectors/postgres/connector.js'; +import { KtxPostgresHistoricSqlQueryClient } from './connectors/postgres/historic-sql-query-client.js'; +import { createSqliteLiveDatabaseIntrospection } from './connectors/sqlite/live-database-introspection.js'; +import { isKtxSqliteConnectionConfig } from './connectors/sqlite/connector.js'; +import { createSqlServerLiveDatabaseIntrospection } from './connectors/sqlserver/live-database-introspection.js'; +import { isKtxSqlServerConnectionConfig } from './connectors/sqlserver/connector.js'; +import { BigQueryHistoricSqlQueryHistoryReader } from './context/ingest/adapters/historic-sql/bigquery-query-history-reader.js'; +import { createDaemonLiveDatabaseIntrospection } from './context/ingest/adapters/live-database/daemon-introspection.js'; +import { createDefaultLocalIngestAdapters, type DefaultLocalIngestAdaptersOptions } from './context/ingest/local-adapters.js'; +import type { HistoricSqlReader } from './context/ingest/adapters/historic-sql/types.js'; +import type { LiveDatabaseIntrospectionPort } from './context/ingest/adapters/live-database/types.js'; +import { LiveDatabaseSourceAdapter } from './context/ingest/adapters/live-database/live-database.adapter.js'; +import { PostgresPgssReader } from './context/ingest/adapters/historic-sql/postgres-pgss-reader.js'; +import { SnowflakeHistoricSqlQueryHistoryReader } from './context/ingest/adapters/historic-sql/snowflake-query-history-reader.js'; +import type { SourceAdapter } from './context/ingest/types.js'; +import type { KtxLocalProject } from './context/project/project.js'; +import { createHttpSqlAnalysisPort } from './context/sql-analysis/http-sql-analysis-port.js'; +import type { SqlAnalysisPort } from './context/sql-analysis/ports.js'; import { createManagedDaemonLookerTableIdentifierParser, createManagedDaemonSqlAnalysisPort, @@ -44,7 +39,7 @@ function hasSnowflakeDriver(connection: unknown): boolean { ); } -type SnowflakeConnectorModule = typeof import('./connectors/snowflake/index.js'); +type SnowflakeConnectorModule = typeof import('./connectors/snowflake/connector.js'); function ktxCliDaemonDatabaseIntrospectionOptions( options: KtxCliLocalIngestAdaptersOptions, @@ -141,9 +136,8 @@ function createKtxCliLiveDatabaseIntrospection( return bigquery.extractSchema(connectionId); } if (hasSnowflakeDriver(connection)) { - const { createSnowflakeLiveDatabaseIntrospection, isKtxSnowflakeConnectionConfig } = await import( - './connectors/snowflake/index.js' - ); + const { createSnowflakeLiveDatabaseIntrospection } = await import('./connectors/snowflake/live-database-introspection.js'); + const { isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/connector.js');; if (!isKtxSnowflakeConnectionConfig(connection)) { return daemon.extractSchema(connectionId); } @@ -342,7 +336,7 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli reader: new SnowflakeHistoricSqlQueryHistoryReader() satisfies HistoricSqlReader, queryClient: { async executeQuery(query: string) { - const connectorModule = await import('./connectors/snowflake/index.js'); + const connectorModule = await import('./connectors/snowflake/connector.js'); const client = await createEphemeralSnowflakeHistoricSqlClient(project, connectionId, connectorModule); return client.executeQuery(query); }, diff --git a/packages/cli/src/local-scan-connectors.test.ts b/packages/cli/src/local-scan-connectors.test.ts index 9426609d..a993faa2 100644 --- a/packages/cli/src/local-scan-connectors.test.ts +++ b/packages/cli/src/local-scan-connectors.test.ts @@ -1,7 +1,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject, loadKtxProject } from './context/project/index.js'; +import { initKtxProject, loadKtxProject } from './context/project/project.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { createKtxCliScanConnector } from './local-scan-connectors.js'; @@ -12,7 +12,7 @@ const bigQueryMock = vi.hoisted(() => ({ }>, })); -vi.mock('./connectors/bigquery/index.js', () => ({ +vi.mock('./connectors/bigquery/connector.js', () => ({ isKtxBigQueryConnectionConfig: (connection: { driver?: unknown } | undefined) => String(connection?.driver ?? '').toLowerCase() === 'bigquery', KtxBigQueryScanConnector: class { diff --git a/packages/cli/src/local-scan-connectors.ts b/packages/cli/src/local-scan-connectors.ts index a808d8e2..10b2dd05 100644 --- a/packages/cli/src/local-scan-connectors.ts +++ b/packages/cli/src/local-scan-connectors.ts @@ -1,5 +1,5 @@ -import type { KtxLocalProject } from './context/project/index.js'; -import type { KtxScanConnector } from './context/scan/index.js'; +import type { KtxLocalProject } from './context/project/project.js'; +import type { KtxScanConnector } from './context/scan/types.js'; const SUPPORTED_DRIVERS = 'sqlite, postgres, mysql, clickhouse, sqlserver, bigquery, snowflake'; @@ -18,49 +18,49 @@ export async function createKtxCliScanConnector( ); } if (driver === 'sqlite' || driver === 'sqlite3') { - const { KtxSqliteScanConnector, isKtxSqliteConnectionConfig } = await import('./connectors/sqlite/index.js'); + const { KtxSqliteScanConnector, isKtxSqliteConnectionConfig } = await import('./connectors/sqlite/connector.js');; if (!isKtxSqliteConnectionConfig(connection)) { throw invalidConnectionConfigError(connectionId, driver); } return new KtxSqliteScanConnector({ connectionId, connection, projectDir: project.projectDir }); } if (driver === 'postgres' || driver === 'postgresql') { - const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('./connectors/postgres/index.js'); + const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('./connectors/postgres/connector.js');; if (!isKtxPostgresConnectionConfig(connection)) { throw invalidConnectionConfigError(connectionId, driver); } return new KtxPostgresScanConnector({ connectionId, connection }); } if (driver === 'mysql') { - const { KtxMysqlScanConnector, isKtxMysqlConnectionConfig } = await import('./connectors/mysql/index.js'); + const { KtxMysqlScanConnector, isKtxMysqlConnectionConfig } = await import('./connectors/mysql/connector.js');; if (!isKtxMysqlConnectionConfig(connection)) { throw invalidConnectionConfigError(connectionId, driver); } return new KtxMysqlScanConnector({ connectionId, connection }); } if (driver === 'clickhouse') { - const { KtxClickHouseScanConnector, isKtxClickHouseConnectionConfig } = await import('./connectors/clickhouse/index.js'); + const { KtxClickHouseScanConnector, isKtxClickHouseConnectionConfig } = await import('./connectors/clickhouse/connector.js');; if (!isKtxClickHouseConnectionConfig(connection)) { throw invalidConnectionConfigError(connectionId, driver); } return new KtxClickHouseScanConnector({ connectionId, connection }); } if (driver === 'sqlserver') { - const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('./connectors/sqlserver/index.js'); + const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('./connectors/sqlserver/connector.js');; if (!isKtxSqlServerConnectionConfig(connection)) { throw invalidConnectionConfigError(connectionId, driver); } return new KtxSqlServerScanConnector({ connectionId, connection }); } if (driver === 'bigquery') { - const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('./connectors/bigquery/index.js'); + const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('./connectors/bigquery/connector.js');; if (!isKtxBigQueryConnectionConfig(connection)) { throw invalidConnectionConfigError(connectionId, driver); } return new KtxBigQueryScanConnector({ connectionId, connection }); } if (driver === 'snowflake') { - const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/index.js'); + const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/connector.js');; if (!isKtxSnowflakeConnectionConfig(connection)) { throw invalidConnectionConfigError(connectionId, driver); } diff --git a/packages/cli/src/managed-local-embeddings.ts b/packages/cli/src/managed-local-embeddings.ts index f068ebdd..b178be47 100644 --- a/packages/cli/src/managed-local-embeddings.ts +++ b/packages/cli/src/managed-local-embeddings.ts @@ -1,4 +1,4 @@ -import type { KtxEmbeddingConfig } from './llm/index.js'; +import type { KtxEmbeddingConfig } from './llm/types.js'; import type { KtxCliIo } from './cli-runtime.js'; import { ensureManagedPythonCommandRuntime, diff --git a/packages/cli/src/managed-python-command.ts b/packages/cli/src/managed-python-command.ts index 36ab7060..ecad702f 100644 --- a/packages/cli/src/managed-python-command.ts +++ b/packages/cli/src/managed-python-command.ts @@ -1,4 +1,4 @@ -import { createPythonSemanticLayerComputePort, type KtxSemanticLayerComputePort } from './context/daemon/index.js'; +import { createPythonSemanticLayerComputePort, type KtxSemanticLayerComputePort } from './context/daemon/semantic-layer-compute.js'; import type { KtxCliIo } from './cli-runtime.js'; import { createClackPromptAdapter, createStaticCliSpinner, type KtxCliSpinner } from './clack.js'; import { diff --git a/packages/cli/src/managed-python-http.ts b/packages/cli/src/managed-python-http.ts index 1f7cb775..0c9b24b3 100644 --- a/packages/cli/src/managed-python-http.ts +++ b/packages/cli/src/managed-python-http.ts @@ -1,18 +1,11 @@ import { request as httpRequest } from 'node:http'; import { request as httpsRequest } from 'node:https'; import { URL } from 'node:url'; -import { - createDaemonLookerTableIdentifierParser, - type DaemonLiveDatabaseIntrospectionOptions, - type KtxDaemonDatabaseHttpJsonRunner, - type KtxDaemonTableIdentifierHttpJsonRunner, - type LookerTableIdentifierParser, -} from './context/ingest/index.js'; -import { - createHttpSqlAnalysisPort, - type KtxSqlAnalysisHttpJsonRunner, - type SqlAnalysisPort, -} from './context/sql-analysis/index.js'; +import { createDaemonLookerTableIdentifierParser, type KtxDaemonTableIdentifierHttpJsonRunner } from './context/ingest/adapters/looker/daemon-table-identifier-parser.js'; +import type { DaemonLiveDatabaseIntrospectionOptions, KtxDaemonDatabaseHttpJsonRunner } from './context/ingest/adapters/live-database/daemon-introspection.js'; +import type { LookerTableIdentifierParser } from './context/ingest/adapters/looker/mapping.js'; +import { createHttpSqlAnalysisPort, type KtxSqlAnalysisHttpJsonRunner } from './context/sql-analysis/http-sql-analysis-port.js'; +import type { SqlAnalysisPort } from './context/sql-analysis/ports.js'; import type { KtxCliIo } from './cli-runtime.js'; import { ensureManagedPythonCommandRuntime, diff --git a/packages/cli/src/mcp-http-server.ts b/packages/cli/src/mcp-http-server.ts index 47c12f1a..0eac2a71 100644 --- a/packages/cli/src/mcp-http-server.ts +++ b/packages/cli/src/mcp-http-server.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'node:crypto'; import { createServer, type IncomingHttpHeaders, type IncomingMessage, type Server, type ServerResponse } from 'node:http'; -import { loadKtxProject } from './context/project/index.js'; +import { loadKtxProject } from './context/project/project.js'; import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; import { isInitializeRequest } from '@modelcontextprotocol/sdk/types.js'; diff --git a/packages/cli/src/mcp-server-factory.ts b/packages/cli/src/mcp-server-factory.ts index ac627214..2967fb8c 100644 --- a/packages/cli/src/mcp-server-factory.ts +++ b/packages/cli/src/mcp-server-factory.ts @@ -1,7 +1,8 @@ -import { KtxIngestEmbeddingPortAdapter } from './context/index.js'; -import { createDefaultKtxMcpServer, createLocalProjectMcpContextPorts } from './context/mcp/index.js'; -import { createLocalProjectMemoryIngest } from './context/memory/index.js'; -import type { KtxLocalProject } from './context/project/index.js'; +import { KtxIngestEmbeddingPortAdapter } from './context/llm/embedding-port.js'; +import { createDefaultKtxMcpServer } from './context/mcp/server.js'; +import { createLocalProjectMcpContextPorts } from './context/mcp/local-project-ports.js'; +import { createLocalProjectMemoryIngest } from './context/memory/local-memory.js'; +import type { KtxLocalProject } from './context/project/project.js'; import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import type { KtxCliIo } from './cli-runtime.js'; import { resolveProjectEmbeddingProvider } from './embedding-resolution.js'; diff --git a/packages/cli/src/mcp-stdio-server.ts b/packages/cli/src/mcp-stdio-server.ts index a497dc70..d4e21eec 100644 --- a/packages/cli/src/mcp-stdio-server.ts +++ b/packages/cli/src/mcp-stdio-server.ts @@ -1,6 +1,6 @@ import process from 'node:process'; import type { Readable, Writable } from 'node:stream'; -import { loadKtxProject } from './context/project/index.js'; +import { loadKtxProject } from './context/project/project.js'; import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; import type { KtxCliIo } from './cli-runtime.js'; diff --git a/packages/cli/src/memory-flow-hud.tsx b/packages/cli/src/memory-flow-hud.tsx index 28686439..8b044122 100644 --- a/packages/cli/src/memory-flow-hud.tsx +++ b/packages/cli/src/memory-flow-hud.tsx @@ -1,5 +1,5 @@ /* @jsxImportSource react */ -import type { MemoryFlowEvent, MemoryFlowReplayInput } from './context/ingest/memory-flow/index.js'; +import type { MemoryFlowEvent, MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; import { Box, Text } from 'ink'; import { type ReactNode } from 'react'; import { buildDemoMetrics, formatCost, formatDuration } from './demo-metrics.js'; diff --git a/packages/cli/src/memory-flow-interactive.test.ts b/packages/cli/src/memory-flow-interactive.test.ts index 5a08ac65..d6976a03 100644 --- a/packages/cli/src/memory-flow-interactive.test.ts +++ b/packages/cli/src/memory-flow-interactive.test.ts @@ -1,5 +1,5 @@ import { EventEmitter } from 'node:events'; -import type { MemoryFlowReplayInput } from './context/ingest/index.js'; +import type { MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; import { describe, expect, it, vi } from 'vitest'; import { memoryFlowCommandForKey, renderMemoryFlowInteractively } from './memory-flow-interactive.js'; diff --git a/packages/cli/src/memory-flow-interactive.ts b/packages/cli/src/memory-flow-interactive.ts index bb7813dd..b445f025 100644 --- a/packages/cli/src/memory-flow-interactive.ts +++ b/packages/cli/src/memory-flow-interactive.ts @@ -1,13 +1,8 @@ import { emitKeypressEvents } from 'node:readline'; -import { - buildMemoryFlowViewModel, - createInitialMemoryFlowInteractionState, - reduceMemoryFlowInteractionState, - renderMemoryFlowInteractive, - type MemoryFlowInteractionCommand, - type MemoryFlowInteractionState, - type MemoryFlowReplayInput, -} from './context/ingest/index.js'; +import { buildMemoryFlowViewModel } from './context/ingest/memory-flow/view-model.js'; +import { createInitialMemoryFlowInteractionState, reduceMemoryFlowInteractionState } from './context/ingest/memory-flow/interaction.js'; +import { renderMemoryFlowInteractive } from './context/ingest/memory-flow/interactive-render.js'; +import type { MemoryFlowInteractionCommand, MemoryFlowInteractionState, MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; interface KtxMemoryFlowKey { name?: string; diff --git a/packages/cli/src/memory-flow-tui.test.tsx b/packages/cli/src/memory-flow-tui.test.tsx index dd17da1e..09d50125 100644 --- a/packages/cli/src/memory-flow-tui.test.tsx +++ b/packages/cli/src/memory-flow-tui.test.tsx @@ -1,5 +1,5 @@ /* @jsxImportSource react */ -import type { MemoryFlowReplayInput } from './context/ingest/index.js'; +import type { MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js'; import { render as renderInkTest } from 'ink-testing-library'; import React, { type ReactNode } from 'react'; import { describe, expect, it, vi } from 'vitest'; diff --git a/packages/cli/src/memory-flow-tui.tsx b/packages/cli/src/memory-flow-tui.tsx index 3f10970e..5e5176cb 100644 --- a/packages/cli/src/memory-flow-tui.tsx +++ b/packages/cli/src/memory-flow-tui.tsx @@ -1,17 +1,19 @@ /* @jsxImportSource react */ +import { buildMemoryFlowViewModel } from './context/ingest/memory-flow/view-model.js'; import { - buildMemoryFlowViewModel, createInitialMemoryFlowInteractionState, findMemoryFlowSearchMatches, - type MemoryFlowColumnId, - type MemoryFlowInteractionCommand, - type MemoryFlowInteractionState, - type MemoryFlowReplayInput, - type MemoryFlowViewModel, reduceMemoryFlowInteractionState, selectedMemoryFlowColumn, selectedMemoryFlowDetails, -} from './context/ingest/index.js'; +} from './context/ingest/memory-flow/interaction.js'; +import type { + MemoryFlowColumnId, + MemoryFlowInteractionCommand, + MemoryFlowInteractionState, + MemoryFlowReplayInput, + MemoryFlowViewModel, +} from './context/ingest/memory-flow/types.js'; import { Box, Text, render as renderInkRuntime, useApp, useInput } from 'ink'; import { type ReactNode, useEffect, useMemo, useRef, useState } from 'react'; import { diff --git a/packages/cli/src/notion-page-picker.ts b/packages/cli/src/notion-page-picker.ts index b5bd5abf..066acec1 100644 --- a/packages/cli/src/notion-page-picker.ts +++ b/packages/cli/src/notion-page-picker.ts @@ -1,6 +1,6 @@ -import { resolveNotionConnectionAuthToken } from './context/connections/index.js'; -import { type NotionApi, type NotionBotInfo, NotionClient } from './context/ingest/index.js'; -import type { KtxProjectConnectionConfig } from './context/project/index.js'; +import { resolveNotionConnectionAuthToken } from './context/connections/notion-config.js'; +import { type NotionApi, type NotionBotInfo, NotionClient } from './context/ingest/adapters/notion/notion-client.js'; +import type { KtxProjectConnectionConfig } from './context/project/config.js'; import type { KtxCliIo } from './cli-runtime.js'; import { profileMark } from './startup-profile.js'; import { diff --git a/packages/cli/src/public-ingest.test.ts b/packages/cli/src/public-ingest.test.ts index a989986f..ced91a92 100644 --- a/packages/cli/src/public-ingest.test.ts +++ b/packages/cli/src/public-ingest.test.ts @@ -1,4 +1,4 @@ -import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/index.js'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; import { describe, expect, it, vi } from 'vitest'; import { buildPublicIngestPlan, diff --git a/packages/cli/src/public-ingest.ts b/packages/cli/src/public-ingest.ts index 7a9b6ace..a88a342d 100644 --- a/packages/cli/src/public-ingest.ts +++ b/packages/cli/src/public-ingest.ts @@ -1,5 +1,6 @@ -import { loadKtxProject, type KtxLocalProject, type KtxProjectConnectionConfig } from './context/project/index.js'; -import type { KtxProgressPort } from './context/scan/index.js'; +import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; +import type { KtxProjectConnectionConfig } from './context/project/config.js'; +import type { KtxProgressPort } from './context/scan/types.js'; import type { KtxCliIo } from './index.js'; import type { KtxIngestArgs, KtxIngestDeps, KtxIngestProgressUpdate } from './ingest.js'; import { diff --git a/packages/cli/src/runtime-requirements.test.ts b/packages/cli/src/runtime-requirements.test.ts index 8681ed64..8eec1116 100644 --- a/packages/cli/src/runtime-requirements.test.ts +++ b/packages/cli/src/runtime-requirements.test.ts @@ -1,4 +1,4 @@ -import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/index.js'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; import { describe, expect, it } from 'vitest'; import { resolveProjectRuntimeRequirements, diff --git a/packages/cli/src/runtime-requirements.ts b/packages/cli/src/runtime-requirements.ts index b6751394..df1c2a56 100644 --- a/packages/cli/src/runtime-requirements.ts +++ b/packages/cli/src/runtime-requirements.ts @@ -1,8 +1,4 @@ -import type { - KtxProjectConfig, - KtxProjectConnectionConfig, - KtxProjectEmbeddingConfig, -} from './context/project/index.js'; +import type { KtxProjectConfig, KtxProjectConnectionConfig, KtxProjectEmbeddingConfig } from './context/project/config.js'; import type { KtxRuntimeFeature } from './managed-python-runtime.js'; import type { KtxPublicIngestPlan } from './public-ingest.js'; diff --git a/packages/cli/src/scan.test.ts b/packages/cli/src/scan.test.ts index 13b07920..fafecd97 100644 --- a/packages/cli/src/scan.test.ts +++ b/packages/cli/src/scan.test.ts @@ -1,13 +1,10 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import type { SourceAdapter } from './context/ingest/index.js'; -import { initKtxProject } from './context/project/index.js'; -import type { - KtxScanReport, - LocalScanRunResult, - RunLocalScanOptions, -} from './context/scan/index.js'; +import type { SourceAdapter } from './context/ingest/types.js'; +import { initKtxProject } from './context/project/project.js'; +import type { KtxScanReport } from './context/scan/types.js'; +import type { LocalScanRunResult, RunLocalScanOptions } from './context/scan/local-scan.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { createCliScanProgress, runKtxScan, type KtxScanDeps } from './scan.js'; @@ -138,29 +135,37 @@ const KtxPostgresScanConnector = vi.hoisted( }, ); -vi.mock('./connectors/sqlserver/index.js', () => ({ - createSqlServerLiveDatabaseIntrospection, +vi.mock('./connectors/sqlserver/connector.js', () => ({ isKtxSqlServerConnectionConfig, KtxSqlServerScanConnector, })); +vi.mock('./connectors/sqlserver/live-database-introspection.js', () => ({ + createSqlServerLiveDatabaseIntrospection, +})); -vi.mock('./connectors/bigquery/index.js', () => ({ - createBigQueryLiveDatabaseIntrospection, +vi.mock('./connectors/bigquery/connector.js', () => ({ isKtxBigQueryConnectionConfig, KtxBigQueryScanConnector, })); +vi.mock('./connectors/bigquery/live-database-introspection.js', () => ({ + createBigQueryLiveDatabaseIntrospection, +})); -vi.mock('./connectors/snowflake/index.js', () => ({ - createSnowflakeLiveDatabaseIntrospection, +vi.mock('./connectors/snowflake/connector.js', () => ({ isKtxSnowflakeConnectionConfig, KtxSnowflakeScanConnector, })); +vi.mock('./connectors/snowflake/live-database-introspection.js', () => ({ + createSnowflakeLiveDatabaseIntrospection, +})); -vi.mock('./connectors/postgres/index.js', () => ({ - createPostgresLiveDatabaseIntrospection, +vi.mock('./connectors/postgres/connector.js', () => ({ isKtxPostgresConnectionConfig, KtxPostgresScanConnector, })); +vi.mock('./connectors/postgres/live-database-introspection.js', () => ({ + createPostgresLiveDatabaseIntrospection, +})); function makeIo(options: { isTTY?: boolean } = {}) { let stdout = ''; diff --git a/packages/cli/src/scan.ts b/packages/cli/src/scan.ts index 9b48aba0..393111b6 100644 --- a/packages/cli/src/scan.ts +++ b/packages/cli/src/scan.ts @@ -1,11 +1,6 @@ -import { - type KtxProgressPort, - type KtxScanMode, - type KtxScanReport, - type KtxScanWarning, - runLocalScan, -} from './context/scan/index.js'; -import { loadKtxProject } from './context/project/index.js'; +import type { KtxProgressPort, KtxScanMode, KtxScanReport, KtxScanWarning } from './context/scan/types.js'; +import { runLocalScan } from './context/scan/local-scan.js'; +import { loadKtxProject } from './context/project/project.js'; import { resolveProjectEmbeddingProvider } from './embedding-resolution.js'; import type { KtxCliIo } from './index.js'; import { createKtxCliLocalIngestAdapters } from './local-adapters.js'; diff --git a/packages/cli/src/setup-agents.test.ts b/packages/cli/src/setup-agents.test.ts index a09ba66b..52d0cdff 100644 --- a/packages/cli/src/setup-agents.test.ts +++ b/packages/cli/src/setup-agents.test.ts @@ -1,7 +1,7 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { readKtxSetupState } from './context/project/index.js'; +import { readKtxSetupState } from './context/project/setup-config.js'; import { strFromU8, unzipSync } from 'fflate'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { diff --git a/packages/cli/src/setup-agents.ts b/packages/cli/src/setup-agents.ts index 6a389257..240622f6 100644 --- a/packages/cli/src/setup-agents.ts +++ b/packages/cli/src/setup-agents.ts @@ -5,11 +5,9 @@ import type { Writable } from 'node:stream'; import { fileURLToPath } from 'node:url'; import { styleText } from 'node:util'; import { log, outro } from '@clack/prompts'; -import { - loadKtxProject, - markKtxSetupStateStepComplete, - serializeKtxProjectConfig, -} from './context/project/index.js'; +import { loadKtxProject } from './context/project/project.js'; +import { markKtxSetupStateStepComplete } from './context/project/setup-config.js'; +import { serializeKtxProjectConfig } from './context/project/config.js'; import { strToU8, zipSync } from 'fflate'; import type { KtxCliIo } from './cli-runtime.js'; import { diff --git a/packages/cli/src/setup-context.test.ts b/packages/cli/src/setup-context.test.ts index 3ea49414..7bcad93e 100644 --- a/packages/cli/src/setup-context.test.ts +++ b/packages/cli/src/setup-context.test.ts @@ -1,14 +1,8 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { - buildDefaultKtxProjectConfig, - parseKtxProjectConfig, - readKtxSetupState, - serializeKtxProjectConfig, - type KtxProjectConfig, - writeKtxSetupState, -} from './context/project/index.js'; +import { buildDefaultKtxProjectConfig, parseKtxProjectConfig, serializeKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; +import { readKtxSetupState, writeKtxSetupState } from './context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { diff --git a/packages/cli/src/setup-context.ts b/packages/cli/src/setup-context.ts index f3da7325..aa519111 100644 --- a/packages/cli/src/setup-context.ts +++ b/packages/cli/src/setup-context.ts @@ -1,13 +1,9 @@ import { mkdirSync, writeFileSync } from 'node:fs'; import { access, mkdir, readdir, readFile, writeFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; -import { - type KtxLocalProject, - loadKtxProject, - markKtxSetupStateStepComplete, - readKtxSetupState, - serializeKtxProjectConfig, -} from './context/project/index.js'; +import { type KtxLocalProject, loadKtxProject } from './context/project/project.js'; +import { markKtxSetupStateStepComplete, readKtxSetupState } from './context/project/setup-config.js'; +import { serializeKtxProjectConfig } from './context/project/config.js'; import type { KtxCliIo } from './cli-runtime.js'; import { buildPublicIngestPlan } from './public-ingest.js'; import { diff --git a/packages/cli/src/setup-database-context-depth.ts b/packages/cli/src/setup-database-context-depth.ts index d62363a6..20df813c 100644 --- a/packages/cli/src/setup-database-context-depth.ts +++ b/packages/cli/src/setup-database-context-depth.ts @@ -1,10 +1,6 @@ import { writeFile } from 'node:fs/promises'; -import { - type KtxLocalProject, - type KtxProjectConnectionConfig, - loadKtxProject, - serializeKtxProjectConfig, -} from './context/project/index.js'; +import { type KtxLocalProject, loadKtxProject } from './context/project/project.js'; +import { type KtxProjectConnectionConfig, serializeKtxProjectConfig } from './context/project/config.js'; import { type KtxDatabaseContextDepth, databaseContextDepth, diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/src/setup-databases.test.ts index 2ca77bee..178ed076 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/src/setup-databases.test.ts @@ -1,7 +1,9 @@ import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; -import { initKtxProject, parseKtxProjectConfig, readKtxSetupState, writeKtxSetupState } from './context/project/index.js'; +import { initKtxProject } from './context/project/project.js'; +import { parseKtxProjectConfig } from './context/project/config.js'; +import { readKtxSetupState, writeKtxSetupState } from './context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type KtxSetupDatabaseDriver, diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index dd7a28ad..c8e735c5 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -3,15 +3,11 @@ import { readFile, writeFile } from 'node:fs/promises'; import { delimiter, dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; -import type { HistoricSqlDialect } from './context/ingest/index.js'; -import { - type KtxProjectConnectionConfig, - loadKtxProject, - markKtxSetupStateStepComplete, - serializeKtxProjectConfig, - setKtxSetupDatabaseConnectionIds, -} from './context/project/index.js'; -import type { KtxTableListEntry } from './context/scan/index.js'; +import type { HistoricSqlDialect } from './context/ingest/adapters/historic-sql/types.js'; +import { type KtxProjectConnectionConfig, serializeKtxProjectConfig } from './context/project/config.js'; +import { loadKtxProject } from './context/project/project.js'; +import { markKtxSetupStateStepComplete, setKtxSetupDatabaseConnectionIds } from './context/project/setup-config.js'; +import type { KtxTableListEntry } from './context/scan/types.js'; import type { KtxCliIo } from './cli-runtime.js'; import { runKtxConnection } from './connection.js'; import { @@ -328,8 +324,12 @@ async function defaultHistoricSqlProbe(input: KtxSetupHistoricSqlProbeInput): Pr const project = await loadKtxProject({ projectDir: input.projectDir }); const connection = project.config.connections[input.connectionId]; - const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient, isKtxPostgresConnectionConfig }] = - await Promise.all([import('./context/ingest/index.js'), import('./connectors/postgres/index.js')]); + const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient }, { isKtxPostgresConnectionConfig }] = + await Promise.all([ + import('./context/ingest/adapters/historic-sql/postgres-pgss-reader.js'), + import('./connectors/postgres/historic-sql-query-client.js'), + import('./connectors/postgres/connector.js'), + ]); const postgresConnection = connection as Parameters[0]; if (!isKtxPostgresConnectionConfig(postgresConnection)) { @@ -365,7 +365,7 @@ async function defaultListSchemas(projectDir: string, connectionId: string): Pro const driver = normalizeDriver(connection?.driver); if (driver === 'postgres') { - const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('./connectors/postgres/index.js'); + const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('./connectors/postgres/connector.js');; if (!isKtxPostgresConnectionConfig(connection)) return []; const connector = new KtxPostgresScanConnector({ connectionId, connection }); try { @@ -376,7 +376,7 @@ async function defaultListSchemas(projectDir: string, connectionId: string): Pro } if (driver === 'sqlserver') { - const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('./connectors/sqlserver/index.js'); + const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('./connectors/sqlserver/connector.js');; if (!isKtxSqlServerConnectionConfig(connection)) return []; const connector = new KtxSqlServerScanConnector({ connectionId, connection }); try { @@ -387,7 +387,7 @@ async function defaultListSchemas(projectDir: string, connectionId: string): Pro } if (driver === 'bigquery') { - const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('./connectors/bigquery/index.js'); + const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('./connectors/bigquery/connector.js');; if (!isKtxBigQueryConnectionConfig(connection)) return []; const connector = new KtxBigQueryScanConnector({ connectionId, connection }); try { @@ -398,7 +398,7 @@ async function defaultListSchemas(projectDir: string, connectionId: string): Pro } if (driver === 'snowflake') { - const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/index.js'); + const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/connector.js');; if (!isKtxSnowflakeConnectionConfig(connection)) return []; const connector = new KtxSnowflakeScanConnector({ connectionId, connection }); try { @@ -430,7 +430,7 @@ async function defaultListTables( const schemas = schemasOverride ?? (driver ? configuredSchemas(connection, driver) : undefined); if (driver === 'postgres') { - const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('./connectors/postgres/index.js'); + const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('./connectors/postgres/connector.js');; if (!isKtxPostgresConnectionConfig(connection)) return []; const connector = new KtxPostgresScanConnector({ connectionId, connection }); try { @@ -441,7 +441,7 @@ async function defaultListTables( } if (driver === 'mysql') { - const { KtxMysqlScanConnector, isKtxMysqlConnectionConfig } = await import('./connectors/mysql/index.js'); + const { KtxMysqlScanConnector, isKtxMysqlConnectionConfig } = await import('./connectors/mysql/connector.js');; if (!isKtxMysqlConnectionConfig(connection)) return []; const connector = new KtxMysqlScanConnector({ connectionId, connection }); try { @@ -452,7 +452,7 @@ async function defaultListTables( } if (driver === 'sqlserver') { - const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('./connectors/sqlserver/index.js'); + const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('./connectors/sqlserver/connector.js');; if (!isKtxSqlServerConnectionConfig(connection)) return []; const connector = new KtxSqlServerScanConnector({ connectionId, connection }); try { @@ -463,7 +463,7 @@ async function defaultListTables( } if (driver === 'bigquery') { - const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('./connectors/bigquery/index.js'); + const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('./connectors/bigquery/connector.js');; if (!isKtxBigQueryConnectionConfig(connection)) return []; const connector = new KtxBigQueryScanConnector({ connectionId, connection }); try { @@ -474,7 +474,7 @@ async function defaultListTables( } if (driver === 'snowflake') { - const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/index.js'); + const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/connector.js');; if (!isKtxSnowflakeConnectionConfig(connection)) return []; const connector = new KtxSnowflakeScanConnector({ connectionId, connection }); try { @@ -485,7 +485,7 @@ async function defaultListTables( } if (driver === 'clickhouse') { - const { KtxClickHouseScanConnector, isKtxClickHouseConnectionConfig } = await import('./connectors/clickhouse/index.js'); + const { KtxClickHouseScanConnector, isKtxClickHouseConnectionConfig } = await import('./connectors/clickhouse/connector.js');; if (!isKtxClickHouseConnectionConfig(connection)) return []; const connector = new KtxClickHouseScanConnector({ connectionId, connection }); try { diff --git a/packages/cli/src/setup-embeddings.test.ts b/packages/cli/src/setup-embeddings.test.ts index 25ba81c3..bf9e2b2d 100644 --- a/packages/cli/src/setup-embeddings.test.ts +++ b/packages/cli/src/setup-embeddings.test.ts @@ -1,7 +1,9 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject, parseKtxProjectConfig, readKtxSetupState, writeKtxSetupState } from './context/project/index.js'; +import { initKtxProject } from './context/project/project.js'; +import { parseKtxProjectConfig } from './context/project/config.js'; +import { readKtxSetupState, writeKtxSetupState } from './context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type KtxSetupEmbeddingsPromptAdapter, runKtxSetupEmbeddingsStep } from './setup-embeddings.js'; diff --git a/packages/cli/src/setup-embeddings.ts b/packages/cli/src/setup-embeddings.ts index b95ea1d5..0aedd264 100644 --- a/packages/cli/src/setup-embeddings.ts +++ b/packages/cli/src/setup-embeddings.ts @@ -1,14 +1,10 @@ import { readFile, writeFile } from 'node:fs/promises'; -import { resolveKtxConfigReference } from './context/core/index.js'; -import { - type KtxProjectConfig, - type KtxProjectEmbeddingConfig, - loadKtxProject, - markKtxSetupStateStepComplete, - readKtxSetupState, - serializeKtxProjectConfig, -} from './context/project/index.js'; -import { type KtxEmbeddingConfig, type KtxEmbeddingHealthCheckResult, runKtxEmbeddingHealthCheck } from './llm/index.js'; +import { resolveKtxConfigReference } from './context/core/config-reference.js'; +import { type KtxProjectConfig, type KtxProjectEmbeddingConfig, serializeKtxProjectConfig } from './context/project/config.js'; +import { loadKtxProject } from './context/project/project.js'; +import { markKtxSetupStateStepComplete, readKtxSetupState } from './context/project/setup-config.js'; +import type { KtxEmbeddingConfig } from './llm/types.js'; +import { type KtxEmbeddingHealthCheckResult, runKtxEmbeddingHealthCheck } from './llm/embedding-health.js'; import type { KtxCliIo } from './cli-runtime.js'; import { createStaticCliSpinner, type KtxCliSpinner } from './clack.js'; import { diff --git a/packages/cli/src/setup-models.test.ts b/packages/cli/src/setup-models.test.ts index 04e96b7a..ba9260be 100644 --- a/packages/cli/src/setup-models.test.ts +++ b/packages/cli/src/setup-models.test.ts @@ -1,7 +1,9 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject, parseKtxProjectConfig, readKtxSetupState, writeKtxSetupState } from './context/project/index.js'; +import { initKtxProject } from './context/project/project.js'; +import { parseKtxProjectConfig } from './context/project/config.js'; +import { readKtxSetupState, writeKtxSetupState } from './context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { BUNDLED_ANTHROPIC_MODELS, diff --git a/packages/cli/src/setup-models.ts b/packages/cli/src/setup-models.ts index 818b12ab..7f7385b1 100644 --- a/packages/cli/src/setup-models.ts +++ b/packages/cli/src/setup-models.ts @@ -1,16 +1,14 @@ import { execFile } from 'node:child_process'; import { writeFile } from 'node:fs/promises'; import { promisify } from 'node:util'; -import { resolveLocalKtxLlmConfig, runClaudeCodeAuthProbe } from './context/index.js'; -import { resolveKtxConfigReference } from './context/core/index.js'; -import { - type KtxProjectConfig, - type KtxProjectLlmConfig, - loadKtxProject, - markKtxSetupStateStepComplete, - serializeKtxProjectConfig, -} from './context/project/index.js'; -import { type KtxLlmConfig, type KtxLlmHealthCheckResult, runKtxLlmHealthCheck } from './llm/index.js'; +import { resolveLocalKtxLlmConfig } from './context/llm/local-config.js'; +import { runClaudeCodeAuthProbe } from './context/llm/claude-code-runtime.js'; +import { resolveKtxConfigReference } from './context/core/config-reference.js'; +import { type KtxProjectConfig, type KtxProjectLlmConfig, serializeKtxProjectConfig } from './context/project/config.js'; +import { loadKtxProject } from './context/project/project.js'; +import { markKtxSetupStateStepComplete } from './context/project/setup-config.js'; +import type { KtxLlmConfig } from './llm/types.js'; +import { type KtxLlmHealthCheckResult, runKtxLlmHealthCheck } from './llm/model-health.js'; import { formatClaudeCodePromptCachingWarning, ignoredClaudeCodePromptCachingFields, diff --git a/packages/cli/src/setup-project.test.ts b/packages/cli/src/setup-project.test.ts index 9c64b456..26b1e35b 100644 --- a/packages/cli/src/setup-project.test.ts +++ b/packages/cli/src/setup-project.test.ts @@ -1,7 +1,9 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject, parseKtxProjectConfig, readKtxSetupState } from './context/project/index.js'; +import { initKtxProject } from './context/project/project.js'; +import { parseKtxProjectConfig } from './context/project/config.js'; +import { readKtxSetupState } from './context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { gray } from './io/symbols.js'; import { type KtxSetupProjectPromptAdapter, runKtxSetupProjectStep } from './setup-project.js'; diff --git a/packages/cli/src/setup-project.ts b/packages/cli/src/setup-project.ts index f5fa5c5c..d7d189e1 100644 --- a/packages/cli/src/setup-project.ts +++ b/packages/cli/src/setup-project.ts @@ -2,14 +2,9 @@ import { existsSync } from 'node:fs'; import { mkdir, readdir, readFile, stat, writeFile } from 'node:fs/promises'; import { homedir } from 'node:os'; import { join, resolve } from 'node:path'; -import { - initKtxProject, - type KtxLocalProject, - loadKtxProject, - markKtxSetupStateStepComplete, - mergeKtxSetupGitignoreEntries, - serializeKtxProjectConfig, -} from './context/project/index.js'; +import { initKtxProject, type KtxLocalProject, loadKtxProject } from './context/project/project.js'; +import { markKtxSetupStateStepComplete, mergeKtxSetupGitignoreEntries } from './context/project/setup-config.js'; +import { serializeKtxProjectConfig } from './context/project/config.js'; import type { KtxCliIo } from './cli-runtime.js'; import { gray } from './io/symbols.js'; import { withTextInputNavigation } from './prompt-navigation.js'; diff --git a/packages/cli/src/setup-runtime.test.ts b/packages/cli/src/setup-runtime.test.ts index 0f2925fa..2fb1f1f2 100644 --- a/packages/cli/src/setup-runtime.test.ts +++ b/packages/cli/src/setup-runtime.test.ts @@ -1,7 +1,8 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { buildDefaultKtxProjectConfig, readKtxSetupState, type KtxProjectConfig } from './context/project/index.js'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; +import { readKtxSetupState } from './context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runKtxSetupRuntimeStep } from './setup-runtime.js'; diff --git a/packages/cli/src/setup-runtime.ts b/packages/cli/src/setup-runtime.ts index f631c457..25612065 100644 --- a/packages/cli/src/setup-runtime.ts +++ b/packages/cli/src/setup-runtime.ts @@ -1,8 +1,5 @@ -import { - loadKtxProject, - markKtxSetupStateStepComplete, - type KtxLocalProject, -} from './context/project/index.js'; +import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; +import { markKtxSetupStateStepComplete } from './context/project/setup-config.js'; import type { KtxCliIo } from './cli-runtime.js'; import { ensureManagedLocalEmbeddingsDaemon, diff --git a/packages/cli/src/setup-sources-notion.test.ts b/packages/cli/src/setup-sources-notion.test.ts index a12eba49..84fce4aa 100644 --- a/packages/cli/src/setup-sources-notion.test.ts +++ b/packages/cli/src/setup-sources-notion.test.ts @@ -1,12 +1,8 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { - initKtxProject, - type KtxProjectConnectionConfig, - parseKtxProjectConfig, - serializeKtxProjectConfig, -} from './context/project/index.js'; +import { initKtxProject } from './context/project/project.js'; +import { type KtxProjectConnectionConfig, parseKtxProjectConfig, serializeKtxProjectConfig } from './context/project/config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runKtxSetupSourcesStep, @@ -19,8 +15,8 @@ const notionMocks = vi.hoisted(() => ({ retrievePage: vi.fn(async () => ({ id: 'page-1' })), })); -vi.mock('./context/ingest/index.js', async (importOriginal) => { - const actual = await importOriginal(); +vi.mock('./context/ingest/adapters/notion/notion-client.js', async (importOriginal) => { + const actual = await importOriginal(); return { ...actual, NotionClient: vi.fn().mockImplementation(function NotionClient(token: string) { diff --git a/packages/cli/src/setup-sources.test.ts b/packages/cli/src/setup-sources.test.ts index 7ab345c7..a9de4436 100644 --- a/packages/cli/src/setup-sources.test.ts +++ b/packages/cli/src/setup-sources.test.ts @@ -1,13 +1,9 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { - initKtxProject, - type KtxProjectConnectionConfig, - parseKtxProjectConfig, - readKtxSetupState, - serializeKtxProjectConfig, -} from './context/project/index.js'; +import { initKtxProject } from './context/project/project.js'; +import { type KtxProjectConnectionConfig, parseKtxProjectConfig, serializeKtxProjectConfig } from './context/project/config.js'; +import { readKtxSetupState } from './context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { KtxCliIo } from './cli-runtime.js'; import { diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index f4042037..ff8eb420 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -2,32 +2,20 @@ import { mkdtemp, readdir, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, relative, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; -import { - localConnectionTypeForConfig, - resolveNotionConnectionAuthToken, -} from './context/connections/index.js'; -import { resolveKtxConfigReference } from './context/core/index.js'; -import { - cloneOrPull, - DEFAULT_METABASE_CLIENT_CONFIG, - discoverMetabaseDatabases, - type DiscoveredMetabaseDatabase, - loadDbtSchemaFiles, - loadProjectInfo, - MetabaseClient, - type NotionApi, - NotionClient, - parseLookmlStagedDir, - parseMetricflowFiles, - testRepoConnection, -} from './context/ingest/index.js'; -import { - type KtxProjectConfig, - type KtxProjectConnectionConfig, - loadKtxProject, - markKtxSetupStateStepComplete, - serializeKtxProjectConfig, -} from './context/project/index.js'; +import { localConnectionTypeForConfig } from './context/connections/local-warehouse-descriptor.js'; +import { resolveNotionConnectionAuthToken } from './context/connections/notion-config.js'; +import { resolveKtxConfigReference } from './context/core/config-reference.js'; +import { cloneOrPull, testRepoConnection } from './context/ingest/repo-fetch.js'; +import { DEFAULT_METABASE_CLIENT_CONFIG, MetabaseClient } from './context/ingest/adapters/metabase/client.js'; +import { discoverMetabaseDatabases, type DiscoveredMetabaseDatabase } from './context/ingest/adapters/metabase/mapping.js'; +import { loadDbtSchemaFiles } from './context/ingest/dbt-shared/schema-files.js'; +import { loadProjectInfo } from './context/ingest/dbt-shared/project-vars.js'; +import { type NotionApi, NotionClient } from './context/ingest/adapters/notion/notion-client.js'; +import { parseLookmlStagedDir } from './context/ingest/adapters/lookml/parse.js'; +import { parseMetricflowFiles } from './context/ingest/adapters/metricflow/deep-parse.js'; +import { type KtxProjectConfig, type KtxProjectConnectionConfig, serializeKtxProjectConfig } from './context/project/config.js'; +import { loadKtxProject } from './context/project/project.js'; +import { markKtxSetupStateStepComplete } from './context/project/setup-config.js'; import type { KtxCliIo } from './cli-runtime.js'; import { pickNotionRootPages } from './notion-page-picker.js'; import { runKtxSourceMapping } from './source-mapping.js'; diff --git a/packages/cli/src/setup.test.ts b/packages/cli/src/setup.test.ts index bef547bc..9c4a3b58 100644 --- a/packages/cli/src/setup.test.ts +++ b/packages/cli/src/setup.test.ts @@ -3,7 +3,7 @@ import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; -import { writeKtxSetupState } from './context/project/index.js'; +import { writeKtxSetupState } from './context/project/setup-config.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { localFakeBundleReport, persistLocalBundleReport } from './ingest.test-utils.js'; diff --git a/packages/cli/src/setup.ts b/packages/cli/src/setup.ts index 88aef227..9b4c3a24 100644 --- a/packages/cli/src/setup.ts +++ b/packages/cli/src/setup.ts @@ -1,13 +1,11 @@ import { existsSync } from 'node:fs'; import { rm } from 'node:fs/promises'; import { basename, join, resolve } from 'node:path'; -import { getLatestLocalIngestStatus, savedMemoryCountsForReport } from './context/ingest/index.js'; -import { - ktxLocalStateDbPath, - loadKtxProject, - readKtxSetupState, - type KtxLocalProject, -} from './context/project/index.js'; +import { getLatestLocalIngestStatus } from './context/ingest/local-ingest.js'; +import { savedMemoryCountsForReport } from './context/ingest/reports.js'; +import { ktxLocalStateDbPath } from './context/project/local-state-db.js'; +import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; +import { readKtxSetupState } from './context/project/setup-config.js'; import type { KtxCliIo } from './cli-runtime.js'; import { formatSetupNextStepLines } from './next-steps.js'; import { runtimeInstallPolicyFromFlags } from './managed-python-command.js'; diff --git a/packages/cli/src/sl.test.ts b/packages/cli/src/sl.test.ts index 6def5177..f1fe0b89 100644 --- a/packages/cli/src/sl.test.ts +++ b/packages/cli/src/sl.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { stripVTControlCharacters } from 'node:util'; import Database from 'better-sqlite3'; -import { initKtxProject } from './context/project/index.js'; +import { initKtxProject } from './context/project/project.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runKtxSl } from './sl.js'; diff --git a/packages/cli/src/sl.ts b/packages/cli/src/sl.ts index 05d7f523..0e5e0fc0 100644 --- a/packages/cli/src/sl.ts +++ b/packages/cli/src/sl.ts @@ -1,18 +1,13 @@ import { readFile } from 'node:fs/promises'; -import { createDefaultLocalQueryExecutor, type KtxSqlQueryExecutorPort } from './context/connections/index.js'; -import { KtxIngestEmbeddingPortAdapter, type KtxEmbeddingPort } from './context/index.js'; -import type { KtxSemanticLayerComputePort } from './context/daemon/index.js'; -import { loadKtxProject, type KtxLocalProject } from './context/project/index.js'; -import { - compileLocalSlQuery, - listLocalSlSources, - readLocalSlSource, - searchLocalSlSources as defaultSearchLocalSlSources, - validateLocalSlSource, - type LocalSlSourceSearchResult, - type LocalSlSourceSummary, - type SemanticLayerQueryInput, -} from './context/sl/index.js'; +import { createDefaultLocalQueryExecutor } from './context/connections/local-query-executor.js'; +import type { KtxSqlQueryExecutorPort } from './context/connections/query-executor.js'; +import { KtxIngestEmbeddingPortAdapter } from './context/llm/embedding-port.js'; +import type { KtxEmbeddingPort } from './context/core/embedding.js'; +import type { KtxSemanticLayerComputePort } from './context/daemon/semantic-layer-compute.js'; +import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; +import { compileLocalSlQuery } from './context/sl/local-query.js'; +import { listLocalSlSources, readLocalSlSource, searchLocalSlSources as defaultSearchLocalSlSources, validateLocalSlSource, type LocalSlSourceSearchResult, type LocalSlSourceSummary } from './context/sl/local-sl.js'; +import type { SemanticLayerQueryInput } from './context/sl/types.js'; import { resolveProjectEmbeddingProvider, type EmbeddingProviderResolution, diff --git a/packages/cli/src/source-mapping.ts b/packages/cli/src/source-mapping.ts index 146bf8e3..5227ceab 100644 --- a/packages/cli/src/source-mapping.ts +++ b/packages/cli/src/source-mapping.ts @@ -1,26 +1,17 @@ -import { localConnectionToWarehouseDescriptor } from './context/connections/index.js'; -import { - DEFAULT_METABASE_CLIENT_CONFIG, - DefaultLookerConnectionClientFactory, - DefaultMetabaseConnectionClientFactory, - KtxYamlMetabaseSourceStateReader, - LocalLookerRuntimeStore, - LocalMetabaseDiscoveryCache, - computeLookerMappingDrift, - computeMetabaseMappingDrift, - discoverLookerConnections, - discoverMetabaseDatabases, - lookerCredentialsFromLocalConnection, - metabaseRuntimeConfigFromLocalConnection, - planMetabaseFanoutChildren, - seedLocalMappingStateFromKtxYaml, - validateLookerMappings, - validateMappingPhysicalMatch, - type LookerMappingClient, - type LocalMetabaseMappingListRow, - type MetabaseRuntimeClient, -} from './context/ingest/index.js'; -import { type KtxLocalProject, ktxLocalStateDbPath, loadKtxProject } from './context/project/index.js'; +import { localConnectionToWarehouseDescriptor } from './context/connections/local-warehouse-descriptor.js'; +import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory } from './context/ingest/adapters/metabase/client.js'; +import { DefaultLookerConnectionClientFactory } from './context/ingest/adapters/looker/factory.js'; +import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache, type LocalMetabaseMappingListRow } from './context/ingest/adapters/metabase/local-source-state-store.js'; +import { LocalLookerRuntimeStore } from './context/ingest/adapters/looker/local-runtime-store.js'; +import { computeLookerMappingDrift, discoverLookerConnections, validateLookerMappings, type LookerMappingClient } from './context/ingest/adapters/looker/mapping.js'; +import { computeMetabaseMappingDrift, discoverMetabaseDatabases, validateMappingPhysicalMatch } from './context/ingest/adapters/metabase/mapping.js'; +import { lookerCredentialsFromLocalConnection } from './context/ingest/adapters/looker/local-looker.adapter.js'; +import { metabaseRuntimeConfigFromLocalConnection } from './context/ingest/adapters/metabase/local-metabase.adapter.js'; +import { planMetabaseFanoutChildren } from './context/ingest/adapters/metabase/fanout-planner.js'; +import { seedLocalMappingStateFromKtxYaml } from './context/ingest/local-mapping-reconcile.js'; +import type { MetabaseRuntimeClient } from './context/ingest/adapters/metabase/client-port.js'; +import { type KtxLocalProject, loadKtxProject } from './context/project/project.js'; +import { ktxLocalStateDbPath } from './context/project/local-state-db.js'; import type { KtxCliIo } from './cli-runtime.js'; import { profileMark } from './startup-profile.js'; diff --git a/packages/cli/src/sql.test.ts b/packages/cli/src/sql.test.ts index c3e95885..5317b2a8 100644 --- a/packages/cli/src/sql.test.ts +++ b/packages/cli/src/sql.test.ts @@ -1,9 +1,10 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { initKtxProject, parseKtxProjectConfig, serializeKtxProjectConfig } from './context/project/index.js'; -import type { KtxScanConnector } from './context/scan/index.js'; -import type { SqlAnalysisPort } from './context/sql-analysis/index.js'; +import { initKtxProject } from './context/project/project.js'; +import { parseKtxProjectConfig, serializeKtxProjectConfig } from './context/project/config.js'; +import type { KtxScanConnector } from './context/scan/types.js'; +import type { SqlAnalysisPort } from './context/sql-analysis/ports.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runKtxSql } from './sql.js'; diff --git a/packages/cli/src/sql.ts b/packages/cli/src/sql.ts index e19f9d1b..833df78e 100644 --- a/packages/cli/src/sql.ts +++ b/packages/cli/src/sql.ts @@ -1,6 +1,6 @@ -import { loadKtxProject, type KtxLocalProject } from './context/project/index.js'; -import type { KtxQueryResult, KtxScanConnector } from './context/scan/index.js'; -import type { SqlAnalysisDialect, SqlAnalysisPort } from './context/sql-analysis/index.js'; +import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; +import type { KtxQueryResult, KtxScanConnector } from './context/scan/types.js'; +import type { SqlAnalysisDialect, SqlAnalysisPort } from './context/sql-analysis/ports.js'; import type { KtxCliIo } from './cli-runtime.js'; import { type KtxOutputMode, resolveOutputMode } from './io/mode.js'; import { createKtxCliScanConnector } from './local-scan-connectors.js'; diff --git a/packages/cli/src/standalone-smoke.test.ts b/packages/cli/src/standalone-smoke.test.ts index 196aeb72..7e6ed56e 100644 --- a/packages/cli/src/standalone-smoke.test.ts +++ b/packages/cli/src/standalone-smoke.test.ts @@ -3,7 +3,7 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { promisify } from 'node:util'; -import { parseKtxProjectConfig } from './context/project/index.js'; +import { parseKtxProjectConfig } from './context/project/config.js'; import Database from 'better-sqlite3'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; diff --git a/packages/cli/src/status-project.test.ts b/packages/cli/src/status-project.test.ts index 816a5c8d..7f79e724 100644 --- a/packages/cli/src/status-project.test.ts +++ b/packages/cli/src/status-project.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { buildDefaultKtxProjectConfig, type KtxLocalProject, type KtxProjectConfig } from './context/project/index.js'; +import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js'; +import type { KtxLocalProject } from './context/project/project.js'; import { buildProjectStatus } from './status-project.js'; function projectWithConfig(config: KtxProjectConfig): KtxLocalProject { diff --git a/packages/cli/src/status-project.ts b/packages/cli/src/status-project.ts index e430dbb9..f84ebe20 100644 --- a/packages/cli/src/status-project.ts +++ b/packages/cli/src/status-project.ts @@ -1,14 +1,8 @@ import { basename } from 'node:path'; -import { runClaudeCodeAuthProbe } from './context/index.js'; -import type { - KtxConfigIssue, - KtxLocalProject, - KtxProjectConfig, - KtxProjectConnectionConfig, - KtxProjectEmbeddingConfig, - KtxProjectLlmConfig, -} from './context/project/index.js'; -import type { PostgresPgssProbeResult } from './context/ingest/index.js'; +import { runClaudeCodeAuthProbe } from './context/llm/claude-code-runtime.js'; +import type { KtxConfigIssue, KtxProjectConfig, KtxProjectConnectionConfig, KtxProjectEmbeddingConfig, KtxProjectLlmConfig } from './context/project/config.js'; +import type { KtxLocalProject } from './context/project/project.js'; +import type { PostgresPgssProbeResult } from './context/ingest/adapters/historic-sql/types.js'; import { formatClaudeCodePromptCachingFix, formatClaudeCodePromptCachingWarning, @@ -418,8 +412,12 @@ function readinessDetail(result: PostgresPgssProbeResult): string { async function defaultPostgresQueryHistoryProbe( input: PostgresQueryHistoryProbeInput, ): Promise { - const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient, isKtxPostgresConnectionConfig }] = - await Promise.all([import('./context/ingest/index.js'), import('./connectors/postgres/index.js')]); + const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient }, { isKtxPostgresConnectionConfig }] = + await Promise.all([ + import('./context/ingest/adapters/historic-sql/postgres-pgss-reader.js'), + import('./connectors/postgres/historic-sql-query-client.js'), + import('./connectors/postgres/connector.js'), + ]); const inputDriver = input.connection.driver ?? 'unknown'; if (!isKtxPostgresConnectionConfig(input.connection)) { @@ -749,7 +747,6 @@ function colorForLevel(useColor: boolean, level: ProjectStatusLevel, text: strin return level === 'ok' ? green(text) : level === 'warn' ? yellow(text) : red(text); } - function abbreviateHome(filePath: string, env: NodeJS.ProcessEnv): string { const home = env.HOME; if (home && (filePath === home || filePath.startsWith(`${home}/`))) { diff --git a/packages/cli/src/text-ingest.test.ts b/packages/cli/src/text-ingest.test.ts index ec451244..b7737f36 100644 --- a/packages/cli/src/text-ingest.test.ts +++ b/packages/cli/src/text-ingest.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import type { MemoryIngestStatus } from './context/memory/index.js'; -import type { KtxLocalProject } from './context/project/index.js'; +import type { MemoryIngestStatus } from './context/memory/memory-runs.js'; +import type { KtxLocalProject } from './context/project/project.js'; import { runKtxTextIngest, type TextMemoryIngestPort } from './text-ingest.js'; function makeIo(options: { isTTY?: boolean } = {}) { diff --git a/packages/cli/src/text-ingest.ts b/packages/cli/src/text-ingest.ts index 9ba64e13..388e58d6 100644 --- a/packages/cli/src/text-ingest.ts +++ b/packages/cli/src/text-ingest.ts @@ -1,7 +1,9 @@ import { readFile as fsReadFile } from 'node:fs/promises'; import { basename, resolve } from 'node:path'; -import { createLocalProjectMemoryIngest, type MemoryAgentInput, type MemoryIngestStatus } from './context/memory/index.js'; -import { loadKtxProject, type KtxLocalProject } from './context/project/index.js'; +import { createLocalProjectMemoryIngest } from './context/memory/local-memory.js'; +import type { MemoryAgentInput } from './context/memory/types.js'; +import type { MemoryIngestStatus } from './context/memory/memory-runs.js'; +import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; import type { KtxCliIo } from './cli-runtime.js'; import { createRepainter, initViewState, renderContextBuildView, type ContextBuildTargetState } from './context-build-view.js'; import { formatDuration } from './demo-metrics.js'; diff --git a/scripts/relationship-benchmark-report.mjs b/scripts/relationship-benchmark-report.mjs index 08f6b01e..7af82d57 100644 --- a/scripts/relationship-benchmark-report.mjs +++ b/scripts/relationship-benchmark-report.mjs @@ -1,14 +1,16 @@ import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { - KTX_RELATIONSHIP_BENCHMARK_MODES, buildKtxRelationshipBenchmarkReport, - currentKtxRelationshipBenchmarkDetector, formatKtxRelationshipBenchmarkReportMarkdown, +} from '../packages/cli/dist/context/scan/relationship-benchmark-report.js'; +import { + KTX_RELATIONSHIP_BENCHMARK_MODES, + currentKtxRelationshipBenchmarkDetector, ktxRelationshipBenchmarkDetectorWithLlm, loadKtxRelationshipBenchmarkFixtures, runKtxRelationshipBenchmarkSuite, -} from '../packages/cli/dist/context/scan/index.js'; +} from '../packages/cli/dist/context/scan/relationship-benchmarks.js'; const scriptDir = dirname(fileURLToPath(import.meta.url)); const ktxRoot = resolve(scriptDir, '..');