mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-13 11:22:11 +02:00
rename klo to ktx
This commit is contained in:
parent
1a42152e6f
commit
3ce510b55b
704 changed files with 10205 additions and 10255 deletions
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "@klo/context",
|
||||
"name": "@ktx/context",
|
||||
"version": "0.0.0-private",
|
||||
"description": "Core context library for database agents",
|
||||
"private": true,
|
||||
|
|
@ -127,7 +127,7 @@
|
|||
"type-check": "tsc -p tsconfig.json --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@klo/llm": "workspace:*",
|
||||
"@ktx/llm": "workspace:*",
|
||||
"@looker/sdk": "^26.6.1",
|
||||
"@looker/sdk-node": "^26.6.1",
|
||||
"@looker/sdk-rtl": "^21.6.5",
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
<role>
|
||||
You are processing ONE WorkUnit of a multi-file ingest bundle. The WorkUnit gives you a slice of raw source files (LookML views, dbt/MetricFlow YAMLs, Metabase card JSONs, or similar) and you must translate that slice into KLO semantic-layer sources and/or knowledge wiki pages, in one pass. Prior WorkUnits in this same job may have already written SL sources and wiki pages; their writes are visible on the working branch and searchable via `wiki_sl_search`.
|
||||
You are processing ONE WorkUnit of a multi-file ingest bundle. The WorkUnit gives you a slice of raw source files (LookML views, dbt/MetricFlow YAMLs, Metabase card JSONs, or similar) and you must translate that slice into KTX semantic-layer sources and/or knowledge wiki pages, in one pass. Prior WorkUnits in this same job may have already written SL sources and wiki pages; their writes are visible on the working branch and searchable via `wiki_sl_search`.
|
||||
</role>
|
||||
|
||||
<stance>
|
||||
Assertive. The bundle was explicitly submitted for ingest. Default to capturing everything the raw files declare that maps cleanly to KLO: one SL source per table/view, one wiki page per non-obvious business rule or alias. Do not abandon a WorkUnit because "some content overlaps with another WU"; use `ingest_triage` to reconcile, do not skip.
|
||||
Assertive. The bundle was explicitly submitted for ingest. Default to capturing everything the raw files declare that maps cleanly to KTX: one SL source per table/view, one wiki page per non-obvious business rule or alias. Do not abandon a WorkUnit because "some content overlaps with another WU"; use `ingest_triage` to reconcile, do not skip.
|
||||
</stance>
|
||||
|
||||
<workflow>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<role>
|
||||
You are ingesting an external technical artifact (a LookML view, dbt model, schema description, business glossary, or other reference document) into KLO organizational memory. The user has explicitly submitted this content for bulk ingest. Assume it is intentional and worth capturing.
|
||||
You are ingesting an external technical artifact (a LookML view, dbt model, schema description, business glossary, or other reference document) into KTX organizational memory. The user has explicitly submitted this content for bulk ingest. Assume it is intentional and worth capturing.
|
||||
</role>
|
||||
|
||||
<stance>
|
||||
|
|
@ -18,7 +18,7 @@ A single artifact typically produces multiple actions: one SL source per table/v
|
|||
</workflow>
|
||||
|
||||
<scope>
|
||||
All wiki writes go to the GLOBAL scope — they will be visible to every user of this KLO project. Phrase wiki pages as objective business knowledge, not personal preference. The `wiki_write` tool handles scope selection automatically for external ingest.
|
||||
All wiki writes go to the GLOBAL scope — they will be visible to every user of this KTX project. Phrase wiki pages as objective business knowledge, not personal preference. The `wiki_write` tool handles scope selection automatically for external ingest.
|
||||
</scope>
|
||||
|
||||
<do_not>
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@ import { fileURLToPath } from 'node:url';
|
|||
const require = createRequire(import.meta.url);
|
||||
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||
const contextDir = resolve(scriptDir, '..');
|
||||
const kloRoot = resolve(contextDir, '../..');
|
||||
const docsDir = join(kloRoot, 'docs');
|
||||
const ktxRoot = resolve(contextDir, '../..');
|
||||
const docsDir = join(ktxRoot, 'docs');
|
||||
const reportPath = join(docsDir, 'hybrid-search-pglite-spike.md');
|
||||
|
||||
async function timed(label, fn) {
|
||||
|
|
@ -63,7 +63,7 @@ async function packageInfo(packageName) {
|
|||
return {
|
||||
name: packageName,
|
||||
version: packageJson.version,
|
||||
path: relative(kloRoot, packageDir),
|
||||
path: relative(ktxRoot, packageDir),
|
||||
bytes: await directoryBytes(packageDir),
|
||||
};
|
||||
}
|
||||
|
|
@ -164,7 +164,7 @@ async function main() {
|
|||
});
|
||||
|
||||
const { PGlite, vector, pg_trgm } = importTimer.value;
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'klo-pglite-report-'));
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'ktx-pglite-report-'));
|
||||
const dataDir = join(tempDir, 'pgdata');
|
||||
|
||||
let db;
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ import { Client } from 'pg';
|
|||
|
||||
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||
const contextDir = resolve(scriptDir, '..');
|
||||
const kloRoot = resolve(contextDir, '../..');
|
||||
const reportPath = join(kloRoot, 'docs', 'hybrid-search-pglite-owner-process.md');
|
||||
const ktxRoot = resolve(contextDir, '../..');
|
||||
const reportPath = join(ktxRoot, 'docs', 'hybrid-search-pglite-owner-process.md');
|
||||
|
||||
async function timed(label, fn) {
|
||||
const started = performance.now();
|
||||
|
|
@ -98,7 +98,7 @@ async function createOwner(dataDir, port) {
|
|||
port,
|
||||
user: 'postgres',
|
||||
database: 'postgres',
|
||||
application_name: 'klo-pglite-owner-report',
|
||||
application_name: 'ktx-pglite-owner-report',
|
||||
connectionTimeoutMillis: 5_000,
|
||||
},
|
||||
};
|
||||
|
|
@ -222,7 +222,7 @@ async function stopOwner(owner) {
|
|||
}
|
||||
|
||||
async function main() {
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'klo-pglite-owner-report-'));
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'ktx-pglite-owner-report-'));
|
||||
const dataDir = join(tempDir, 'pgdata');
|
||||
const port = await allocatePort();
|
||||
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ import { Client } from 'pg';
|
|||
|
||||
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||
const contextDir = resolve(scriptDir, '..');
|
||||
const kloRoot = resolve(contextDir, '../..');
|
||||
const reportPath = join(kloRoot, 'docs', 'hybrid-search-pglite-sl-adapter-prototype.md');
|
||||
const ktxRoot = resolve(contextDir, '../..');
|
||||
const reportPath = join(ktxRoot, 'docs', 'hybrid-search-pglite-sl-adapter-prototype.md');
|
||||
|
||||
async function timed(label, fn) {
|
||||
const started = performance.now();
|
||||
|
|
@ -91,7 +91,7 @@ async function createOwner(dataDir, port) {
|
|||
port,
|
||||
user: 'postgres',
|
||||
database: 'postgres',
|
||||
application_name: 'klo-pglite-sl-prototype-report',
|
||||
application_name: 'ktx-pglite-sl-prototype-report',
|
||||
connectionTimeoutMillis: 5_000,
|
||||
},
|
||||
};
|
||||
|
|
@ -193,7 +193,7 @@ async function stopOwner(owner) {
|
|||
}
|
||||
|
||||
async function main() {
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'klo-pglite-sl-prototype-report-'));
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'ktx-pglite-sl-prototype-report-'));
|
||||
const dataDir = join(tempDir, 'pgdata');
|
||||
const port = await allocatePort();
|
||||
let owner;
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
import { dirname, join, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
KLO_RELATIONSHIP_BENCHMARK_MODES,
|
||||
buildKloRelationshipBenchmarkReport,
|
||||
currentKloRelationshipBenchmarkDetector,
|
||||
formatKloRelationshipBenchmarkReportMarkdown,
|
||||
kloRelationshipBenchmarkDetectorWithLlm,
|
||||
loadKloRelationshipBenchmarkFixtures,
|
||||
runKloRelationshipBenchmarkSuite,
|
||||
KTX_RELATIONSHIP_BENCHMARK_MODES,
|
||||
buildKtxRelationshipBenchmarkReport,
|
||||
currentKtxRelationshipBenchmarkDetector,
|
||||
formatKtxRelationshipBenchmarkReportMarkdown,
|
||||
ktxRelationshipBenchmarkDetectorWithLlm,
|
||||
loadKtxRelationshipBenchmarkFixtures,
|
||||
runKtxRelationshipBenchmarkSuite,
|
||||
} from '../dist/scan/index.js';
|
||||
|
||||
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||
|
|
@ -15,38 +15,38 @@ const packageRoot = resolve(scriptDir, '..');
|
|||
const fixtureRoot = join(packageRoot, 'test/fixtures/relationship-benchmarks');
|
||||
|
||||
async function buildDetector() {
|
||||
const backend = process.env.KLO_BENCHMARK_LLM_BACKEND;
|
||||
const backend = process.env.KTX_BENCHMARK_LLM_BACKEND;
|
||||
if (!backend || backend === 'none') {
|
||||
return currentKloRelationshipBenchmarkDetector();
|
||||
return currentKtxRelationshipBenchmarkDetector();
|
||||
}
|
||||
if (backend !== 'vertex') {
|
||||
throw new Error(`Unsupported KLO_BENCHMARK_LLM_BACKEND: ${backend}`);
|
||||
throw new Error(`Unsupported KTX_BENCHMARK_LLM_BACKEND: ${backend}`);
|
||||
}
|
||||
const project = process.env.KLO_BENCHMARK_VERTEX_PROJECT;
|
||||
const location = process.env.KLO_BENCHMARK_VERTEX_LOCATION;
|
||||
const model = process.env.KLO_BENCHMARK_LLM_MODEL ?? 'claude-sonnet-4-6';
|
||||
const project = process.env.KTX_BENCHMARK_VERTEX_PROJECT;
|
||||
const location = process.env.KTX_BENCHMARK_VERTEX_LOCATION;
|
||||
const model = process.env.KTX_BENCHMARK_LLM_MODEL ?? 'claude-sonnet-4-6';
|
||||
if (!project || !location) {
|
||||
throw new Error('KLO_BENCHMARK_VERTEX_PROJECT and KLO_BENCHMARK_VERTEX_LOCATION are required for vertex backend');
|
||||
throw new Error('KTX_BENCHMARK_VERTEX_PROJECT and KTX_BENCHMARK_VERTEX_LOCATION are required for vertex backend');
|
||||
}
|
||||
const { createKloLlmProvider } = await import('@klo/llm');
|
||||
const provider = createKloLlmProvider({
|
||||
const { createKtxLlmProvider } = await import('@ktx/llm');
|
||||
const provider = createKtxLlmProvider({
|
||||
backend: 'vertex',
|
||||
vertex: { project, location },
|
||||
modelSlots: { default: model },
|
||||
});
|
||||
return kloRelationshipBenchmarkDetectorWithLlm(provider);
|
||||
return ktxRelationshipBenchmarkDetectorWithLlm(provider);
|
||||
}
|
||||
|
||||
const fixtures = await loadKloRelationshipBenchmarkFixtures(fixtureRoot);
|
||||
const fixtures = await loadKtxRelationshipBenchmarkFixtures(fixtureRoot);
|
||||
const detector = await buildDetector();
|
||||
const suite = await runKloRelationshipBenchmarkSuite({
|
||||
const suite = await runKtxRelationshipBenchmarkSuite({
|
||||
fixtures,
|
||||
detector,
|
||||
});
|
||||
const report = buildKloRelationshipBenchmarkReport({
|
||||
const report = buildKtxRelationshipBenchmarkReport({
|
||||
fixtures,
|
||||
suite,
|
||||
modes: KLO_RELATIONSHIP_BENCHMARK_MODES,
|
||||
modes: KTX_RELATIONSHIP_BENCHMARK_MODES,
|
||||
});
|
||||
|
||||
process.stdout.write(formatKloRelationshipBenchmarkReportMarkdown(report));
|
||||
process.stdout.write(formatKtxRelationshipBenchmarkReportMarkdown(report));
|
||||
|
|
|
|||
|
|
@ -1,16 +1,16 @@
|
|||
---
|
||||
name: dbt_ingest
|
||||
description: Map dbt `schema.yml` / `properties.yml` models and sources into KLO semantic-layer overlays and column notes. Covers `sources:` vs `models:`, column `data_tests` (not_null, unique, accepted_values, relationships), and how bundle-time writes complement manifest backfill from git sync. Load when the WorkUnit's `skillNames` includes `dbt_ingest` or when raw files are dbt YAML under `models/` / `sources/`.
|
||||
description: Map dbt `schema.yml` / `properties.yml` models and sources into KTX semantic-layer overlays and column notes. Covers `sources:` vs `models:`, column `data_tests` (not_null, unique, accepted_values, relationships), and how bundle-time writes complement manifest backfill from git sync. Load when the WorkUnit's `skillNames` includes `dbt_ingest` or when raw files are dbt YAML under `models/` / `sources/`.
|
||||
callers: [memory_agent]
|
||||
---
|
||||
|
||||
# dbt → KLO (bundle ingest)
|
||||
# dbt → KTX (bundle ingest)
|
||||
|
||||
Use this skill for **uploaded** dbt projects (`dbt_project.yml` at stage root, `models/**`, `sources/**`, `schema.yml`). There is **no** `fetch()` in v1 — scheduled `dbt parse` / `manifest.json` pulls are out of scope; host-provided dbt sync may still backfill structured test metadata into `_schema` on the next sync.
|
||||
|
||||
## Mapping (models / sources → SL)
|
||||
|
||||
| dbt | KLO | Notes |
|
||||
| dbt | KTX | Notes |
|
||||
|-----|--------|--------|
|
||||
| `models:` entry with `columns:` | **Overlay** on the manifest table with the same name (after `wiki_sl_search` / `sl_describe_table`) | One SL source per physical table; model name may differ from DB name — resolve with `read_raw_file` + warehouse context. |
|
||||
| `sources:` → `tables:` | Same as models; use `identifier` when present instead of logical `name`. | Schema + name must match how the connection sees tables. |
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
---
|
||||
name: knowledge_capture
|
||||
description: KLO's knowledge base — wiki pages for durable, reusable business knowledge. Covers capture workflow for user preferences, metric definitions, organizational conventions, and cross-references between knowledge pages and semantic-layer sources. Loaded by the post-turn memory-agent only. The research agent reads wiki via `wiki_read`/`wiki_search` but does not write it.
|
||||
description: KTX's knowledge base — wiki pages for durable, reusable business knowledge. Covers capture workflow for user preferences, metric definitions, organizational conventions, and cross-references between knowledge pages and semantic-layer sources. Loaded by the post-turn memory-agent only. The research agent reads wiki via `wiki_read`/`wiki_search` but does not write it.
|
||||
callers: [memory_agent]
|
||||
---
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
---
|
||||
name: looker_ingest
|
||||
description: Extract durable KLO knowledge and semantic-layer contribution proposals from staged Looker runtime dashboard, Look, and explore JSON. Load for WorkUnits whose raw files are under explores/, dashboards/, or looks/.
|
||||
description: Extract durable KTX knowledge and semantic-layer contribution proposals from staged Looker runtime dashboard, Look, and explore JSON. Load for WorkUnits whose raw files are under explores/, dashboards/, or looks/.
|
||||
callers: [memory_agent]
|
||||
---
|
||||
|
||||
# Looker Runtime Ingest
|
||||
|
||||
Looker runtime ingest turns API-staged dashboards, Looks, and explores into durable KLO memory. Runtime entities are evidence. They are not themselves the final knowledge shape.
|
||||
Looker runtime ingest turns API-staged dashboards, Looks, and explores into durable KTX memory. Runtime entities are evidence. They are not themselves the final knowledge shape.
|
||||
|
||||
## Required Workflow
|
||||
|
||||
|
|
@ -72,7 +72,7 @@ The staged explore file carries warehouse target fields populated before the WU
|
|||
- `rawSqlTableName`: Looker's verbatim `sql_table_name`. Keep it as provenance only.
|
||||
- `targetTable`: the parsed target-table union. Use this as the sole branch condition.
|
||||
|
||||
When `targetTable.ok === true`, the explore has a complete KLO backing target. Before writing:
|
||||
When `targetTable.ok === true`, the explore has a complete KTX backing target. Before writing:
|
||||
|
||||
1. Use `targetTable.catalog`, `targetTable.schema`, and `targetTable.name` for `source_tables` preflight matching through `sl_discover` or `sl_read_source`.
|
||||
2. Use Looker field `sql`, labels, descriptions, and type metadata to derive source columns, measures, segments, joins, and grain.
|
||||
|
|
|
|||
|
|
@ -1,16 +1,16 @@
|
|||
---
|
||||
name: lookml_ingest
|
||||
description: Map a LookML view/model/explore into KLO semantic layer sources. Covers the LookML to KLO primitive table, provenance tagging, and three worked examples (overlay, standalone from derived_table, standalone with sql_always_where). Load when the turn contains `.lkml` content.
|
||||
description: Map a LookML view/model/explore into KTX semantic layer sources. Covers the LookML to KTX primitive table, provenance tagging, and three worked examples (overlay, standalone from derived_table, standalone with sql_always_where). Load when the turn contains `.lkml` content.
|
||||
callers: [memory_agent]
|
||||
---
|
||||
|
||||
# LookML to KLO Semantic Layer
|
||||
# LookML to KTX Semantic Layer
|
||||
|
||||
LookML views map to SL sources, `measure:` to measures, `explore: { join: }` to the join graph. This skill lays out the mapping and the three capture shapes.
|
||||
|
||||
## Mapping table
|
||||
|
||||
| LookML | KLO form | Notes |
|
||||
| LookML | KTX form | Notes |
|
||||
|---|---|---|
|
||||
| `view: X { sql_table_name: …; measure:/dimension:/join: }` | **Overlay** at `<connId>/X.yaml` with `measures`, `columns` (computed), `joins`, `segments` | Manifest-backed; inherit grain/columns |
|
||||
| `view: X { derived_table: { sql: … } }` | **Standalone** with top-level `sql:`, explicit `grain:` + `columns:` | No manifest entry exists |
|
||||
|
|
@ -23,7 +23,7 @@ Type map: `date`/`datetime`/`timestamp` → `time`; `yesno` → `boolean`; `numb
|
|||
|
||||
## Decision rules
|
||||
|
||||
LookML writes target the run connection directly. Unlike Looker runtime ingestion, the LookML adapter is configured on the warehouse KLO connection, so do not look for `targetWarehouseConnectionId` and do not route through a mapping array.
|
||||
LookML writes target the run connection directly. Unlike Looker runtime ingestion, the LookML adapter is configured on the warehouse KTX connection, so do not look for `targetWarehouseConnectionId` and do not route through a mapping array.
|
||||
|
||||
Before any SL write, inspect the WorkUnit notes.
|
||||
|
||||
|
|
@ -56,7 +56,7 @@ A prior replay hallucinated `date_date`, `date_week` into `sql:`, `columns:`, an
|
|||
2. If the table isn't in the manifest, fall back to `sql_execution({ sql: "SELECT column_name FROM <dataset>.INFORMATION_SCHEMA.COLUMNS WHERE table_name = '<table>'" })` (session shape — a connection is already pinned by the ingest session).
|
||||
3. Use only those names in `sql:`, `columns:`, and `grain:`. Map each `dimension_group` to ONE `{ name: <physical_col>, type: time, role: time }` entry — never one per timeframe.
|
||||
|
||||
| LookML input | KLO `columns:` entry |
|
||||
| LookML input | KTX `columns:` entry |
|
||||
|---|---|
|
||||
| `dimension_group: month { type: time; timeframes: [month]; sql: ${TABLE}.month_date ;; }` | `{ name: month_date, type: time, role: time }` |
|
||||
| `dimension_group: date { type: time; timeframes: [raw, date, week, month]; sql: ${TABLE}.date ;; }` | `{ name: date, type: time, role: time }` — single entry, NOT `date_raw`/`date_date`/`date_week` |
|
||||
|
|
@ -94,7 +94,7 @@ explore: fct_labs {
|
|||
}
|
||||
```
|
||||
|
||||
KLO overlay at `<connId>/fct_labs.yaml`:
|
||||
KTX overlay at `<connId>/fct_labs.yaml`:
|
||||
|
||||
```yaml
|
||||
name: fct_labs
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
---
|
||||
name: metabase_ingest
|
||||
description: Convert Metabase questions, models, and metrics into KLO Semantic Layer source definitions. Covers result-metadata to KSL column type mapping, FK/PK detection, near-duplicate deduplication, pre-aggregation decomposition, join-graph connectivity, and how to react to priorProvenance from earlier ingest syncs. Load when the WorkUnit contains `cards/<id>.json` files under a Metabase bundle.
|
||||
description: Convert Metabase questions, models, and metrics into KTX Semantic Layer source definitions. Covers result-metadata to KSL column type mapping, FK/PK detection, near-duplicate deduplication, pre-aggregation decomposition, join-graph connectivity, and how to react to priorProvenance from earlier ingest syncs. Load when the WorkUnit contains `cards/<id>.json` files under a Metabase bundle.
|
||||
callers: [memory_agent]
|
||||
---
|
||||
|
||||
# Metabase to KLO Semantic Layer
|
||||
# Metabase to KTX Semantic Layer
|
||||
|
||||
Each WorkUnit represents one Metabase collection's cards for one Metabase database (mapped to exactly one KLO connection). Every `cards/<id>.json` file carries the resolved SQL, result_metadata, card type, collection path, and referenced-card ids. The WU's `sync-config.json` tells you which sync mode is active and which selections apply. `databases/<id>.json` tells you the target KLO connection.
|
||||
Each WorkUnit represents one Metabase collection's cards for one Metabase database (mapped to exactly one KTX connection). Every `cards/<id>.json` file carries the resolved SQL, result_metadata, card type, collection path, and referenced-card ids. The WU's `sync-config.json` tells you which sync mode is active and which selections apply. `databases/<id>.json` tells you the target KTX connection.
|
||||
|
||||
## Context format
|
||||
|
||||
|
|
@ -194,7 +194,7 @@ If a source is derived from multiple cards (e.g. a generalized source for a near
|
|||
|
||||
## Quality standards
|
||||
|
||||
Source definitions must follow klo-sl YAML conventions:
|
||||
Source definitions must follow ktx-sl YAML conventions:
|
||||
- `source_type`: `"table"` (physical table/view) or `"sql"` (arbitrary SQL / derived view).
|
||||
- `table`: required when `source_type: "table"` (e.g. `"public.orders"`).
|
||||
- `sql`: required when `source_type: "sql"`.
|
||||
|
|
|
|||
|
|
@ -1,16 +1,16 @@
|
|||
---
|
||||
name: metricflow_ingest
|
||||
description: Map a MetricFlow semantic_model or metric into KLO semantic layer sources. Covers the MetricFlow to KLO primitive table, `extends:` inheritance flattening, metric-type handling (simple / derived / ratio / cumulative / conversion), `model: ref('x')` resolution, and four worked examples. Load when the turn contains `.yml`/`.yaml` files with top-level `semantic_models:` or `metrics:`.
|
||||
description: Map a MetricFlow semantic_model or metric into KTX semantic layer sources. Covers the MetricFlow to KTX primitive table, `extends:` inheritance flattening, metric-type handling (simple / derived / ratio / cumulative / conversion), `model: ref('x')` resolution, and four worked examples. Load when the turn contains `.yml`/`.yaml` files with top-level `semantic_models:` or `metrics:`.
|
||||
callers: [memory_agent]
|
||||
---
|
||||
|
||||
# MetricFlow to KLO Semantic Layer
|
||||
# MetricFlow to KTX Semantic Layer
|
||||
|
||||
A MetricFlow `semantic_model` maps to an SL source; MetricFlow `measures` map to KLO measures; MetricFlow `entities` map to KLO `joins`; MetricFlow `metrics` (top-level) map to KLO measures OR to cross-model derived measures. Files in one WorkUnit are ALWAYS part of the same logical entity (a connected component, possibly spanning `extends:` + cross-model metric refs). Flatten inheritance and cross-file references at write time.
|
||||
A MetricFlow `semantic_model` maps to an SL source; MetricFlow `measures` map to KTX measures; MetricFlow `entities` map to KTX `joins`; MetricFlow `metrics` (top-level) map to KTX measures OR to cross-model derived measures. Files in one WorkUnit are ALWAYS part of the same logical entity (a connected component, possibly spanning `extends:` + cross-model metric refs). Flatten inheritance and cross-file references at write time.
|
||||
|
||||
## Mapping table
|
||||
|
||||
| MetricFlow | KLO form | Notes |
|
||||
| MetricFlow | KTX form | Notes |
|
||||
|---|---|---|
|
||||
| `semantic_model: X { model: ref('t') }` with measures + dimensions | **Overlay** at `<connId>/X.yaml` with `measures`, `columns` (computed), `joins` | The `model:` ref resolves to a manifest table. |
|
||||
| `semantic_model: X { model: source('s','t') }` | **Overlay** at `<connId>/X.yaml` over table `t`. | Same shape; `source()` still resolves to a physical table. |
|
||||
|
|
@ -23,11 +23,11 @@ A MetricFlow `semantic_model` maps to an SL source; MetricFlow `measures` map to
|
|||
| `metrics: [{ type: simple, filter: <jinja> }]` | **New measure** on the same source, with the filter translated to SQL and attached via `filter:` | Translate Jinja `{{ Dimension('x__y') }}` to the column name `y`. |
|
||||
| `metrics: [{ type: derived, type_params: { expr, metrics } }]` | **Derived measure** on whichever source owns the referenced measures, with `expr:` referencing measure names | If the metric spans models, still write it once on the source owning the "primary" measure (the one the agent judges most central). Mention the cross-model chain in the description. |
|
||||
| `metrics: [{ type: ratio, type_params: { numerator, denominator } }]` | Same as derived; `expr: "numerator / NULLIF(denominator, 0)"` if no explicit expr | Safe-division by default. |
|
||||
| `metrics: [{ type: cumulative, type_params: { window, grain_to_date } }]` | **Standalone** source with a window-function SQL; reference the resulting column as a normal measure | KLO SL has no first-class cumulative primitive (spec Non-goals). |
|
||||
| `metrics: [{ type: conversion }]` | **Flag for human** — do NOT write. Emit a wiki note describing the intended semantics. | No KLO equivalent in v1. |
|
||||
| `metrics: [{ type: cumulative, type_params: { window, grain_to_date } }]` | **Standalone** source with a window-function SQL; reference the resulting column as a normal measure | KTX SL has no first-class cumulative primitive (spec Non-goals). |
|
||||
| `metrics: [{ type: conversion }]` | **Flag for human** — do NOT write. Emit a wiki note describing the intended semantics. | No KTX equivalent in v1. |
|
||||
| Metric not mappable | Wiki page `<metric_name>-definition.md` with the full YAML body quoted | Capture the intent even if we can't emit SL. |
|
||||
|
||||
Type map: MetricFlow `time` to KLO `time`; `categorical` to `string`; `number` to `number`; `boolean` to `boolean`. Follow `expr` over `name` when both differ — `expr` is the physical column.
|
||||
Type map: MetricFlow `time` to KTX `time`; `categorical` to `string`; `number` to `number`; `boolean` to `boolean`. Follow `expr` over `name` when both differ — `expr` is the physical column.
|
||||
|
||||
## Flattening `extends:`
|
||||
|
||||
|
|
@ -55,7 +55,7 @@ After every `sl_write_source`, call `sl_validate`. The warehouse will reject inv
|
|||
|
||||
## Cumulative metrics — sql-standalone fallback
|
||||
|
||||
KLO SL has no first-class `window:` or `grain_to_date:` primitive in v1 (spec Non-goals). Translate a MetricFlow cumulative metric to a standalone SL source with a window-function SQL:
|
||||
KTX SL has no first-class `window:` or `grain_to_date:` primitive in v1 (spec Non-goals). Translate a MetricFlow cumulative metric to a standalone SL source with a window-function SQL:
|
||||
|
||||
```yaml
|
||||
# MetricFlow input:
|
||||
|
|
@ -68,7 +68,7 @@ metrics:
|
|||
```
|
||||
|
||||
```yaml
|
||||
# KLO standalone output:
|
||||
# KTX standalone output:
|
||||
name: cum_revenue_7d
|
||||
source_type: sql
|
||||
sql: |
|
||||
|
|
@ -106,7 +106,7 @@ Do NOT emit SL for this. Instead:
|
|||
- Write a wiki page at `knowledge/global/<metric_name>-intent.md` quoting the full YAML body and a one-line explanation of the intended semantics (base event → conversion event within window).
|
||||
- Call `emit_unmapped_fallback` with `rawPath` set to the MetricFlow file path, `reason: "conversion_metric_unsupported"`, and `fallback: "flagged"`.
|
||||
|
||||
When KLO SL gains conversion primitives, re-ingesting will find the prior wiki note (via `priorProvenance`) and replace it with an SL source.
|
||||
When KTX SL gains conversion primitives, re-ingesting will find the prior wiki note (via `priorProvenance`) and replace it with an SL source.
|
||||
|
||||
## Provenance markers
|
||||
|
||||
|
|
@ -137,7 +137,7 @@ semantic_models:
|
|||
```
|
||||
|
||||
```yaml
|
||||
# KLO overlay at <connId>/orders.yaml:
|
||||
# KTX overlay at <connId>/orders.yaml:
|
||||
# <!-- from: raw-sources/.../models/orders.yml#L1-10 -->
|
||||
name: orders
|
||||
description: Order fact table.
|
||||
|
|
@ -179,7 +179,7 @@ metrics:
|
|||
```
|
||||
|
||||
```yaml
|
||||
# KLO overlay at <connId>/orders_ext.yaml (one file; inheritance flattened):
|
||||
# KTX overlay at <connId>/orders_ext.yaml (one file; inheritance flattened):
|
||||
# <!-- from: raw-sources/.../models/orders.yml#L1-10 -->
|
||||
# <!-- from: raw-sources/.../models/orders_ext.yml#L1-8 -->
|
||||
# <!-- from: raw-sources/.../metrics/orders_final.yml#L1-10 -->
|
||||
|
|
@ -217,7 +217,7 @@ metrics:
|
|||
metrics: [{name: revenue}, {name: cost}]
|
||||
```
|
||||
|
||||
Because the WorkUnit bundles all three files (cross-component union via the metric), write the derived measure on ONE of the two sources — pick the source whose domain "owns" the metric (here, `sales` — margin is inherently a sales metric). Cross-source references aren't native in KLO SL; treat the metric's operands as already-resolvable in the target source's query context OR emit a standalone SQL that joins the two tables:
|
||||
Because the WorkUnit bundles all three files (cross-component union via the metric), write the derived measure on ONE of the two sources — pick the source whose domain "owns" the metric (here, `sales` — margin is inherently a sales metric). Cross-source references aren't native in KTX SL; treat the metric's operands as already-resolvable in the target source's query context OR emit a standalone SQL that joins the two tables:
|
||||
|
||||
```yaml
|
||||
# <connId>/sales.yaml
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
---
|
||||
name: notion_synthesize
|
||||
description: Synthesize durable KLO wiki pages and semantic-layer sources from staged Notion pages, databases, data-source rows, and clustered Notion evidence. Load when a WorkUnit contains Notion raw files or Notion evidence chunks.
|
||||
description: Synthesize durable KTX wiki pages and semantic-layer sources from staged Notion pages, databases, data-source rows, and clustered Notion evidence. Load when a WorkUnit contains Notion raw files or Notion evidence chunks.
|
||||
callers: [memory_agent]
|
||||
---
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
---
|
||||
name: sl
|
||||
description: KLO's semantic layer — a structured catalog of sources (tables/views), measures, joins, and segments expressed as YAML. Covers the schema and how to query it via `semantic_query`. Use when the task involves querying pre-defined metrics (ARR, churn, retention, LTV, MAU) or reading SL source YAML to understand the catalog. Capture is handled by the `sl_capture` skill (memory-agent only).
|
||||
description: KTX's semantic layer — a structured catalog of sources (tables/views), measures, joins, and segments expressed as YAML. Covers the schema and how to query it via `semantic_query`. Use when the task involves querying pre-defined metrics (ARR, churn, retention, LTV, MAU) or reading SL source YAML to understand the catalog. Capture is handled by the `sl_capture` skill (memory-agent only).
|
||||
---
|
||||
|
||||
# Semantic Layer
|
||||
|
||||
KLO's semantic layer (SL) is a structured catalog. Each **source** represents a table, a SQL view, or an overlay that enriches a manifest-backed table with measures, computed columns, joins, and named segments. The catalog is the single source of truth for reusable business metrics.
|
||||
KTX's semantic layer (SL) is a structured catalog. Each **source** represents a table, a SQL view, or an overlay that enriches a manifest-backed table with measures, computed columns, joins, and named segments. The catalog is the single source of truth for reusable business metrics.
|
||||
|
||||
This skill covers two parts:
|
||||
- **Part 1** — Schema reference (what an SL source looks like).
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
---
|
||||
name: sl_capture
|
||||
description: How to capture new reusable patterns into KLO's semantic layer — when a measure, segment, or join belongs in the catalog and how to write it generically so it stays small and useful over time. Loaded by the post-turn memory-agent only. The research agent does not write to the SL.
|
||||
description: How to capture new reusable patterns into KTX's semantic layer — when a measure, segment, or join belongs in the catalog and how to write it generically so it stays small and useful over time. Loaded by the post-turn memory-agent only. The research agent does not write to the SL.
|
||||
callers: [memory_agent]
|
||||
---
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { KloMessageBuilder, type KloLlmProvider, type KloModelRole } from '@klo/llm';
|
||||
import { KtxMessageBuilder, type KtxLlmProvider, type KtxModelRole } from '@ktx/llm';
|
||||
import { generateText, stepCountIs, type TelemetrySettings, type Tool } from 'ai';
|
||||
import { noopLogger, type KloLogger } from '../core/index.js';
|
||||
import { summarizeKloLlmDebugRequest, type KloLlmDebugRequestRecorder } from '../llm/index.js';
|
||||
import { noopLogger, type KtxLogger } from '../core/index.js';
|
||||
import { summarizeKtxLlmDebugRequest, type KtxLlmDebugRequestRecorder } from '../llm/index.js';
|
||||
|
||||
export type RunLoopStopReason = 'budget' | 'natural' | 'error';
|
||||
|
||||
|
|
@ -11,7 +11,7 @@ export interface RunLoopStepInfo {
|
|||
}
|
||||
|
||||
export interface RunLoopParams {
|
||||
modelRole: KloModelRole;
|
||||
modelRole: KtxModelRole;
|
||||
systemPrompt: string;
|
||||
userPrompt: string;
|
||||
toolSet: Record<string, Tool>;
|
||||
|
|
@ -30,14 +30,14 @@ export interface AgentTelemetryPort {
|
|||
}
|
||||
|
||||
export interface AgentRunnerServiceDeps {
|
||||
llmProvider: KloLlmProvider;
|
||||
llmProvider: KtxLlmProvider;
|
||||
telemetry?: AgentTelemetryPort;
|
||||
debugRequestRecorder?: KloLlmDebugRequestRecorder;
|
||||
logger?: KloLogger;
|
||||
debugRequestRecorder?: KtxLlmDebugRequestRecorder;
|
||||
logger?: KtxLogger;
|
||||
}
|
||||
|
||||
export class AgentRunnerService {
|
||||
private readonly logger: KloLogger;
|
||||
private readonly logger: KtxLogger;
|
||||
|
||||
constructor(private readonly deps: AgentRunnerServiceDeps) {
|
||||
this.logger = deps.logger ?? noopLogger;
|
||||
|
|
@ -47,7 +47,7 @@ export class AgentRunnerService {
|
|||
let stepIndex = 0;
|
||||
try {
|
||||
const model = this.deps.llmProvider.getModel(params.modelRole);
|
||||
const builder = new KloMessageBuilder(this.deps.llmProvider);
|
||||
const builder = new KtxMessageBuilder(this.deps.llmProvider);
|
||||
const built = builder.wrapSimple({
|
||||
system: params.systemPrompt,
|
||||
messages: [{ role: 'user', content: params.userPrompt }],
|
||||
|
|
@ -56,8 +56,8 @@ export class AgentRunnerService {
|
|||
});
|
||||
|
||||
await this.deps.debugRequestRecorder?.record(
|
||||
summarizeKloLlmDebugRequest({
|
||||
operationName: params.telemetryTags.operationName ?? 'klo-agent-runner',
|
||||
summarizeKtxLlmDebugRequest({
|
||||
operationName: params.telemetryTags.operationName ?? 'ktx-agent-runner',
|
||||
source: params.telemetryTags.source,
|
||||
jobId: params.telemetryTags.jobId,
|
||||
unitKey: params.telemetryTags.unitKey,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
export type {
|
||||
KloSqlQueryExecutionInput,
|
||||
KloSqlQueryExecutionResult,
|
||||
KloSqlQueryExecutorPort,
|
||||
KtxSqlQueryExecutionInput,
|
||||
KtxSqlQueryExecutionResult,
|
||||
KtxSqlQueryExecutorPort,
|
||||
} from './query-executor.js';
|
||||
export { createDefaultLocalQueryExecutor, type DefaultLocalQueryExecutorOptions } from './local-query-executor.js';
|
||||
export { normalizeQueryRows } from './query-executor.js';
|
||||
|
|
@ -17,11 +17,11 @@ export {
|
|||
type LocalWarehouseDescriptor,
|
||||
} from './local-warehouse-descriptor.js';
|
||||
export {
|
||||
KLO_NOTION_ORG_KNOWLEDGE_WARNING,
|
||||
KTX_NOTION_ORG_KNOWLEDGE_WARNING,
|
||||
notionConnectionToPullConfig,
|
||||
parseNotionConnectionConfig,
|
||||
redactNotionConnectionConfig,
|
||||
resolveNotionAuthToken,
|
||||
type KloNotionConnectionConfig,
|
||||
type RedactedKloNotionConnectionConfig,
|
||||
type KtxNotionConnectionConfig,
|
||||
type RedactedKtxNotionConnectionConfig,
|
||||
} from './notion-config.js';
|
||||
|
|
|
|||
|
|
@ -1,26 +1,26 @@
|
|||
import { createPostgresQueryExecutor } from './postgres-query-executor.js';
|
||||
import type {
|
||||
KloSqlQueryExecutionInput,
|
||||
KloSqlQueryExecutionResult,
|
||||
KloSqlQueryExecutorPort,
|
||||
KtxSqlQueryExecutionInput,
|
||||
KtxSqlQueryExecutionResult,
|
||||
KtxSqlQueryExecutorPort,
|
||||
} from './query-executor.js';
|
||||
import { createSqliteQueryExecutor } from './sqlite-query-executor.js';
|
||||
|
||||
export interface DefaultLocalQueryExecutorOptions {
|
||||
postgres?: KloSqlQueryExecutorPort;
|
||||
sqlite?: KloSqlQueryExecutorPort;
|
||||
postgres?: KtxSqlQueryExecutorPort;
|
||||
sqlite?: KtxSqlQueryExecutorPort;
|
||||
}
|
||||
|
||||
function driverFor(input: KloSqlQueryExecutionInput): string {
|
||||
function driverFor(input: KtxSqlQueryExecutionInput): string {
|
||||
return String(input.connection?.driver ?? '').toLowerCase();
|
||||
}
|
||||
|
||||
export function createDefaultLocalQueryExecutor(options: DefaultLocalQueryExecutorOptions = {}): KloSqlQueryExecutorPort {
|
||||
export function createDefaultLocalQueryExecutor(options: DefaultLocalQueryExecutorOptions = {}): KtxSqlQueryExecutorPort {
|
||||
const postgres = options.postgres ?? createPostgresQueryExecutor();
|
||||
const sqlite = options.sqlite ?? createSqliteQueryExecutor();
|
||||
|
||||
return {
|
||||
async execute(input: KloSqlQueryExecutionInput): Promise<KloSqlQueryExecutionResult> {
|
||||
async execute(input: KtxSqlQueryExecutionInput): Promise<KtxSqlQueryExecutionResult> {
|
||||
const driver = driverFor(input);
|
||||
if (driver === 'postgres' || driver === 'postgresql') {
|
||||
return postgres.execute(input);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KloProjectConnectionConfig } from '../project/config.js';
|
||||
import type { KtxProjectConnectionConfig } from '../project/config.js';
|
||||
import type { ConnectionType } from './connection-type.js';
|
||||
|
||||
export interface LocalWarehouseDescriptor {
|
||||
|
|
@ -32,7 +32,7 @@ const DRIVER_TO_CONNECTION_TYPE: Record<string, ConnectionType> = {
|
|||
|
||||
export function localConnectionToWarehouseDescriptor(
|
||||
id: string,
|
||||
connection: KloProjectConnectionConfig | undefined,
|
||||
connection: KtxProjectConnectionConfig | undefined,
|
||||
): LocalWarehouseDescriptor | null {
|
||||
if (!connection) {
|
||||
return null;
|
||||
|
|
@ -74,7 +74,7 @@ export function localConnectionToWarehouseDescriptor(
|
|||
return info;
|
||||
}
|
||||
|
||||
export function localConnectionTypeForConfig(id: string, connection: KloProjectConnectionConfig | undefined): string {
|
||||
export function localConnectionTypeForConfig(id: string, connection: KtxProjectConnectionConfig | undefined): string {
|
||||
const descriptor = localConnectionToWarehouseDescriptor(id, connection);
|
||||
if (descriptor) {
|
||||
return descriptor.connection_type;
|
||||
|
|
@ -85,7 +85,7 @@ export function localConnectionTypeForConfig(id: string, connection: KloProjectC
|
|||
|
||||
export function localConnectionInfoFromConfig(
|
||||
id: string,
|
||||
connection: KloProjectConnectionConfig | undefined,
|
||||
connection: KtxProjectConnectionConfig | undefined,
|
||||
): LocalConnectionInfo | null {
|
||||
if (!connection) {
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ describe('standalone Notion connection config', () => {
|
|||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-notion-config-'));
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-notion-config-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
|
|||
|
|
@ -2,17 +2,17 @@ import { readFile } from 'node:fs/promises';
|
|||
import { homedir } from 'node:os';
|
||||
import { resolve } from 'node:path';
|
||||
import { type NotionPullConfig, notionPullConfigSchema } from '../ingest/adapters/notion/types.js';
|
||||
import type { KloProjectConnectionConfig } from '../project/config.js';
|
||||
import type { KtxProjectConnectionConfig } from '../project/config.js';
|
||||
|
||||
export const KLO_NOTION_ORG_KNOWLEDGE_WARNING =
|
||||
export const KTX_NOTION_ORG_KNOWLEDGE_WARNING =
|
||||
'Anything accessible to this Notion integration can become organization knowledge.';
|
||||
|
||||
type KloNotionCrawlMode = 'all_accessible' | 'selected_roots';
|
||||
type KtxNotionCrawlMode = 'all_accessible' | 'selected_roots';
|
||||
|
||||
export interface KloNotionConnectionConfig extends KloProjectConnectionConfig {
|
||||
export interface KtxNotionConnectionConfig extends KtxProjectConnectionConfig {
|
||||
driver: 'notion';
|
||||
auth_token_ref: string;
|
||||
crawl_mode: KloNotionCrawlMode;
|
||||
crawl_mode: KtxNotionCrawlMode;
|
||||
root_page_ids: string[];
|
||||
root_database_ids: string[];
|
||||
root_data_source_ids: string[];
|
||||
|
|
@ -22,17 +22,17 @@ export interface KloNotionConnectionConfig extends KloProjectConnectionConfig {
|
|||
last_successful_cursor: string | null;
|
||||
}
|
||||
|
||||
export interface RedactedKloNotionConnectionConfig {
|
||||
export interface RedactedKtxNotionConnectionConfig {
|
||||
driver: 'notion';
|
||||
hasAuthToken: boolean;
|
||||
crawlMode: KloNotionCrawlMode;
|
||||
crawlMode: KtxNotionCrawlMode;
|
||||
rootPageIds: string[];
|
||||
rootDatabaseIds: string[];
|
||||
rootDataSourceIds: string[];
|
||||
maxPagesPerRun: number;
|
||||
maxKnowledgeCreatesPerRun: number;
|
||||
maxKnowledgeUpdatesPerRun: number;
|
||||
warning: typeof KLO_NOTION_ORG_KNOWLEDGE_WARNING;
|
||||
warning: typeof KTX_NOTION_ORG_KNOWLEDGE_WARNING;
|
||||
}
|
||||
|
||||
interface ResolveNotionTokenOptions {
|
||||
|
|
@ -84,7 +84,7 @@ function boundedInteger(value: unknown, fallback: number, name: string, min: num
|
|||
return parsed;
|
||||
}
|
||||
|
||||
export function parseNotionConnectionConfig(raw: unknown): KloNotionConnectionConfig {
|
||||
export function parseNotionConnectionConfig(raw: unknown): KtxNotionConnectionConfig {
|
||||
const input = record(raw);
|
||||
if (input.driver !== 'notion') {
|
||||
throw new Error('Notion connection config requires driver: notion');
|
||||
|
|
@ -135,7 +135,7 @@ export function parseNotionConnectionConfig(raw: unknown): KloNotionConnectionCo
|
|||
};
|
||||
}
|
||||
|
||||
export function redactNotionConnectionConfig(config: KloNotionConnectionConfig): RedactedKloNotionConnectionConfig {
|
||||
export function redactNotionConnectionConfig(config: KtxNotionConnectionConfig): RedactedKtxNotionConnectionConfig {
|
||||
return {
|
||||
driver: 'notion',
|
||||
hasAuthToken: Boolean(config.auth_token_ref),
|
||||
|
|
@ -146,7 +146,7 @@ export function redactNotionConnectionConfig(config: KloNotionConnectionConfig):
|
|||
maxPagesPerRun: config.max_pages_per_run,
|
||||
maxKnowledgeCreatesPerRun: config.max_knowledge_creates_per_run,
|
||||
maxKnowledgeUpdatesPerRun: config.max_knowledge_updates_per_run,
|
||||
warning: KLO_NOTION_ORG_KNOWLEDGE_WARNING,
|
||||
warning: KTX_NOTION_ORG_KNOWLEDGE_WARNING,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -179,7 +179,7 @@ export async function resolveNotionAuthToken(
|
|||
}
|
||||
|
||||
export async function notionConnectionToPullConfig(
|
||||
config: KloNotionConnectionConfig,
|
||||
config: KtxNotionConnectionConfig,
|
||||
options: ResolveNotionTokenOptions = {},
|
||||
): Promise<NotionPullConfig> {
|
||||
return notionPullConfigSchema.parse({
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ describe('createPostgresQueryExecutor', () => {
|
|||
expect(client.connect).toHaveBeenCalledTimes(1);
|
||||
expect(calls[0]).toBe('BEGIN READ ONLY');
|
||||
expect(calls[1]).toEqual({
|
||||
text: 'select * from (select status, count(*) as order_count from public.orders group by status) as klo_query_result limit 50',
|
||||
text: 'select * from (select status, count(*) as order_count from public.orders group by status) as ktx_query_result limit 50',
|
||||
rowMode: 'array',
|
||||
});
|
||||
expect(calls[2]).toBe('COMMIT');
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { Client, type ClientConfig } from 'pg';
|
||||
import type {
|
||||
KloSqlQueryExecutionInput,
|
||||
KloSqlQueryExecutionResult,
|
||||
KloSqlQueryExecutorPort,
|
||||
KtxSqlQueryExecutionInput,
|
||||
KtxSqlQueryExecutionResult,
|
||||
KtxSqlQueryExecutorPort,
|
||||
} from './query-executor.js';
|
||||
import { limitSqlForExecution } from './read-only-sql.js';
|
||||
|
||||
|
|
@ -24,7 +24,7 @@ interface PostgresQueryExecutorOptions {
|
|||
clientFactory?: (config: ClientConfig) => PgClientLike;
|
||||
}
|
||||
|
||||
function connectionDriver(input: KloSqlQueryExecutionInput): string {
|
||||
function connectionDriver(input: KtxSqlQueryExecutionInput): string {
|
||||
return String(input.connection?.driver ?? '').toLowerCase();
|
||||
}
|
||||
|
||||
|
|
@ -32,10 +32,10 @@ function createDefaultClient(config: ClientConfig): PgClientLike {
|
|||
return new Client(config);
|
||||
}
|
||||
|
||||
export function createPostgresQueryExecutor(options: PostgresQueryExecutorOptions = {}): KloSqlQueryExecutorPort {
|
||||
export function createPostgresQueryExecutor(options: PostgresQueryExecutorOptions = {}): KtxSqlQueryExecutorPort {
|
||||
const clientFactory = options.clientFactory ?? createDefaultClient;
|
||||
return {
|
||||
async execute(input: KloSqlQueryExecutionInput): Promise<KloSqlQueryExecutionResult> {
|
||||
async execute(input: KtxSqlQueryExecutionInput): Promise<KtxSqlQueryExecutionResult> {
|
||||
const driver = connectionDriver(input);
|
||||
if (driver !== 'postgres' && driver !== 'postgresql') {
|
||||
throw new Error(`Local Postgres execution cannot run driver "${input.connection?.driver ?? 'unknown'}".`);
|
||||
|
|
@ -52,7 +52,7 @@ export function createPostgresQueryExecutor(options: PostgresQueryExecutorOption
|
|||
statement_timeout: options.statementTimeoutMs ?? 30_000,
|
||||
query_timeout: options.queryTimeoutMs ?? 35_000,
|
||||
connectionTimeoutMillis: options.connectionTimeoutMs ?? 5_000,
|
||||
application_name: 'klo-local-query',
|
||||
application_name: 'ktx-local-query',
|
||||
});
|
||||
await client.connect();
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
import type { KloProjectConnectionConfig } from '../project/index.js';
|
||||
import type { KtxProjectConnectionConfig } from '../project/index.js';
|
||||
|
||||
export interface KloSqlQueryExecutionInput {
|
||||
export interface KtxSqlQueryExecutionInput {
|
||||
connectionId: string;
|
||||
projectDir?: string;
|
||||
connection: KloProjectConnectionConfig | undefined;
|
||||
connection: KtxProjectConnectionConfig | undefined;
|
||||
sql: string;
|
||||
maxRows?: number;
|
||||
}
|
||||
|
||||
export interface KloSqlQueryExecutionResult {
|
||||
export interface KtxSqlQueryExecutionResult {
|
||||
headers: string[];
|
||||
rows: unknown[][];
|
||||
totalRows: number;
|
||||
|
|
@ -16,8 +16,8 @@ export interface KloSqlQueryExecutionResult {
|
|||
rowCount: number | null;
|
||||
}
|
||||
|
||||
export interface KloSqlQueryExecutorPort {
|
||||
execute(input: KloSqlQueryExecutionInput): Promise<KloSqlQueryExecutionResult>;
|
||||
export interface KtxSqlQueryExecutorPort {
|
||||
execute(input: KtxSqlQueryExecutionInput): Promise<KtxSqlQueryExecutionResult>;
|
||||
}
|
||||
|
||||
export function normalizeQueryRows(rows: unknown[]): unknown[][] {
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ describe('assertReadOnlySql', () => {
|
|||
describe('limitSqlForExecution', () => {
|
||||
it('wraps compiled SQL and strips trailing semicolons', () => {
|
||||
expect(limitSqlForExecution('select * from public.orders; ', 25)).toBe(
|
||||
'select * from (select * from public.orders) as klo_query_result limit 25',
|
||||
'select * from (select * from public.orders) as ktx_query_result limit 25',
|
||||
);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -18,5 +18,5 @@ export function limitSqlForExecution(sql: string, maxRows: number | undefined):
|
|||
if (!Number.isInteger(maxRows) || maxRows <= 0) {
|
||||
throw new Error('maxRows must be a positive integer.');
|
||||
}
|
||||
return `select * from (${trimmed}) as klo_query_result limit ${maxRows}`;
|
||||
return `select * from (${trimmed}) as ktx_query_result limit ${maxRows}`;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ describe('createSqliteQueryExecutor', () => {
|
|||
let dbPath: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-sqlite-query-'));
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-sqlite-query-'));
|
||||
dbPath = join(tempDir, 'warehouse.db');
|
||||
const db = new Database(dbPath);
|
||||
db.exec(`
|
||||
|
|
@ -81,23 +81,23 @@ describe('createSqliteQueryExecutor', () => {
|
|||
});
|
||||
|
||||
it('resolves env references for SQLite database urls', async () => {
|
||||
const originalDatabaseUrl = process.env.KLO_SQLITE_TEST_URL;
|
||||
process.env.KLO_SQLITE_TEST_URL = `sqlite:${dbPath}`;
|
||||
const originalDatabaseUrl = process.env.KTX_SQLITE_TEST_URL;
|
||||
process.env.KTX_SQLITE_TEST_URL = `sqlite:${dbPath}`;
|
||||
|
||||
try {
|
||||
expect(
|
||||
sqliteDatabasePathFromConnection({
|
||||
connectionId: 'warehouse',
|
||||
projectDir: tempDir,
|
||||
connection: { driver: 'sqlite', url: 'env:KLO_SQLITE_TEST_URL', readonly: true },
|
||||
connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL', readonly: true },
|
||||
sql: 'select 1',
|
||||
}),
|
||||
).toBe(dbPath);
|
||||
} finally {
|
||||
if (originalDatabaseUrl === undefined) {
|
||||
delete process.env.KLO_SQLITE_TEST_URL;
|
||||
delete process.env.KTX_SQLITE_TEST_URL;
|
||||
} else {
|
||||
process.env.KLO_SQLITE_TEST_URL = originalDatabaseUrl;
|
||||
process.env.KTX_SQLITE_TEST_URL = originalDatabaseUrl;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -4,16 +4,16 @@ import Database from 'better-sqlite3';
|
|||
import { readFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import type {
|
||||
KloSqlQueryExecutionInput,
|
||||
KloSqlQueryExecutionResult,
|
||||
KloSqlQueryExecutorPort,
|
||||
KtxSqlQueryExecutionInput,
|
||||
KtxSqlQueryExecutionResult,
|
||||
KtxSqlQueryExecutorPort,
|
||||
} from './query-executor.js';
|
||||
import { normalizeQueryRows } from './query-executor.js';
|
||||
import { limitSqlForExecution } from './read-only-sql.js';
|
||||
|
||||
type SqliteConnectionConfig = Record<string, unknown> | undefined;
|
||||
|
||||
function connectionDriver(input: KloSqlQueryExecutionInput): string {
|
||||
function connectionDriver(input: KtxSqlQueryExecutionInput): string {
|
||||
return String(input.connection?.driver ?? '').toLowerCase();
|
||||
}
|
||||
|
||||
|
|
@ -49,7 +49,7 @@ function sqlitePathFromUrl(url: string): string {
|
|||
return url;
|
||||
}
|
||||
|
||||
export function sqliteDatabasePathFromConnection(input: KloSqlQueryExecutionInput): string {
|
||||
export function sqliteDatabasePathFromConnection(input: KtxSqlQueryExecutionInput): string {
|
||||
const driver = connectionDriver(input);
|
||||
if (driver !== 'sqlite' && driver !== 'sqlite3') {
|
||||
throw new Error(`Local SQLite execution cannot run driver "${input.connection?.driver ?? 'unknown'}".`);
|
||||
|
|
@ -70,9 +70,9 @@ export function sqliteDatabasePathFromConnection(input: KloSqlQueryExecutionInpu
|
|||
return isAbsolute(candidate) ? candidate : resolve(input.projectDir ?? process.cwd(), candidate);
|
||||
}
|
||||
|
||||
export function createSqliteQueryExecutor(): KloSqlQueryExecutorPort {
|
||||
export function createSqliteQueryExecutor(): KtxSqlQueryExecutorPort {
|
||||
return {
|
||||
async execute(input: KloSqlQueryExecutionInput): Promise<KloSqlQueryExecutionResult> {
|
||||
async execute(input: KtxSqlQueryExecutionInput): Promise<KtxSqlQueryExecutionResult> {
|
||||
const sql = limitSqlForExecution(input.sql, input.maxRows);
|
||||
const dbPath = sqliteDatabasePathFromConnection(input);
|
||||
const db = new Database(dbPath, { readonly: true, fileMustExist: true });
|
||||
|
|
|
|||
|
|
@ -2,33 +2,33 @@ import { mkdir, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { resolveKloConfigReference, resolveKloHomePath } from './config-reference.js';
|
||||
import { resolveKtxConfigReference, resolveKtxHomePath } from './config-reference.js';
|
||||
|
||||
describe('KLO config references', () => {
|
||||
describe('KTX config references', () => {
|
||||
it('resolves env references without returning empty values', () => {
|
||||
expect(resolveKloConfigReference('env:AI_GATEWAY_API_KEY', { AI_GATEWAY_API_KEY: ' gateway-key ' })).toBe(
|
||||
expect(resolveKtxConfigReference('env:AI_GATEWAY_API_KEY', { AI_GATEWAY_API_KEY: ' gateway-key ' })).toBe(
|
||||
'gateway-key',
|
||||
);
|
||||
expect(resolveKloConfigReference('env:AI_GATEWAY_API_KEY', { AI_GATEWAY_API_KEY: ' ' })).toBeUndefined();
|
||||
expect(resolveKloConfigReference('env:AI_GATEWAY_API_KEY', {})).toBeUndefined();
|
||||
expect(resolveKtxConfigReference('env:AI_GATEWAY_API_KEY', { AI_GATEWAY_API_KEY: ' ' })).toBeUndefined();
|
||||
expect(resolveKtxConfigReference('env:AI_GATEWAY_API_KEY', {})).toBeUndefined();
|
||||
});
|
||||
|
||||
it('resolves file references and trims file content', async () => {
|
||||
const dir = join(tmpdir(), `klo-config-reference-${process.pid}`);
|
||||
const dir = join(tmpdir(), `ktx-config-reference-${process.pid}`);
|
||||
await mkdir(dir, { recursive: true });
|
||||
const keyPath = join(dir, 'gateway-key.txt');
|
||||
await writeFile(keyPath, 'file-gateway-key\n', 'utf8');
|
||||
|
||||
expect(resolveKloConfigReference(`file:${keyPath}`, {})).toBe('file-gateway-key');
|
||||
expect(resolveKtxConfigReference(`file:${keyPath}`, {})).toBe('file-gateway-key');
|
||||
});
|
||||
|
||||
it('returns literal values unchanged after trimming blank-only values', () => {
|
||||
expect(resolveKloConfigReference('provider/model', {})).toBe('provider/model');
|
||||
expect(resolveKloConfigReference(' ', {})).toBeUndefined();
|
||||
expect(resolveKloConfigReference(undefined, {})).toBeUndefined();
|
||||
expect(resolveKtxConfigReference('provider/model', {})).toBe('provider/model');
|
||||
expect(resolveKtxConfigReference(' ', {})).toBeUndefined();
|
||||
expect(resolveKtxConfigReference(undefined, {})).toBeUndefined();
|
||||
});
|
||||
|
||||
it('resolves home-prefixed paths', () => {
|
||||
expect(resolveKloHomePath('~/klo/key.txt')).toContain('/klo/key.txt');
|
||||
expect(resolveKtxHomePath('~/ktx/key.txt')).toContain('/ktx/key.txt');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { readFileSync } from 'node:fs';
|
|||
import { homedir } from 'node:os';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
export function resolveKloHomePath(path: string): string {
|
||||
export function resolveKtxHomePath(path: string): string {
|
||||
if (path === '~') {
|
||||
return homedir();
|
||||
}
|
||||
|
|
@ -14,7 +14,7 @@ export function resolveKloHomePath(path: string): string {
|
|||
return resolve(path);
|
||||
}
|
||||
|
||||
export function resolveKloConfigReference(value: string | undefined, env: NodeJS.ProcessEnv): string | undefined {
|
||||
export function resolveKtxConfigReference(value: string | undefined, env: NodeJS.ProcessEnv): string | undefined {
|
||||
if (!value) {
|
||||
return undefined;
|
||||
}
|
||||
|
|
@ -26,7 +26,7 @@ export function resolveKloConfigReference(value: string | undefined, env: NodeJS
|
|||
}
|
||||
|
||||
if (value.startsWith('file:')) {
|
||||
const filePath = resolveKloHomePath(value.slice('file:'.length).trim());
|
||||
const filePath = resolveKtxHomePath(value.slice('file:'.length).trim());
|
||||
const fileValue = readFileSync(filePath, 'utf8').trim();
|
||||
return fileValue.length > 0 ? fileValue : undefined;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
export interface KloStorageConfig {
|
||||
export interface KtxStorageConfig {
|
||||
configDir?: string;
|
||||
homeDir?: string;
|
||||
worktreesDir?: string;
|
||||
}
|
||||
|
||||
export interface KloGitConfig {
|
||||
export interface KtxGitConfig {
|
||||
userName: string;
|
||||
userEmail: string;
|
||||
bootstrapMessage?: string;
|
||||
|
|
@ -12,31 +12,31 @@ export interface KloGitConfig {
|
|||
bootstrapAuthorEmail?: string;
|
||||
}
|
||||
|
||||
export interface KloCoreConfig {
|
||||
storage: KloStorageConfig;
|
||||
git: KloGitConfig;
|
||||
export interface KtxCoreConfig {
|
||||
storage: KtxStorageConfig;
|
||||
git: KtxGitConfig;
|
||||
}
|
||||
|
||||
export interface KloLogger {
|
||||
export interface KtxLogger {
|
||||
debug(message: string): void;
|
||||
log(message: string): void;
|
||||
warn(message: string): void;
|
||||
error(message: string, error?: unknown): void;
|
||||
}
|
||||
|
||||
export const noopLogger: KloLogger = {
|
||||
export const noopLogger: KtxLogger = {
|
||||
debug: () => undefined,
|
||||
log: () => undefined,
|
||||
warn: () => undefined,
|
||||
error: () => undefined,
|
||||
};
|
||||
|
||||
export function resolveConfigDir(config: KloCoreConfig): string {
|
||||
export function resolveConfigDir(config: KtxCoreConfig): string {
|
||||
const homeDir = config.storage.homeDir ?? '/tmp';
|
||||
return config.storage.configDir ?? `${homeDir}/klo/config`;
|
||||
return config.storage.configDir ?? `${homeDir}/ktx/config`;
|
||||
}
|
||||
|
||||
export function resolveWorktreesDir(config: KloCoreConfig): string {
|
||||
export function resolveWorktreesDir(config: KtxCoreConfig): string {
|
||||
const homeDir = config.storage.homeDir ?? '/tmp';
|
||||
return config.storage.worktreesDir ?? `${homeDir}/.worktrees`;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
export interface KloEmbeddingPort {
|
||||
export interface KtxEmbeddingPort {
|
||||
maxBatchSize: number;
|
||||
computeEmbedding(text: string): Promise<number[]>;
|
||||
computeEmbeddingsBulk(texts: string[]): Promise<number[][]>;
|
||||
|
|
|
|||
|
|
@ -1,18 +1,18 @@
|
|||
export interface KloFileWriteResult {
|
||||
export interface KtxFileWriteResult {
|
||||
commitHash?: string | null;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface KloFileReadResult {
|
||||
export interface KtxFileReadResult {
|
||||
content: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface KloFileListResult {
|
||||
export interface KtxFileListResult {
|
||||
files: string[];
|
||||
}
|
||||
|
||||
export interface KloFileHistoryEntry {
|
||||
export interface KtxFileHistoryEntry {
|
||||
sha?: string;
|
||||
message?: string;
|
||||
author?: string;
|
||||
|
|
@ -20,7 +20,7 @@ export interface KloFileHistoryEntry {
|
|||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface KloFileStorePort<TSelf = unknown> {
|
||||
export interface KtxFileStorePort<TSelf = unknown> {
|
||||
writeFile(
|
||||
path: string,
|
||||
content: string,
|
||||
|
|
@ -28,16 +28,16 @@ export interface KloFileStorePort<TSelf = unknown> {
|
|||
authorEmail: string,
|
||||
commitMessage: string,
|
||||
options?: { skipLock?: boolean },
|
||||
): Promise<KloFileWriteResult>;
|
||||
readFile(path: string): Promise<KloFileReadResult>;
|
||||
): Promise<KtxFileWriteResult>;
|
||||
readFile(path: string): Promise<KtxFileReadResult>;
|
||||
deleteFile(
|
||||
path: string,
|
||||
author: string,
|
||||
authorEmail: string,
|
||||
commitMessage: string,
|
||||
options?: { skipLock?: boolean },
|
||||
): Promise<KloFileWriteResult | null>;
|
||||
listFiles(path: string, recursive?: boolean): Promise<KloFileListResult>;
|
||||
getFileHistory(path: string): Promise<KloFileHistoryEntry[] | unknown>;
|
||||
): Promise<KtxFileWriteResult | null>;
|
||||
listFiles(path: string, recursive?: boolean): Promise<KtxFileListResult>;
|
||||
getFileHistory(path: string): Promise<KtxFileHistoryEntry[] | unknown>;
|
||||
forWorktree(workdir: string): TSelf;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { SimpleGit } from 'simple-git';
|
||||
import type { KloCoreConfig } from './config.js';
|
||||
import type { KtxCoreConfig } from './config.js';
|
||||
import { createSimpleGit } from './git-env.js';
|
||||
import { GitService } from './git.service.js';
|
||||
|
||||
|
|
@ -21,7 +21,7 @@ describe('GitService.assertWorktreeClean', () => {
|
|||
await writeFile(join(workdir, 'init'), 'init');
|
||||
await git.add('.');
|
||||
await git.commit('init');
|
||||
const coreConfig: KloCoreConfig = {
|
||||
const coreConfig: KtxCoreConfig = {
|
||||
storage: { configDir: workdir, homeDir: workdir },
|
||||
git: { userName: 'Test', userEmail: 't@test' },
|
||||
};
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { mkdir, mkdtemp, readdir, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { SimpleGit } from 'simple-git';
|
||||
import type { KloCoreConfig } from './config.js';
|
||||
import type { KtxCoreConfig } from './config.js';
|
||||
import { createSimpleGit } from './git-env.js';
|
||||
import { GitService } from './git.service.js';
|
||||
|
||||
|
|
@ -22,7 +22,7 @@ describe('GitService.deleteDirectories', () => {
|
|||
await git.add('.');
|
||||
await git.commit('init');
|
||||
|
||||
const coreConfig: KloCoreConfig = {
|
||||
const coreConfig: KtxCoreConfig = {
|
||||
storage: { configDir: workdir, homeDir: workdir },
|
||||
git: { userName: 'Test', userEmail: 't@test' },
|
||||
};
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { SimpleGit } from 'simple-git';
|
||||
import type { KloCoreConfig } from './config.js';
|
||||
import type { KtxCoreConfig } from './config.js';
|
||||
import { createSimpleGit } from './git-env.js';
|
||||
import { GitService } from './git.service.js';
|
||||
|
||||
|
|
@ -21,7 +21,7 @@ describe('GitService.resetHardTo', () => {
|
|||
await writeFile(join(workdir, 'init'), 'init');
|
||||
await git.add('.');
|
||||
await git.commit('init');
|
||||
const coreConfig: KloCoreConfig = {
|
||||
const coreConfig: KtxCoreConfig = {
|
||||
storage: { configDir: workdir, homeDir: workdir },
|
||||
git: { userName: 'Test', userEmail: 't@test' },
|
||||
};
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { mkdtemp, realpath, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import type { KloCoreConfig } from './config.js';
|
||||
import type { KtxCoreConfig } from './config.js';
|
||||
import { GitService } from './git.service.js';
|
||||
|
||||
// These tests drive a real git repo inside a temp directory — simple-git shells out to the
|
||||
|
|
@ -15,7 +15,7 @@ describe('GitService', () => {
|
|||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'git-service-spec-'));
|
||||
|
||||
const coreConfig: KloCoreConfig = {
|
||||
const coreConfig: KtxCoreConfig = {
|
||||
storage: { configDir: tempDir, homeDir: tempDir },
|
||||
git: {
|
||||
userName: 'Test User',
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { promises as fs } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import type { SimpleGit } from 'simple-git';
|
||||
import { noopLogger, resolveConfigDir, type KloCoreConfig, type KloLogger } from './config.js';
|
||||
import { noopLogger, resolveConfigDir, type KtxCoreConfig, type KtxLogger } from './config.js';
|
||||
import { createSimpleGit } from './git-env.js';
|
||||
|
||||
export interface GitCommitInfo {
|
||||
|
|
@ -32,13 +32,13 @@ export type SquashMergeResult =
|
|||
| { ok: false; conflict: true; conflictPaths: string[] };
|
||||
|
||||
export class GitService {
|
||||
private readonly logger: KloLogger;
|
||||
private readonly logger: KtxLogger;
|
||||
private git!: SimpleGit;
|
||||
private configDir: string;
|
||||
|
||||
constructor(
|
||||
private readonly config: KloCoreConfig,
|
||||
logger?: KloLogger,
|
||||
private readonly config: KtxCoreConfig,
|
||||
logger?: KtxLogger,
|
||||
) {
|
||||
this.logger = logger ?? noopLogger;
|
||||
this.configDir = resolveConfigDir(config);
|
||||
|
|
@ -73,10 +73,10 @@ export class GitService {
|
|||
// can rely on `revParseHead()` returning a SHA. Idempotent: skip if HEAD already exists.
|
||||
const head = await this.revParseHead();
|
||||
if (!head) {
|
||||
await this.git.commit(this.config.git.bootstrapMessage ?? 'Initialize klo project repository', {
|
||||
await this.git.commit(this.config.git.bootstrapMessage ?? 'Initialize ktx project repository', {
|
||||
'--allow-empty': null,
|
||||
'--author': `${this.config.git.bootstrapAuthor ?? 'klo system'} <${
|
||||
this.config.git.bootstrapAuthorEmail ?? 'system@klo.local'
|
||||
'--author': `${this.config.git.bootstrapAuthor ?? 'ktx system'} <${
|
||||
this.config.git.bootstrapAuthorEmail ?? 'system@ktx.local'
|
||||
}>`,
|
||||
});
|
||||
this.logger.log('Wrote bootstrap commit to config repo');
|
||||
|
|
@ -676,7 +676,7 @@ export class GitService {
|
|||
|
||||
/**
|
||||
* Remove the worktree entry and its on-disk directory. Uses `--force` because session
|
||||
* worktrees are klo-internal — a clean working tree is not required.
|
||||
* worktrees are ktx-internal — a clean working tree is not required.
|
||||
*/
|
||||
async removeWorktree(path: string): Promise<void> {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -1,19 +1,19 @@
|
|||
export type { KloCoreConfig, KloGitConfig, KloLogger, KloStorageConfig } from './config.js';
|
||||
export type { KtxCoreConfig, KtxGitConfig, KtxLogger, KtxStorageConfig } from './config.js';
|
||||
export { noopLogger, resolveConfigDir, resolveWorktreesDir } from './config.js';
|
||||
export { resolveKloConfigReference, resolveKloHomePath } from './config-reference.js';
|
||||
export type { KloEmbeddingPort } from './embedding.js';
|
||||
export { resolveKtxConfigReference, resolveKtxHomePath } from './config-reference.js';
|
||||
export type { KtxEmbeddingPort } from './embedding.js';
|
||||
export {
|
||||
REDACTED_KLO_CREDENTIAL_VALUE,
|
||||
redactKloSensitiveMetadata,
|
||||
redactKloSensitiveText,
|
||||
redactKloSensitiveValue,
|
||||
REDACTED_KTX_CREDENTIAL_VALUE,
|
||||
redactKtxSensitiveMetadata,
|
||||
redactKtxSensitiveText,
|
||||
redactKtxSensitiveValue,
|
||||
} from './redaction.js';
|
||||
export type {
|
||||
KloFileHistoryEntry,
|
||||
KloFileListResult,
|
||||
KloFileReadResult,
|
||||
KloFileStorePort,
|
||||
KloFileWriteResult,
|
||||
KtxFileHistoryEntry,
|
||||
KtxFileListResult,
|
||||
KtxFileReadResult,
|
||||
KtxFileStorePort,
|
||||
KtxFileWriteResult,
|
||||
} from './file-store.js';
|
||||
export type { GitCommitInfo, SquashMergeResult, WorktreeEntry } from './git.service.js';
|
||||
export { GitService } from './git.service.js';
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
export const REDACTED_KLO_CREDENTIAL_VALUE = '<redacted>';
|
||||
export const REDACTED_KTX_CREDENTIAL_VALUE = '<redacted>';
|
||||
|
||||
const SENSITIVE_FIELD_NAME = /(password|secret|token|api[_-]?key|private[_-]?key|passphrase|credential|authorization|url)/i;
|
||||
const URL_CREDENTIAL_PATTERN = /([a-z][a-z0-9+.-]*:\/\/[^:\s/@]+:)([^@\s/]+)(@)/gi;
|
||||
|
|
@ -11,37 +11,37 @@ function isSensitiveField(key: string): boolean {
|
|||
return SENSITIVE_FIELD_NAME.test(key);
|
||||
}
|
||||
|
||||
export function redactKloSensitiveValue(key: string, value: unknown): unknown {
|
||||
export function redactKtxSensitiveValue(key: string, value: unknown): unknown {
|
||||
if (isSensitiveField(key)) {
|
||||
return REDACTED_KLO_CREDENTIAL_VALUE;
|
||||
return REDACTED_KTX_CREDENTIAL_VALUE;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((item) => redactKloSensitiveValue(key, item));
|
||||
return value.map((item) => redactKtxSensitiveValue(key, item));
|
||||
}
|
||||
if (isRecord(value)) {
|
||||
return redactKloSensitiveMetadata(value);
|
||||
return redactKtxSensitiveMetadata(value);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function redactKloSensitiveMetadata(metadata: Record<string, unknown>): Record<string, unknown> {
|
||||
export function redactKtxSensitiveMetadata(metadata: Record<string, unknown>): Record<string, unknown> {
|
||||
const redacted: Record<string, unknown> = {};
|
||||
for (const [key, value] of Object.entries(metadata)) {
|
||||
if (Array.isArray(value)) {
|
||||
redacted[key] = value.map((item) =>
|
||||
isRecord(item) ? redactKloSensitiveMetadata(item) : redactKloSensitiveValue(key, item),
|
||||
isRecord(item) ? redactKtxSensitiveMetadata(item) : redactKtxSensitiveValue(key, item),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (isRecord(value)) {
|
||||
redacted[key] = redactKloSensitiveValue(key, value);
|
||||
redacted[key] = redactKtxSensitiveValue(key, value);
|
||||
continue;
|
||||
}
|
||||
redacted[key] = redactKloSensitiveValue(key, value);
|
||||
redacted[key] = redactKtxSensitiveValue(key, value);
|
||||
}
|
||||
return redacted;
|
||||
}
|
||||
|
||||
export function redactKloSensitiveText(value: string): string {
|
||||
return value.replace(URL_CREDENTIAL_PATTERN, `$1${REDACTED_KLO_CREDENTIAL_VALUE}$3`);
|
||||
export function redactKtxSensitiveText(value: string): string {
|
||||
return value.replace(URL_CREDENTIAL_PATTERN, `$1${REDACTED_KTX_CREDENTIAL_VALUE}$3`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { mkdtemp, realpath, rm, stat } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { KloCoreConfig } from './config.js';
|
||||
import type { KtxCoreConfig } from './config.js';
|
||||
import { GitService } from './git.service.js';
|
||||
import { SessionWorktreeService, type WorktreeConfigPort } from './session-worktree.service.js';
|
||||
|
||||
|
|
@ -20,7 +20,7 @@ describe('SessionWorktreeService', () => {
|
|||
homeDir = await mkdtemp(join(tmpdir(), 'sws-spec-'));
|
||||
homeDir = await realpath(homeDir);
|
||||
|
||||
const coreConfig: KloCoreConfig = {
|
||||
const coreConfig: KtxCoreConfig = {
|
||||
storage: { configDir: homeDir, homeDir },
|
||||
git: {
|
||||
userName: 'System User',
|
||||
|
|
@ -113,7 +113,7 @@ describe('SessionWorktreeService', () => {
|
|||
await expect(stat(session.workdir)).resolves.toBeTruthy();
|
||||
|
||||
const { readFile } = await import('node:fs/promises');
|
||||
const raw = await readFile(join(session.workdir, '.klo-outcome'), 'utf-8');
|
||||
const raw = await readFile(join(session.workdir, '.ktx-outcome'), 'utf-8');
|
||||
const parsed = JSON.parse(raw);
|
||||
expect(parsed.outcome).toBe('conflict');
|
||||
expect(parsed.chatId).toBe('chat-cleanup-conflict');
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { mkdir, stat, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { noopLogger, resolveWorktreesDir, type KloCoreConfig, type KloLogger } from './config.js';
|
||||
import { noopLogger, resolveWorktreesDir, type KtxCoreConfig, type KtxLogger } from './config.js';
|
||||
import { GitService } from './git.service.js';
|
||||
|
||||
export type SessionOutcome = 'success' | 'empty' | 'conflict' | 'crash';
|
||||
|
|
@ -28,14 +28,14 @@ export interface SessionWorktree<TConfig> {
|
|||
}
|
||||
|
||||
export interface SessionWorktreeServiceDeps<TConfig extends WorktreeConfigPort<TConfig>> {
|
||||
coreConfig: KloCoreConfig;
|
||||
coreConfig: KtxCoreConfig;
|
||||
gitService: GitService;
|
||||
configService: TConfig;
|
||||
logger?: KloLogger;
|
||||
logger?: KtxLogger;
|
||||
}
|
||||
|
||||
export class SessionWorktreeService<TConfig extends WorktreeConfigPort<TConfig> = WorktreeConfigPort<never>> {
|
||||
private readonly logger: KloLogger;
|
||||
private readonly logger: KtxLogger;
|
||||
private readonly worktreesRoot: string;
|
||||
|
||||
constructor(private readonly deps: SessionWorktreeServiceDeps<TConfig>) {
|
||||
|
|
@ -101,7 +101,7 @@ export class SessionWorktreeService<TConfig extends WorktreeConfigPort<TConfig>
|
|||
...(extra?.conflictPaths ? { conflictPaths: extra.conflictPaths } : {}),
|
||||
};
|
||||
try {
|
||||
await writeFile(join(session.workdir, '.klo-outcome'), JSON.stringify(payload, null, 2), 'utf-8');
|
||||
await writeFile(join(session.workdir, '.ktx-outcome'), JSON.stringify(payload, null, 2), 'utf-8');
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`cleanup(${outcome}) failed to write sentinel for ${session.chatId}: ${
|
||||
|
|
|
|||
|
|
@ -4,21 +4,21 @@ import { URL } from 'node:url';
|
|||
import { spawn } from 'node:child_process';
|
||||
import type { SemanticLayerQueryInput, SemanticLayerSource } from '../sl/index.js';
|
||||
|
||||
export interface KloSemanticLayerComputeQueryResult {
|
||||
export interface KtxSemanticLayerComputeQueryResult {
|
||||
sql: string;
|
||||
dialect: string;
|
||||
columns: Array<Record<string, unknown>>;
|
||||
plan: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface KloSemanticLayerComputeValidationResult {
|
||||
export interface KtxSemanticLayerComputeValidationResult {
|
||||
valid: boolean;
|
||||
errors: string[];
|
||||
warnings: string[];
|
||||
perSourceWarnings: Record<string, string[]>;
|
||||
}
|
||||
|
||||
export interface KloSemanticLayerSourceGenerationColumnInput {
|
||||
export interface KtxSemanticLayerSourceGenerationColumnInput {
|
||||
name: string;
|
||||
type: string;
|
||||
primaryKey?: boolean;
|
||||
|
|
@ -26,15 +26,15 @@ export interface KloSemanticLayerSourceGenerationColumnInput {
|
|||
comment?: string | null;
|
||||
}
|
||||
|
||||
export interface KloSemanticLayerSourceGenerationTableInput {
|
||||
export interface KtxSemanticLayerSourceGenerationTableInput {
|
||||
name: string;
|
||||
catalog?: string | null;
|
||||
db?: string | null;
|
||||
comment?: string | null;
|
||||
columns: KloSemanticLayerSourceGenerationColumnInput[];
|
||||
columns: KtxSemanticLayerSourceGenerationColumnInput[];
|
||||
}
|
||||
|
||||
export interface KloSemanticLayerSourceGenerationLinkInput {
|
||||
export interface KtxSemanticLayerSourceGenerationLinkInput {
|
||||
fromTable: string;
|
||||
fromColumn: string;
|
||||
toTable: string;
|
||||
|
|
@ -42,57 +42,57 @@ export interface KloSemanticLayerSourceGenerationLinkInput {
|
|||
relationshipType: string;
|
||||
}
|
||||
|
||||
export interface KloSemanticLayerSourceGenerationInput {
|
||||
tables: KloSemanticLayerSourceGenerationTableInput[];
|
||||
links: KloSemanticLayerSourceGenerationLinkInput[];
|
||||
export interface KtxSemanticLayerSourceGenerationInput {
|
||||
tables: KtxSemanticLayerSourceGenerationTableInput[];
|
||||
links: KtxSemanticLayerSourceGenerationLinkInput[];
|
||||
dialect?: string;
|
||||
}
|
||||
|
||||
export interface KloSemanticLayerSourceGenerationResult {
|
||||
export interface KtxSemanticLayerSourceGenerationResult {
|
||||
sources: Array<Record<string, unknown>>;
|
||||
sourceCount: number;
|
||||
}
|
||||
|
||||
export interface KloSemanticLayerComputePort {
|
||||
export interface KtxSemanticLayerComputePort {
|
||||
query(input: {
|
||||
sources: Array<Record<string, unknown> | SemanticLayerSource>;
|
||||
query: SemanticLayerQueryInput;
|
||||
dialect: string;
|
||||
}): Promise<KloSemanticLayerComputeQueryResult>;
|
||||
}): Promise<KtxSemanticLayerComputeQueryResult>;
|
||||
validateSources(input: {
|
||||
sources: Array<Record<string, unknown> | SemanticLayerSource>;
|
||||
dialect: string;
|
||||
recentlyTouched?: string[];
|
||||
}): Promise<KloSemanticLayerComputeValidationResult>;
|
||||
generateSources(input: KloSemanticLayerSourceGenerationInput): Promise<KloSemanticLayerSourceGenerationResult>;
|
||||
}): Promise<KtxSemanticLayerComputeValidationResult>;
|
||||
generateSources(input: KtxSemanticLayerSourceGenerationInput): Promise<KtxSemanticLayerSourceGenerationResult>;
|
||||
}
|
||||
|
||||
export type KloDaemonCommand = 'semantic-query' | 'semantic-validate' | 'semantic-generate-sources';
|
||||
export type KtxDaemonCommand = 'semantic-query' | 'semantic-validate' | 'semantic-generate-sources';
|
||||
|
||||
export type KloDaemonJsonRunner = (
|
||||
subcommand: KloDaemonCommand,
|
||||
export type KtxDaemonJsonRunner = (
|
||||
subcommand: KtxDaemonCommand,
|
||||
payload: Record<string, unknown>,
|
||||
) => Promise<Record<string, unknown>>;
|
||||
|
||||
export type KloDaemonHttpJsonRunner = (path: string, payload: Record<string, unknown>) => Promise<Record<string, unknown>>;
|
||||
export type KtxDaemonHttpJsonRunner = (path: string, payload: Record<string, unknown>) => Promise<Record<string, unknown>>;
|
||||
|
||||
export interface PythonSemanticLayerComputeOptions {
|
||||
command?: string;
|
||||
args?: string[];
|
||||
cwd?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
runJson?: KloDaemonJsonRunner;
|
||||
runJson?: KtxDaemonJsonRunner;
|
||||
}
|
||||
|
||||
export interface HttpSemanticLayerComputeOptions {
|
||||
baseUrl: string;
|
||||
requestJson?: KloDaemonHttpJsonRunner;
|
||||
requestJson?: KtxDaemonHttpJsonRunner;
|
||||
}
|
||||
|
||||
function parseJsonObject(raw: string, subcommand: string): Record<string, unknown> {
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
throw new Error(`klo-daemon ${subcommand} returned non-object JSON`);
|
||||
throw new Error(`ktx-daemon ${subcommand} returned non-object JSON`);
|
||||
}
|
||||
return parsed as Record<string, unknown>;
|
||||
}
|
||||
|
|
@ -100,8 +100,8 @@ function parseJsonObject(raw: string, subcommand: string): Record<string, unknow
|
|||
function runProcessJson(
|
||||
options: Required<Pick<PythonSemanticLayerComputeOptions, 'command' | 'args'>> &
|
||||
Pick<PythonSemanticLayerComputeOptions, 'cwd' | 'env'>,
|
||||
): KloDaemonJsonRunner {
|
||||
return async (subcommand: KloDaemonCommand, payload: Record<string, unknown>): Promise<Record<string, unknown>> =>
|
||||
): KtxDaemonJsonRunner {
|
||||
return async (subcommand: KtxDaemonCommand, payload: Record<string, unknown>): Promise<Record<string, unknown>> =>
|
||||
new Promise((resolve, reject) => {
|
||||
const child = spawn(options.command, [...options.args, subcommand], {
|
||||
cwd: options.cwd,
|
||||
|
|
@ -118,7 +118,7 @@ function runProcessJson(
|
|||
const stdoutText = Buffer.concat(stdout).toString('utf8').trim();
|
||||
const stderrText = Buffer.concat(stderr).toString('utf8').trim();
|
||||
if (code !== 0) {
|
||||
reject(new Error(`klo-daemon ${subcommand} failed: ${stderrText || `exit code ${code}`}`));
|
||||
reject(new Error(`ktx-daemon ${subcommand} failed: ${stderrText || `exit code ${code}`}`));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
|
|
@ -135,7 +135,7 @@ function normalizedBaseUrl(baseUrl: string): string {
|
|||
return baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`;
|
||||
}
|
||||
|
||||
function postJson(baseUrl: string): KloDaemonHttpJsonRunner {
|
||||
function postJson(baseUrl: string): KtxDaemonHttpJsonRunner {
|
||||
return async (path, payload) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const target = new URL(path.replace(/^\//, ''), normalizedBaseUrl(baseUrl));
|
||||
|
|
@ -158,7 +158,7 @@ function postJson(baseUrl: string): KloDaemonHttpJsonRunner {
|
|||
const text = Buffer.concat(chunks).toString('utf8');
|
||||
const statusCode = response.statusCode ?? 0;
|
||||
if (statusCode < 200 || statusCode >= 300) {
|
||||
reject(new Error(`klo-daemon HTTP ${path} failed with ${statusCode}: ${text}`));
|
||||
reject(new Error(`ktx-daemon HTTP ${path} failed with ${statusCode}: ${text}`));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
|
|
@ -190,7 +190,7 @@ function recordArray(value: unknown): Array<Record<string, unknown>> {
|
|||
: [];
|
||||
}
|
||||
|
||||
function sourceGenerationPayload(input: KloSemanticLayerSourceGenerationInput): Record<string, unknown> {
|
||||
function sourceGenerationPayload(input: KtxSemanticLayerSourceGenerationInput): Record<string, unknown> {
|
||||
return {
|
||||
tables: input.tables.map((table) => ({
|
||||
name: table.name,
|
||||
|
|
@ -216,7 +216,7 @@ function sourceGenerationPayload(input: KloSemanticLayerSourceGenerationInput):
|
|||
};
|
||||
}
|
||||
|
||||
function sourceGenerationResult(raw: Record<string, unknown>): KloSemanticLayerSourceGenerationResult {
|
||||
function sourceGenerationResult(raw: Record<string, unknown>): KtxSemanticLayerSourceGenerationResult {
|
||||
return {
|
||||
sources: recordArray(raw.sources),
|
||||
sourceCount: typeof raw.source_count === 'number' ? raw.source_count : recordArray(raw.sources).length,
|
||||
|
|
@ -225,9 +225,9 @@ function sourceGenerationResult(raw: Record<string, unknown>): KloSemanticLayerS
|
|||
|
||||
export function createPythonSemanticLayerComputePort(
|
||||
options: PythonSemanticLayerComputeOptions = {},
|
||||
): KloSemanticLayerComputePort {
|
||||
): KtxSemanticLayerComputePort {
|
||||
const command = options.command ?? 'python';
|
||||
const args = options.args ?? ['-m', 'klo_daemon'];
|
||||
const args = options.args ?? ['-m', 'ktx_daemon'];
|
||||
const runJson = options.runJson ?? runProcessJson({ command, args, cwd: options.cwd, env: options.env });
|
||||
|
||||
return {
|
||||
|
|
@ -266,7 +266,7 @@ export function createPythonSemanticLayerComputePort(
|
|||
|
||||
export function createHttpSemanticLayerComputePort(
|
||||
options: HttpSemanticLayerComputeOptions,
|
||||
): KloSemanticLayerComputePort {
|
||||
): KtxSemanticLayerComputePort {
|
||||
const requestJson = options.requestJson ?? postJson(options.baseUrl);
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { kloContextPackageInfo } from './index.js';
|
||||
import { ktxContextPackageInfo } from './index.js';
|
||||
|
||||
describe('kloContextPackageInfo', () => {
|
||||
describe('ktxContextPackageInfo', () => {
|
||||
it('identifies the context package', () => {
|
||||
expect(kloContextPackageInfo).toEqual({
|
||||
name: '@klo/context',
|
||||
expect(ktxContextPackageInfo).toEqual({
|
||||
name: '@ktx/context',
|
||||
version: '0.0.0-private',
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
export interface KloContextPackageInfo {
|
||||
name: '@klo/context';
|
||||
export interface KtxContextPackageInfo {
|
||||
name: '@ktx/context';
|
||||
version: '0.0.0-private';
|
||||
}
|
||||
|
||||
export const kloContextPackageInfo: KloContextPackageInfo = {
|
||||
name: '@klo/context',
|
||||
export const ktxContextPackageInfo: KtxContextPackageInfo = {
|
||||
name: '@ktx/context',
|
||||
version: '0.0.0-private',
|
||||
};
|
||||
|
||||
|
|
@ -36,107 +36,107 @@ export * from './prompts/index.js';
|
|||
export * from './search/index.js';
|
||||
export * from './sql-analysis/index.js';
|
||||
export type {
|
||||
KloColumnAnalysisResult,
|
||||
KloColumnDescriptionPromptInput,
|
||||
KloColumnEmbeddingForeignKeys,
|
||||
KloColumnEmbeddingTextInput,
|
||||
KloColumnSampleInput,
|
||||
KloColumnSampleResult,
|
||||
KloColumnSampleUpdate,
|
||||
KloColumnStatsInput,
|
||||
KloColumnStatsResult,
|
||||
KloConnectionDriver,
|
||||
KloConnectorCapabilities,
|
||||
KloCredentialEnvelope,
|
||||
KloCredentialEnvReference,
|
||||
KloCredentialFileReference,
|
||||
KloDataDictionaryColumnState,
|
||||
KloDataDictionarySampleDecision,
|
||||
KloDataDictionarySettings,
|
||||
KloDataDictionarySkipReason,
|
||||
KloDataSourceDescriptionPromptInput,
|
||||
KloDescriptionCachePort,
|
||||
KloDescriptionColumn,
|
||||
KloDescriptionColumnTable,
|
||||
KloDescriptionGenerationSettings,
|
||||
KloDescriptionGeneratorOptions,
|
||||
KloDescriptionSource,
|
||||
KloDescriptionTableInput,
|
||||
KloDescriptionUpdate,
|
||||
KloEmbeddingPort as KloScanEmbeddingPort,
|
||||
KloEmbeddingUpdate,
|
||||
KloEnrichedColumn,
|
||||
KloEnrichedRelationship,
|
||||
KloEnrichedSchema,
|
||||
KloEnrichedTable,
|
||||
KloEnrichmentScanPhaseResult,
|
||||
KloGenerateColumnDescriptionsInput,
|
||||
KloGenerateDataSourceDescriptionInput,
|
||||
KloGenerateTableDescriptionInput,
|
||||
KloOptionalConnectorCapabilities,
|
||||
KloProgressPort,
|
||||
KloQueryResult as KloScanQueryResult,
|
||||
KloReadOnlyQueryInput,
|
||||
KloRelationshipEndpoint,
|
||||
KloRelationshipSource,
|
||||
KloRelationshipType,
|
||||
KloRelationshipUpdate,
|
||||
KloResolvedCredentialEnvelope,
|
||||
KloScanArtifactPaths,
|
||||
KloScanConnector,
|
||||
KloScanContext,
|
||||
KloScanDiffSummary,
|
||||
KloScanEnrichmentSummary,
|
||||
KloScanInput,
|
||||
KloScanLoggerPort,
|
||||
KloScanMetadataStore,
|
||||
KloScanMode,
|
||||
KloScanOrchestratorOptions,
|
||||
KloScanOrchestratorRunInput,
|
||||
KloScanOrchestratorRunResult,
|
||||
KloScanRelationshipSummary,
|
||||
KloScanReport,
|
||||
KloScanTrigger,
|
||||
KloScanWarning,
|
||||
KloScanWarningCode,
|
||||
KloSchemaColumn,
|
||||
KloSchemaDimensionType,
|
||||
KloSchemaForeignKey,
|
||||
KloSchemaScope,
|
||||
KloSchemaSnapshot,
|
||||
KloSchemaTable,
|
||||
KloSchemaTableKind,
|
||||
KloSkippedRelationship,
|
||||
KloStructuralScanPhaseResult,
|
||||
KloStructuralSyncPlan,
|
||||
KloStructuralSyncStats,
|
||||
KloTableDescriptionPromptInput,
|
||||
KloTableRef,
|
||||
KloTableSampleInput,
|
||||
KloTableSampleResult,
|
||||
KloColumnTypeMapping,
|
||||
KtxColumnAnalysisResult,
|
||||
KtxColumnDescriptionPromptInput,
|
||||
KtxColumnEmbeddingForeignKeys,
|
||||
KtxColumnEmbeddingTextInput,
|
||||
KtxColumnSampleInput,
|
||||
KtxColumnSampleResult,
|
||||
KtxColumnSampleUpdate,
|
||||
KtxColumnStatsInput,
|
||||
KtxColumnStatsResult,
|
||||
KtxConnectionDriver,
|
||||
KtxConnectorCapabilities,
|
||||
KtxCredentialEnvelope,
|
||||
KtxCredentialEnvReference,
|
||||
KtxCredentialFileReference,
|
||||
KtxDataDictionaryColumnState,
|
||||
KtxDataDictionarySampleDecision,
|
||||
KtxDataDictionarySettings,
|
||||
KtxDataDictionarySkipReason,
|
||||
KtxDataSourceDescriptionPromptInput,
|
||||
KtxDescriptionCachePort,
|
||||
KtxDescriptionColumn,
|
||||
KtxDescriptionColumnTable,
|
||||
KtxDescriptionGenerationSettings,
|
||||
KtxDescriptionGeneratorOptions,
|
||||
KtxDescriptionSource,
|
||||
KtxDescriptionTableInput,
|
||||
KtxDescriptionUpdate,
|
||||
KtxEmbeddingPort as KtxScanEmbeddingPort,
|
||||
KtxEmbeddingUpdate,
|
||||
KtxEnrichedColumn,
|
||||
KtxEnrichedRelationship,
|
||||
KtxEnrichedSchema,
|
||||
KtxEnrichedTable,
|
||||
KtxEnrichmentScanPhaseResult,
|
||||
KtxGenerateColumnDescriptionsInput,
|
||||
KtxGenerateDataSourceDescriptionInput,
|
||||
KtxGenerateTableDescriptionInput,
|
||||
KtxOptionalConnectorCapabilities,
|
||||
KtxProgressPort,
|
||||
KtxQueryResult as KtxScanQueryResult,
|
||||
KtxReadOnlyQueryInput,
|
||||
KtxRelationshipEndpoint,
|
||||
KtxRelationshipSource,
|
||||
KtxRelationshipType,
|
||||
KtxRelationshipUpdate,
|
||||
KtxResolvedCredentialEnvelope,
|
||||
KtxScanArtifactPaths,
|
||||
KtxScanConnector,
|
||||
KtxScanContext,
|
||||
KtxScanDiffSummary,
|
||||
KtxScanEnrichmentSummary,
|
||||
KtxScanInput,
|
||||
KtxScanLoggerPort,
|
||||
KtxScanMetadataStore,
|
||||
KtxScanMode,
|
||||
KtxScanOrchestratorOptions,
|
||||
KtxScanOrchestratorRunInput,
|
||||
KtxScanOrchestratorRunResult,
|
||||
KtxScanRelationshipSummary,
|
||||
KtxScanReport,
|
||||
KtxScanTrigger,
|
||||
KtxScanWarning,
|
||||
KtxScanWarningCode,
|
||||
KtxSchemaColumn,
|
||||
KtxSchemaDimensionType,
|
||||
KtxSchemaForeignKey,
|
||||
KtxSchemaScope,
|
||||
KtxSchemaSnapshot,
|
||||
KtxSchemaTable,
|
||||
KtxSchemaTableKind,
|
||||
KtxSkippedRelationship,
|
||||
KtxStructuralScanPhaseResult,
|
||||
KtxStructuralSyncPlan,
|
||||
KtxStructuralSyncStats,
|
||||
KtxTableDescriptionPromptInput,
|
||||
KtxTableRef,
|
||||
KtxTableSampleInput,
|
||||
KtxTableSampleResult,
|
||||
KtxColumnTypeMapping,
|
||||
} from './scan/index.js';
|
||||
export {
|
||||
appendKloWordLimitInstruction,
|
||||
buildKloColumnDescriptionPrompt,
|
||||
buildKloColumnEmbeddingText,
|
||||
buildKloDataSourceDescriptionPrompt,
|
||||
buildKloTableDescriptionPrompt,
|
||||
createKloConnectorCapabilities,
|
||||
defaultKloDataDictionarySettings,
|
||||
inferKloDimensionType,
|
||||
isKloDataDictionaryCandidate,
|
||||
kloColumnTypeMappingFromNative,
|
||||
KloDescriptionGenerator,
|
||||
KloScanOrchestrator,
|
||||
normalizeKloNativeType,
|
||||
REDACTED_KLO_CREDENTIAL_VALUE,
|
||||
redactKloCredentialEnvelope,
|
||||
redactKloCredentialValue,
|
||||
redactKloScanMetadata,
|
||||
redactKloScanReport,
|
||||
redactKloScanWarning,
|
||||
shouldKloSampleColumnForDictionary,
|
||||
appendKtxWordLimitInstruction,
|
||||
buildKtxColumnDescriptionPrompt,
|
||||
buildKtxColumnEmbeddingText,
|
||||
buildKtxDataSourceDescriptionPrompt,
|
||||
buildKtxTableDescriptionPrompt,
|
||||
createKtxConnectorCapabilities,
|
||||
defaultKtxDataDictionarySettings,
|
||||
inferKtxDimensionType,
|
||||
isKtxDataDictionaryCandidate,
|
||||
ktxColumnTypeMappingFromNative,
|
||||
KtxDescriptionGenerator,
|
||||
KtxScanOrchestrator,
|
||||
normalizeKtxNativeType,
|
||||
REDACTED_KTX_CREDENTIAL_VALUE,
|
||||
redactKtxCredentialEnvelope,
|
||||
redactKtxCredentialValue,
|
||||
redactKtxScanMetadata,
|
||||
redactKtxScanReport,
|
||||
redactKtxScanWarning,
|
||||
shouldKtxSampleColumnForDictionary,
|
||||
} from './scan/index.js';
|
||||
export * from './skills/index.js';
|
||||
export * from './sl/index.js';
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type { DbtParsedTable } from './parse-schema.js';
|
||||
import { findMatchingKloTable, matchDbtTables, type DbtHostTableLite } from './match-tables.js';
|
||||
import { findMatchingKtxTable, matchDbtTables, type DbtHostTableLite } from './match-tables.js';
|
||||
|
||||
const hostTables: DbtHostTableLite[] = [
|
||||
{ id: '1', name: 'orders', catalog: 'warehouse', db: 'analytics', columns: [{ id: 'c1', name: 'id' }] },
|
||||
|
|
@ -23,20 +23,20 @@ function table(input: Partial<DbtParsedTable>): DbtParsedTable {
|
|||
describe('dbt descriptions table matching', () => {
|
||||
it('uses schema plus name first and checks catalog when dbt database is present', () => {
|
||||
expect(
|
||||
findMatchingKloTable(table({ database: 'warehouse', schema: 'analytics' }), hostTables, null)?.id,
|
||||
findMatchingKtxTable(table({ database: 'warehouse', schema: 'analytics' }), hostTables, null)?.id,
|
||||
).toBe('1');
|
||||
});
|
||||
|
||||
it('does not fall back to name-only for source tables', () => {
|
||||
expect(findMatchingKloTable(table({ resourceType: 'source' }), hostTables, null)).toBeUndefined();
|
||||
expect(findMatchingKtxTable(table({ resourceType: 'source' }), hostTables, null)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('uses targetSchema for models and name-only only when unique', () => {
|
||||
expect(findMatchingKloTable(table({ resourceType: 'model' }), hostTables, 'staging')?.id).toBe('2');
|
||||
expect(findMatchingKloTable(table({ name: 'customers', resourceType: 'model' }), hostTables, null)?.id).toBe(
|
||||
expect(findMatchingKtxTable(table({ resourceType: 'model' }), hostTables, 'staging')?.id).toBe('2');
|
||||
expect(findMatchingKtxTable(table({ name: 'customers', resourceType: 'model' }), hostTables, null)?.id).toBe(
|
||||
'3',
|
||||
);
|
||||
expect(findMatchingKloTable(table({ resourceType: 'model' }), hostTables, null)).toBeUndefined();
|
||||
expect(findMatchingKtxTable(table({ resourceType: 'model' }), hostTables, null)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('summarizes matched columns and descriptions', () => {
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ export function matchDbtTables(
|
|||
targetSchema?: string | null,
|
||||
): DbtTableMatch[] {
|
||||
return dbtTables.map((dbtTable) => {
|
||||
const hostTable = findMatchingKloTable(dbtTable, hostTables, targetSchema);
|
||||
const hostTable = findMatchingKtxTable(dbtTable, hostTables, targetSchema);
|
||||
|
||||
if (!hostTable) {
|
||||
return {
|
||||
|
|
@ -63,7 +63,7 @@ export function matchDbtTables(
|
|||
});
|
||||
}
|
||||
|
||||
export function findMatchingKloTable(
|
||||
export function findMatchingKtxTable(
|
||||
dbtTable: DbtParsedTable,
|
||||
hostTables: DbtHostTableLite[],
|
||||
targetSchema?: string | null,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { parse as parseYaml } from 'yaml';
|
||||
import { type KloLogger, noopLogger } from '../../../core/index.js';
|
||||
import { type KtxLogger, noopLogger } from '../../../core/index.js';
|
||||
import { resolveJinjaVariables } from '../../dbt-shared/project-vars.js';
|
||||
|
||||
export interface DbtParsedColumn {
|
||||
|
|
@ -65,7 +65,7 @@ interface ParseDbtSchemaOptions {
|
|||
path?: string;
|
||||
variables?: Map<string, string>;
|
||||
projectName?: string | null;
|
||||
logger?: KloLogger;
|
||||
logger?: KtxLogger;
|
||||
}
|
||||
|
||||
interface DbtSchemaYaml {
|
||||
|
|
@ -133,7 +133,7 @@ export function parseDbtSchemaFile(content: string, options: ParseDbtSchemaOptio
|
|||
export function parseDbtSchemaFiles(
|
||||
files: DbtSchemaFile[],
|
||||
variables?: Map<string, string>,
|
||||
options: { projectName?: string | null; logger?: KloLogger } = {},
|
||||
options: { projectName?: string | null; logger?: KtxLogger } = {},
|
||||
): DbtSchemaParseResult {
|
||||
return new DbtSchemaParser(options.logger ?? noopLogger).parseFiles(files, variables, options.projectName ?? null);
|
||||
}
|
||||
|
|
@ -147,7 +147,7 @@ export function computeDbtSchemaHash(files: DbtSchemaFile[]): string {
|
|||
}
|
||||
|
||||
class DbtSchemaParser {
|
||||
constructor(private readonly logger: KloLogger) {}
|
||||
constructor(private readonly logger: KtxLogger) {}
|
||||
|
||||
parseFile(yamlContent: string, options: ParseDbtSchemaOptions = {}): DbtSchemaParseResult {
|
||||
this.logger.debug(`Parsing schema file: ${options.path ?? 'unknown'}`);
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import type { KloDescriptionUpdate } from '../../../scan/enrichment-types.js';
|
||||
import { findMatchingKloTable, type DbtHostTableLite } from './match-tables.js';
|
||||
import type { KtxDescriptionUpdate } from '../../../scan/enrichment-types.js';
|
||||
import { findMatchingKtxTable, type DbtHostTableLite } from './match-tables.js';
|
||||
import type { DbtSchemaParseResult } from './parse-schema.js';
|
||||
|
||||
export interface DbtDescriptionUpdates {
|
||||
dbt: KloDescriptionUpdate[];
|
||||
aiInvalidations: KloDescriptionUpdate[];
|
||||
dbt: KtxDescriptionUpdate[];
|
||||
aiInvalidations: KtxDescriptionUpdate[];
|
||||
}
|
||||
|
||||
export function toDescriptionUpdates(input: {
|
||||
|
|
@ -13,11 +13,11 @@ export function toDescriptionUpdates(input: {
|
|||
hostTables: DbtHostTableLite[];
|
||||
targetSchema: string | null;
|
||||
}): DbtDescriptionUpdates {
|
||||
const dbt: KloDescriptionUpdate[] = [];
|
||||
const aiInvalidations: KloDescriptionUpdate[] = [];
|
||||
const dbt: KtxDescriptionUpdate[] = [];
|
||||
const aiInvalidations: KtxDescriptionUpdate[] = [];
|
||||
|
||||
for (const dbtTable of input.parseResult.tables) {
|
||||
const hostTable = findMatchingKloTable(dbtTable, input.hostTables, input.targetSchema);
|
||||
const hostTable = findMatchingKtxTable(dbtTable, input.hostTables, input.targetSchema);
|
||||
if (!hostTable) {
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import type { KloMetadataUpdate } from '../../../scan/enrichment-types.js';
|
||||
import { findMatchingKloTable, type DbtHostTableLite } from './match-tables.js';
|
||||
import type { KtxMetadataUpdate } from '../../../scan/enrichment-types.js';
|
||||
import { findMatchingKtxTable, type DbtHostTableLite } from './match-tables.js';
|
||||
import type { DbtSchemaParseResult } from './parse-schema.js';
|
||||
|
||||
export function toMetadataUpdates(input: {
|
||||
|
|
@ -7,11 +7,11 @@ export function toMetadataUpdates(input: {
|
|||
parseResult: DbtSchemaParseResult;
|
||||
hostTables: DbtHostTableLite[];
|
||||
targetSchema: string | null;
|
||||
}): KloMetadataUpdate[] {
|
||||
const updates: KloMetadataUpdate[] = [];
|
||||
}): KtxMetadataUpdate[] {
|
||||
const updates: KtxMetadataUpdate[] = [];
|
||||
|
||||
for (const dbtTable of input.parseResult.tables) {
|
||||
const hostTable = findMatchingKloTable(dbtTable, input.hostTables, input.targetSchema);
|
||||
const hostTable = findMatchingKtxTable(dbtTable, input.hostTables, input.targetSchema);
|
||||
if (!hostTable) {
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import type { KloJoinUpdate } from '../../../scan/enrichment-types.js';
|
||||
import type { KtxJoinUpdate } from '../../../scan/enrichment-types.js';
|
||||
import type { DbtHostTableLite } from './match-tables.js';
|
||||
import type { DbtSchemaParseResult } from './parse-schema.js';
|
||||
|
||||
export interface DbtRelationshipUpdates {
|
||||
joins: KloJoinUpdate[];
|
||||
joins: KtxJoinUpdate[];
|
||||
skippedNoMatch: number;
|
||||
}
|
||||
|
||||
|
|
@ -19,7 +19,7 @@ export function toRelationshipUpdates(input: {
|
|||
tablesByName.set(table.name.toLowerCase(), table);
|
||||
}
|
||||
|
||||
const joins: KloJoinUpdate[] = [];
|
||||
const joins: KtxJoinUpdate[] = [];
|
||||
let skippedNoMatch = 0;
|
||||
|
||||
for (const relationship of input.parseResult.relationships) {
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ export class DbtSourceAdapter implements SourceAdapter {
|
|||
}
|
||||
await fetchDbtRepo({
|
||||
config,
|
||||
cacheDir: join(this.options.homeDir ?? '.klo/cache', 'dbt', ctx.connectionId),
|
||||
cacheDir: join(this.options.homeDir ?? '.ktx/cache', 'dbt', ctx.connectionId),
|
||||
stagedDir,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ describe('fetchDbtRepo', () => {
|
|||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-dbt-fetch-'));
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-dbt-fetch-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import {
|
|||
HistoricSqlVersionUnsupportedError,
|
||||
} from './errors.js';
|
||||
import type {
|
||||
KloPostgresQueryClient,
|
||||
KtxPostgresQueryClient,
|
||||
PostgresPgssProbeResult,
|
||||
PostgresPgssReader,
|
||||
PostgresPgssRow,
|
||||
|
|
@ -58,19 +58,19 @@ const POSTGRES_EXTENSION_REMEDIATION = [
|
|||
|
||||
const POSTGRES_GRANTS_REMEDIATION = 'GRANT pg_read_all_stats TO <connection role>;';
|
||||
|
||||
function queryClient(client: unknown): KloPostgresQueryClient {
|
||||
function queryClient(client: unknown): KtxPostgresQueryClient {
|
||||
if (
|
||||
client &&
|
||||
typeof client === 'object' &&
|
||||
'executeQuery' in client &&
|
||||
typeof (client as { executeQuery?: unknown }).executeQuery === 'function'
|
||||
) {
|
||||
return client as KloPostgresQueryClient;
|
||||
return client as KtxPostgresQueryClient;
|
||||
}
|
||||
throw new Error('Historic SQL Postgres PGSS reader requires a query client with executeQuery(sql, params?)');
|
||||
}
|
||||
|
||||
async function execute(client: KloPostgresQueryClient, sql: string, params?: unknown[]): Promise<QueryResultLike> {
|
||||
async function execute(client: KtxPostgresQueryClient, sql: string, params?: unknown[]): Promise<QueryResultLike> {
|
||||
const result = await client.executeQuery(sql, params);
|
||||
if ('error' in result && typeof result.error === 'string' && result.error.length > 0) {
|
||||
throw new Error(result.error);
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { dirname, join, relative } from 'node:path';
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type { SqlAnalysisPort } from '../../../sql-analysis/index.js';
|
||||
import { stagePgStatStatementsTemplates, writePgssBaselineAtomic, type PgssBaseline } from './stage-pgss.js';
|
||||
import type { HistoricSqlPullConfig, KloPostgresQueryClient, PostgresPgssReader, PostgresPgssRow } from './types.js';
|
||||
import type { HistoricSqlPullConfig, KtxPostgresQueryClient, PostgresPgssReader, PostgresPgssRow } from './types.js';
|
||||
|
||||
const FIXTURE_ROOT = join(__dirname, '__fixtures__/postgres');
|
||||
|
||||
|
|
@ -45,7 +45,7 @@ async function tempDir(prefix: string): Promise<string> {
|
|||
return mkdtemp(join(tmpdir(), prefix));
|
||||
}
|
||||
|
||||
function fakePgClient(): KloPostgresQueryClient {
|
||||
function fakePgClient(): KtxPostgresQueryClient {
|
||||
return {
|
||||
async executeQuery() {
|
||||
return { headers: [], rows: [] };
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import {
|
|||
type PgssBaseline,
|
||||
} from './stage-pgss.js';
|
||||
import { historicSqlManifestSchema, historicSqlMetadataSchema, historicSqlUsageSchema } from './types.js';
|
||||
import type { KloPostgresQueryClient, PostgresPgssReader, PostgresPgssRow } from './types.js';
|
||||
import type { KtxPostgresQueryClient, PostgresPgssReader, PostgresPgssRow } from './types.js';
|
||||
|
||||
async function tempDir(prefix: string): Promise<string> {
|
||||
return mkdtemp(join(tmpdir(), prefix));
|
||||
|
|
@ -21,7 +21,7 @@ async function readJson<T>(root: string, relPath: string): Promise<T> {
|
|||
return JSON.parse(await readFile(join(root, relPath), 'utf-8')) as T;
|
||||
}
|
||||
|
||||
function fakePgClient(): KloPostgresQueryClient {
|
||||
function fakePgClient(): KtxPostgresQueryClient {
|
||||
return {
|
||||
async executeQuery() {
|
||||
return { headers: [], rows: [] };
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import {
|
|||
type HistoricSqlMetadata,
|
||||
type HistoricSqlPullConfig,
|
||||
type HistoricSqlUsage,
|
||||
type KloPostgresQueryClient,
|
||||
type KtxPostgresQueryClient,
|
||||
type PostgresPgssAggregateRow,
|
||||
type PostgresPgssReader,
|
||||
type PostgresPgssRow,
|
||||
|
|
@ -43,7 +43,7 @@ export type PgssBaseline = z.infer<typeof pgssBaselineSchema>;
|
|||
export interface StagePgStatStatementsTemplatesInput {
|
||||
stagedDir: string;
|
||||
connectionId: string;
|
||||
queryClient: KloPostgresQueryClient;
|
||||
queryClient: KtxPostgresQueryClient;
|
||||
reader: PostgresPgssReader;
|
||||
sqlAnalysis: SqlAnalysisPort;
|
||||
pullConfig: HistoricSqlPullConfig;
|
||||
|
|
@ -95,7 +95,7 @@ function pgssTemplateId(row: Pick<PostgresPgssRow, 'dbid' | 'queryid'>): string
|
|||
}
|
||||
|
||||
export function pgssBaselinePath(rootDir: string | undefined, connectionId: string): string {
|
||||
return join(rootDir ?? join(process.cwd(), '.klo/cache/historic-sql'), connectionId, 'pgss-baseline.json');
|
||||
return join(rootDir ?? join(process.cwd(), '.ktx/cache/historic-sql'), connectionId, 'pgss-baseline.json');
|
||||
}
|
||||
|
||||
export async function readPgssBaseline(path: string): Promise<PgssBaseline | null> {
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ export interface HistoricSqlQueryHistoryReader {
|
|||
): AsyncIterable<HistoricSqlRawQueryRow>;
|
||||
}
|
||||
|
||||
export interface KloPostgresQueryClient {
|
||||
export interface KtxPostgresQueryClient {
|
||||
executeQuery(sql: string, params?: unknown[]): Promise<{ headers: string[]; rows: unknown[][]; totalRows?: number }>;
|
||||
}
|
||||
|
||||
|
|
@ -61,9 +61,9 @@ export interface PostgresPgssSnapshot {
|
|||
}
|
||||
|
||||
export interface PostgresPgssReader {
|
||||
probe(client: KloPostgresQueryClient): Promise<PostgresPgssProbeResult>;
|
||||
probe(client: KtxPostgresQueryClient): Promise<PostgresPgssProbeResult>;
|
||||
readSnapshot(
|
||||
client: KloPostgresQueryClient,
|
||||
client: KtxPostgresQueryClient,
|
||||
options: { minCalls: number; maxTemplates: number },
|
||||
): Promise<PostgresPgssSnapshot>;
|
||||
}
|
||||
|
|
@ -101,7 +101,7 @@ export interface HistoricSqlSourceAdapterDeps {
|
|||
reader: HistoricSqlQueryHistoryReader;
|
||||
queryClient: unknown;
|
||||
postgresReader?: PostgresPgssReader;
|
||||
postgresQueryClient?: KloPostgresQueryClient;
|
||||
postgresQueryClient?: KtxPostgresQueryClient;
|
||||
postgresBaselineRootDir?: string;
|
||||
now?: () => Date;
|
||||
onPullSucceeded?: (ctx: {
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@ import { mkdtemp } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import type { KloSchemaSnapshot } from '../../../scan/types.js';
|
||||
import type { KtxSchemaSnapshot } from '../../../scan/types.js';
|
||||
import { chunkLiveDatabaseStagedDir } from './chunk.js';
|
||||
import { liveDatabaseTablePath, writeLiveDatabaseSnapshot } from './stage.js';
|
||||
|
||||
function snapshot(): KloSchemaSnapshot {
|
||||
function snapshot(): KtxSchemaSnapshot {
|
||||
return {
|
||||
connectionId: 'conn-1',
|
||||
driver: 'postgres',
|
||||
|
|
@ -60,7 +60,7 @@ function snapshot(): KloSchemaSnapshot {
|
|||
|
||||
describe('chunkLiveDatabaseStagedDir', () => {
|
||||
it('emits one work unit per table on the first run', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'klo-live-db-chunk-'));
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-chunk-'));
|
||||
await writeLiveDatabaseSnapshot(dir, snapshot());
|
||||
|
||||
const result = await chunkLiveDatabaseStagedDir(dir);
|
||||
|
|
@ -75,7 +75,7 @@ describe('chunkLiveDatabaseStagedDir', () => {
|
|||
});
|
||||
|
||||
it('keeps only changed tables during incremental syncs and records table evictions', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'klo-live-db-diff-'));
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-diff-'));
|
||||
await writeLiveDatabaseSnapshot(dir, snapshot());
|
||||
const ordersPath = liveDatabaseTablePath({ catalog: null, db: 'public', name: 'orders' });
|
||||
const customersPath = liveDatabaseTablePath({ catalog: null, db: 'public', name: 'customers' });
|
||||
|
|
@ -92,7 +92,7 @@ describe('chunkLiveDatabaseStagedDir', () => {
|
|||
});
|
||||
|
||||
it('fans out all table work units when the foreign-key index changes', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'klo-live-db-fk-'));
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-fk-'));
|
||||
await writeLiveDatabaseSnapshot(dir, snapshot());
|
||||
|
||||
const result = await chunkLiveDatabaseStagedDir(dir, {
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import type { ChunkResult, DiffSet, WorkUnit } from '../../types.js';
|
||||
import type { KloSchemaTable } from '../../../scan/types.js';
|
||||
import type { KtxSchemaTable } from '../../../scan/types.js';
|
||||
import { LIVE_DATABASE_FOREIGN_KEYS_FILE, LIVE_DATABASE_META_FILE, readLiveDatabaseTableFiles } from './stage.js';
|
||||
|
||||
function unitKey(table: KloSchemaTable): string {
|
||||
function unitKey(table: KtxSchemaTable): string {
|
||||
const parts = [table.catalog, table.db, table.name]
|
||||
.filter((part): part is string => typeof part === 'string' && part.length > 0)
|
||||
.map((part) =>
|
||||
|
|
@ -15,7 +15,7 @@ function unitKey(table: KloSchemaTable): string {
|
|||
return `live-database-${parts.join('-') || 'table'}`;
|
||||
}
|
||||
|
||||
function displayName(table: KloSchemaTable): string {
|
||||
function displayName(table: KtxSchemaTable): string {
|
||||
return [table.catalog, table.db, table.name].filter(Boolean).join('.');
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,25 +2,25 @@ import { spawn } from 'node:child_process';
|
|||
import { request as httpRequest } from 'node:http';
|
||||
import { request as httpsRequest } from 'node:https';
|
||||
import { URL } from 'node:url';
|
||||
import type { KloProjectConnectionConfig } from '../../../project/config.js';
|
||||
import type { KloSchemaColumn, KloSchemaForeignKey, KloSchemaSnapshot, KloSchemaTable } from '../../../scan/types.js';
|
||||
import { inferKloDimensionType, normalizeKloNativeType } from '../../../scan/type-normalization.js';
|
||||
import type { KtxProjectConnectionConfig } from '../../../project/config.js';
|
||||
import type { KtxSchemaColumn, KtxSchemaForeignKey, KtxSchemaSnapshot, KtxSchemaTable } from '../../../scan/types.js';
|
||||
import { inferKtxDimensionType, normalizeKtxNativeType } from '../../../scan/type-normalization.js';
|
||||
import type { LiveDatabaseIntrospectionPort } from './types.js';
|
||||
|
||||
export type KloDaemonDatabaseIntrospectionCommand = 'database-introspect';
|
||||
export type KtxDaemonDatabaseIntrospectionCommand = 'database-introspect';
|
||||
|
||||
export type KloDaemonDatabaseJsonRunner = (
|
||||
subcommand: KloDaemonDatabaseIntrospectionCommand,
|
||||
export type KtxDaemonDatabaseJsonRunner = (
|
||||
subcommand: KtxDaemonDatabaseIntrospectionCommand,
|
||||
payload: Record<string, unknown>,
|
||||
) => Promise<Record<string, unknown>>;
|
||||
|
||||
export type KloDaemonDatabaseHttpJsonRunner = (
|
||||
export type KtxDaemonDatabaseHttpJsonRunner = (
|
||||
path: string,
|
||||
payload: Record<string, unknown>,
|
||||
) => Promise<Record<string, unknown>>;
|
||||
|
||||
export interface DaemonLiveDatabaseIntrospectionOptions {
|
||||
connections: Record<string, KloProjectConnectionConfig>;
|
||||
connections: Record<string, KtxProjectConnectionConfig>;
|
||||
schemas?: string[];
|
||||
statementTimeoutMs?: number;
|
||||
connectionTimeoutSeconds?: number;
|
||||
|
|
@ -29,8 +29,8 @@ export interface DaemonLiveDatabaseIntrospectionOptions {
|
|||
cwd?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
baseUrl?: string;
|
||||
runJson?: KloDaemonDatabaseJsonRunner;
|
||||
requestJson?: KloDaemonDatabaseHttpJsonRunner;
|
||||
runJson?: KtxDaemonDatabaseJsonRunner;
|
||||
requestJson?: KtxDaemonDatabaseHttpJsonRunner;
|
||||
now?: () => Date;
|
||||
}
|
||||
|
||||
|
|
@ -39,7 +39,7 @@ const DEFAULT_SCHEMAS = ['public'];
|
|||
function parseJsonObject(raw: string, subcommand: string): Record<string, unknown> {
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
throw new Error(`klo-daemon ${subcommand} returned non-object JSON`);
|
||||
throw new Error(`ktx-daemon ${subcommand} returned non-object JSON`);
|
||||
}
|
||||
return parsed as Record<string, unknown>;
|
||||
}
|
||||
|
|
@ -47,7 +47,7 @@ function parseJsonObject(raw: string, subcommand: string): Record<string, unknow
|
|||
function runProcessJson(
|
||||
options: Required<Pick<DaemonLiveDatabaseIntrospectionOptions, 'command' | 'args'>> &
|
||||
Pick<DaemonLiveDatabaseIntrospectionOptions, 'cwd' | 'env'>,
|
||||
): KloDaemonDatabaseJsonRunner {
|
||||
): KtxDaemonDatabaseJsonRunner {
|
||||
return async (subcommand, payload) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const child = spawn(options.command, [...options.args, subcommand], {
|
||||
|
|
@ -65,7 +65,7 @@ function runProcessJson(
|
|||
const stdoutText = Buffer.concat(stdout).toString('utf8').trim();
|
||||
const stderrText = Buffer.concat(stderr).toString('utf8').trim();
|
||||
if (code !== 0) {
|
||||
reject(new Error(`klo-daemon ${subcommand} failed: ${stderrText || `exit code ${code}`}`));
|
||||
reject(new Error(`ktx-daemon ${subcommand} failed: ${stderrText || `exit code ${code}`}`));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
|
|
@ -82,7 +82,7 @@ function normalizedBaseUrl(baseUrl: string): string {
|
|||
return baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`;
|
||||
}
|
||||
|
||||
function postJson(baseUrl: string): KloDaemonDatabaseHttpJsonRunner {
|
||||
function postJson(baseUrl: string): KtxDaemonDatabaseHttpJsonRunner {
|
||||
return async (path, payload) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const target = new URL(path.replace(/^\//, ''), normalizedBaseUrl(baseUrl));
|
||||
|
|
@ -105,7 +105,7 @@ function postJson(baseUrl: string): KloDaemonDatabaseHttpJsonRunner {
|
|||
const text = Buffer.concat(chunks).toString('utf8');
|
||||
const statusCode = response.statusCode ?? 0;
|
||||
if (statusCode < 200 || statusCode >= 300) {
|
||||
reject(new Error(`klo-daemon HTTP ${path} failed with ${statusCode}: ${text}`));
|
||||
reject(new Error(`ktx-daemon HTTP ${path} failed with ${statusCode}: ${text}`));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
|
|
@ -135,7 +135,7 @@ function recordArray(value: unknown): Array<Record<string, unknown>> {
|
|||
|
||||
function requiredString(value: unknown, field: string): string {
|
||||
if (typeof value !== 'string' || value.length === 0) {
|
||||
throw new Error(`klo-daemon database introspection response is missing string field ${field}`);
|
||||
throw new Error(`ktx-daemon database introspection response is missing string field ${field}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
|
@ -154,9 +154,9 @@ function normalizeDriver(driver: unknown): string {
|
|||
}
|
||||
|
||||
function requirePostgresConnection(
|
||||
connections: Record<string, KloProjectConnectionConfig>,
|
||||
connections: Record<string, KtxProjectConnectionConfig>,
|
||||
connectionId: string,
|
||||
): KloProjectConnectionConfig & { url: string } {
|
||||
): KtxProjectConnectionConfig & { url: string } {
|
||||
const connection = connections[connectionId];
|
||||
const driver = normalizeDriver(connection?.driver);
|
||||
if (driver !== 'postgres') {
|
||||
|
|
@ -168,23 +168,23 @@ function requirePostgresConnection(
|
|||
if (typeof connection.url !== 'string' || connection.url.trim().length === 0) {
|
||||
throw new Error(`Local live-database ingest requires connections.${connectionId}.url.`);
|
||||
}
|
||||
return connection as KloProjectConnectionConfig & { url: string };
|
||||
return connection as KtxProjectConnectionConfig & { url: string };
|
||||
}
|
||||
|
||||
function mapColumn(raw: Record<string, unknown>): KloSchemaColumn {
|
||||
function mapColumn(raw: Record<string, unknown>): KtxSchemaColumn {
|
||||
const nativeType = requiredString(raw.type, 'tables[].columns[].type');
|
||||
return {
|
||||
name: requiredString(raw.name, 'tables[].columns[].name'),
|
||||
nativeType,
|
||||
normalizedType: normalizeKloNativeType(nativeType),
|
||||
dimensionType: inferKloDimensionType(nativeType),
|
||||
normalizedType: normalizeKtxNativeType(nativeType),
|
||||
dimensionType: inferKtxDimensionType(nativeType),
|
||||
nullable: raw.nullable !== false ? true : false,
|
||||
primaryKey: raw.primary_key === true,
|
||||
comment: nullableString(raw.comment),
|
||||
};
|
||||
}
|
||||
|
||||
function mapForeignKey(raw: Record<string, unknown>): KloSchemaForeignKey {
|
||||
function mapForeignKey(raw: Record<string, unknown>): KtxSchemaForeignKey {
|
||||
return {
|
||||
fromColumn: requiredString(raw.from_column, 'tables[].foreign_keys[].from_column'),
|
||||
toCatalog: null,
|
||||
|
|
@ -195,7 +195,7 @@ function mapForeignKey(raw: Record<string, unknown>): KloSchemaForeignKey {
|
|||
};
|
||||
}
|
||||
|
||||
function mapTable(raw: Record<string, unknown>): KloSchemaTable {
|
||||
function mapTable(raw: Record<string, unknown>): KtxSchemaTable {
|
||||
return {
|
||||
catalog: nullableString(raw.catalog),
|
||||
db: nullableString(raw.db),
|
||||
|
|
@ -211,7 +211,7 @@ function mapTable(raw: Record<string, unknown>): KloSchemaTable {
|
|||
function mapDaemonSnapshot(
|
||||
raw: Record<string, unknown>,
|
||||
input: { connectionId: string; extractedAt: string; schemas: string[] },
|
||||
): KloSchemaSnapshot {
|
||||
): KtxSchemaSnapshot {
|
||||
return {
|
||||
connectionId: requiredString(raw.connection_id, 'connection_id') || input.connectionId,
|
||||
driver: 'postgres',
|
||||
|
|
@ -227,13 +227,13 @@ export function createDaemonLiveDatabaseIntrospection(
|
|||
): LiveDatabaseIntrospectionPort {
|
||||
const schemas = options.schemas ?? DEFAULT_SCHEMAS;
|
||||
const command = options.command ?? 'python';
|
||||
const args = options.args ?? ['-m', 'klo_daemon'];
|
||||
const args = options.args ?? ['-m', 'ktx_daemon'];
|
||||
const runJson = options.runJson ?? runProcessJson({ command, args, cwd: options.cwd, env: options.env });
|
||||
const requestJson = options.requestJson ?? (options.baseUrl ? postJson(options.baseUrl) : undefined);
|
||||
const now = options.now ?? (() => new Date());
|
||||
|
||||
return {
|
||||
async extractSchema(connectionId: string): Promise<KloSchemaSnapshot> {
|
||||
async extractSchema(connectionId: string): Promise<KtxSchemaSnapshot> {
|
||||
const connection = requirePostgresConnection(options.connections, connectionId);
|
||||
const payload = {
|
||||
connection_id: connectionId,
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type { KloSchemaSnapshot } from '../../../scan/types.js';
|
||||
import { buildLiveDatabaseTableNaturalKey, kloSchemaSnapshotToExtractedSchema } from './extracted-schema.js';
|
||||
import type { KtxSchemaSnapshot } from '../../../scan/types.js';
|
||||
import { buildLiveDatabaseTableNaturalKey, ktxSchemaSnapshotToExtractedSchema } from './extracted-schema.js';
|
||||
|
||||
function snapshot(): KloSchemaSnapshot {
|
||||
function snapshot(): KtxSchemaSnapshot {
|
||||
return {
|
||||
connectionId: 'conn-1',
|
||||
driver: 'postgres',
|
||||
|
|
@ -72,9 +72,9 @@ function snapshot(): KloSchemaSnapshot {
|
|||
};
|
||||
}
|
||||
|
||||
describe('kloSchemaSnapshotToExtractedSchema', () => {
|
||||
describe('ktxSchemaSnapshotToExtractedSchema', () => {
|
||||
it('preserves structural table, column, comment, and key metadata', () => {
|
||||
const extracted = kloSchemaSnapshotToExtractedSchema(snapshot());
|
||||
const extracted = ktxSchemaSnapshotToExtractedSchema(snapshot());
|
||||
|
||||
expect(extracted.tables).toEqual([
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KloSchemaSnapshot, KloSchemaTable } from '../../../scan/types.js';
|
||||
import type { KtxSchemaSnapshot, KtxSchemaTable } from '../../../scan/types.js';
|
||||
|
||||
export interface LiveDatabaseExtractedForeignKey {
|
||||
fromTable: string;
|
||||
|
|
@ -30,11 +30,11 @@ export interface LiveDatabaseExtractedSchema {
|
|||
tables: LiveDatabaseExtractedTable[];
|
||||
}
|
||||
|
||||
export function buildLiveDatabaseTableNaturalKey(table: Pick<KloSchemaTable, 'catalog' | 'db' | 'name'>): string {
|
||||
export function buildLiveDatabaseTableNaturalKey(table: Pick<KtxSchemaTable, 'catalog' | 'db' | 'name'>): string {
|
||||
return `${table.catalog ?? ''}|${table.db ?? ''}|${table.name}`;
|
||||
}
|
||||
|
||||
export function kloSchemaSnapshotToExtractedSchema(snapshot: KloSchemaSnapshot): LiveDatabaseExtractedSchema {
|
||||
export function ktxSchemaSnapshotToExtractedSchema(snapshot: KtxSchemaSnapshot): LiveDatabaseExtractedSchema {
|
||||
return {
|
||||
connectionId: snapshot.connectionId,
|
||||
tables: snapshot.tables.map((table) => ({
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ describe('LiveDatabaseSourceAdapter', () => {
|
|||
introspection: { extractSchema },
|
||||
now: () => new Date('2026-04-27T00:00:00.000Z'),
|
||||
});
|
||||
const dir = await mkdtemp(join(tmpdir(), 'klo-live-db-adapter-'));
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-adapter-'));
|
||||
|
||||
await adapter.fetch(undefined, dir, { connectionId: 'conn-1', sourceKey: 'live-database' });
|
||||
|
||||
|
|
|
|||
|
|
@ -10,9 +10,9 @@ import {
|
|||
readLiveDatabaseTableFiles,
|
||||
writeLiveDatabaseSnapshot,
|
||||
} from './stage.js';
|
||||
import type { KloSchemaSnapshot } from '../../../scan/types.js';
|
||||
import type { KtxSchemaSnapshot } from '../../../scan/types.js';
|
||||
|
||||
function snapshot(): KloSchemaSnapshot {
|
||||
function snapshot(): KtxSchemaSnapshot {
|
||||
return {
|
||||
connectionId: 'conn-1',
|
||||
driver: 'postgres',
|
||||
|
|
@ -93,7 +93,7 @@ function snapshot(): KloSchemaSnapshot {
|
|||
|
||||
describe('live-database staged snapshot files', () => {
|
||||
it('writes deterministic metadata, table, and foreign-key files', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'klo-live-db-stage-'));
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-stage-'));
|
||||
await writeLiveDatabaseSnapshot(dir, snapshot());
|
||||
|
||||
await expect(readFile(join(dir, LIVE_DATABASE_META_FILE), 'utf8')).resolves.toContain('"connectionId": "conn-1"');
|
||||
|
|
@ -122,7 +122,7 @@ describe('live-database staged snapshot files', () => {
|
|||
});
|
||||
|
||||
it('redacts sensitive snapshot metadata before writing connection metadata', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'klo-live-db-redacted-stage-'));
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-redacted-stage-'));
|
||||
await writeLiveDatabaseSnapshot(dir, {
|
||||
...snapshot(),
|
||||
metadata: {
|
||||
|
|
@ -146,7 +146,7 @@ describe('live-database staged snapshot files', () => {
|
|||
});
|
||||
|
||||
it('returns false for a directory that is missing live database metadata', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'klo-live-db-empty-'));
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-empty-'));
|
||||
expect(await detectLiveDatabaseStagedDir(dir)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ import { Buffer } from 'node:buffer';
|
|||
import type { Dirent } from 'node:fs';
|
||||
import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { join, relative } from 'node:path';
|
||||
import { redactKloSensitiveMetadata } from '../../../core/redaction.js';
|
||||
import type { KloSchemaSnapshot, KloSchemaTable, KloTableRef } from '../../../scan/types.js';
|
||||
import { redactKtxSensitiveMetadata } from '../../../core/redaction.js';
|
||||
import type { KtxSchemaSnapshot, KtxSchemaTable, KtxTableRef } from '../../../scan/types.js';
|
||||
|
||||
export const LIVE_DATABASE_META_FILE = 'connection.json';
|
||||
export const LIVE_DATABASE_FOREIGN_KEYS_FILE = 'foreign-keys.json';
|
||||
|
|
@ -11,7 +11,7 @@ const LIVE_DATABASE_TABLES_DIR = 'tables';
|
|||
|
||||
interface LiveDatabaseTableFile {
|
||||
path: string;
|
||||
table: KloSchemaTable;
|
||||
table: KtxSchemaTable;
|
||||
}
|
||||
|
||||
interface ForeignKeyIndexEntry {
|
||||
|
|
@ -29,11 +29,11 @@ function encodePathPart(value: string | null | undefined): string {
|
|||
return Buffer.from(value ?? '_', 'utf8').toString('base64url');
|
||||
}
|
||||
|
||||
function tableSortKey(table: KloTableRef): string {
|
||||
function tableSortKey(table: KtxTableRef): string {
|
||||
return `${table.catalog ?? ''}\u0000${table.db ?? ''}\u0000${table.name}`;
|
||||
}
|
||||
|
||||
export function liveDatabaseTablePath(table: KloTableRef): string {
|
||||
export function liveDatabaseTablePath(table: KtxTableRef): string {
|
||||
return `${LIVE_DATABASE_TABLES_DIR}/${encodePathPart(table.catalog)}.${encodePathPart(table.db)}.${encodePathPart(
|
||||
table.name,
|
||||
)}.json`;
|
||||
|
|
@ -62,7 +62,7 @@ function stableJson(value: unknown): string {
|
|||
return `${JSON.stringify(value, null, 2)}\n`;
|
||||
}
|
||||
|
||||
function foreignKeyIndex(snapshot: KloSchemaSnapshot): ForeignKeyIndexEntry[] {
|
||||
function foreignKeyIndex(snapshot: KtxSchemaSnapshot): ForeignKeyIndexEntry[] {
|
||||
const entries: ForeignKeyIndexEntry[] = [];
|
||||
for (const table of snapshot.tables) {
|
||||
for (const fk of table.foreignKeys) {
|
||||
|
|
@ -88,7 +88,7 @@ function foreignKeyIndex(snapshot: KloSchemaSnapshot): ForeignKeyIndexEntry[] {
|
|||
return entries;
|
||||
}
|
||||
|
||||
export async function writeLiveDatabaseSnapshot(stagedDir: string, snapshot: KloSchemaSnapshot): Promise<void> {
|
||||
export async function writeLiveDatabaseSnapshot(stagedDir: string, snapshot: KtxSchemaSnapshot): Promise<void> {
|
||||
await mkdir(join(stagedDir, LIVE_DATABASE_TABLES_DIR), { recursive: true });
|
||||
const sortedTables = [...snapshot.tables].sort((a, b) => tableSortKey(a).localeCompare(tableSortKey(b)));
|
||||
const metadata = {
|
||||
|
|
@ -96,7 +96,7 @@ export async function writeLiveDatabaseSnapshot(stagedDir: string, snapshot: Klo
|
|||
driver: snapshot.driver,
|
||||
extractedAt: snapshot.extractedAt,
|
||||
scope: snapshot.scope,
|
||||
metadata: redactKloSensitiveMetadata(snapshot.metadata),
|
||||
metadata: redactKtxSensitiveMetadata(snapshot.metadata),
|
||||
tableCount: sortedTables.length,
|
||||
};
|
||||
await writeFile(join(stagedDir, LIVE_DATABASE_META_FILE), stableJson(metadata));
|
||||
|
|
@ -115,7 +115,7 @@ export async function readLiveDatabaseTableFiles(stagedDir: string): Promise<Liv
|
|||
for (const file of files.filter((path) => path.endsWith('.json'))) {
|
||||
const path = `${LIVE_DATABASE_TABLES_DIR}/${file}`;
|
||||
const raw = await readFile(join(stagedDir, path), 'utf8');
|
||||
const parsed = JSON.parse(raw) as KloSchemaTable;
|
||||
const parsed = JSON.parse(raw) as KtxSchemaTable;
|
||||
if (parsed && typeof parsed.name === 'string' && Array.isArray(parsed.columns)) {
|
||||
out.push({ path, table: parsed });
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import type { KloSchemaSnapshot } from '../../../scan/types.js';
|
||||
import type { KtxSchemaSnapshot } from '../../../scan/types.js';
|
||||
|
||||
export interface LiveDatabaseIntrospectionPort {
|
||||
extractSchema(connectionId: string): Promise<KloSchemaSnapshot>;
|
||||
extractSchema(connectionId: string): Promise<KtxSchemaSnapshot>;
|
||||
}
|
||||
|
||||
export interface LiveDatabaseSourceAdapterDeps {
|
||||
|
|
|
|||
|
|
@ -39,6 +39,6 @@ describe('createDaemonLookerTableIdentifierParser', () => {
|
|||
requestJson: async () => ({ results: null }),
|
||||
});
|
||||
|
||||
await expect(parser.parse([])).rejects.toThrow('klo-daemon table identifier parser returned invalid results');
|
||||
await expect(parser.parse([])).rejects.toThrow('ktx-daemon table identifier parser returned invalid results');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -7,14 +7,14 @@ import type {
|
|||
LookerTableIdentifierParser,
|
||||
} from './mapping.js';
|
||||
|
||||
export type KloDaemonTableIdentifierHttpJsonRunner = (
|
||||
export type KtxDaemonTableIdentifierHttpJsonRunner = (
|
||||
path: string,
|
||||
payload: Record<string, unknown>,
|
||||
) => Promise<Record<string, unknown>>;
|
||||
|
||||
export interface DaemonLookerTableIdentifierParserOptions {
|
||||
baseUrl: string;
|
||||
requestJson?: KloDaemonTableIdentifierHttpJsonRunner;
|
||||
requestJson?: KtxDaemonTableIdentifierHttpJsonRunner;
|
||||
}
|
||||
|
||||
export function createDaemonLookerTableIdentifierParser(
|
||||
|
|
@ -25,7 +25,7 @@ export function createDaemonLookerTableIdentifierParser(
|
|||
async parse(items: LookerTableIdentifierParseItem[]): Promise<Record<string, LookerParsedIdentifier>> {
|
||||
const raw = await requestJson('/sql/parse-table-identifier', { items });
|
||||
if (!raw.results || typeof raw.results !== 'object' || Array.isArray(raw.results)) {
|
||||
throw new Error('klo-daemon table identifier parser returned invalid results');
|
||||
throw new Error('ktx-daemon table identifier parser returned invalid results');
|
||||
}
|
||||
return raw.results as Record<string, LookerParsedIdentifier>;
|
||||
},
|
||||
|
|
@ -36,7 +36,7 @@ function normalizedBaseUrl(baseUrl: string): string {
|
|||
return baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`;
|
||||
}
|
||||
|
||||
function postJson(baseUrl: string): KloDaemonTableIdentifierHttpJsonRunner {
|
||||
function postJson(baseUrl: string): KtxDaemonTableIdentifierHttpJsonRunner {
|
||||
return async (path, payload) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const target = new URL(path.replace(/^\//, ''), normalizedBaseUrl(baseUrl));
|
||||
|
|
@ -59,13 +59,13 @@ function postJson(baseUrl: string): KloDaemonTableIdentifierHttpJsonRunner {
|
|||
const text = Buffer.concat(chunks).toString('utf8');
|
||||
const statusCode = response.statusCode ?? 0;
|
||||
if (statusCode < 200 || statusCode >= 300) {
|
||||
reject(new Error(`klo-daemon HTTP ${path} failed with ${statusCode}: ${text}`));
|
||||
reject(new Error(`ktx-daemon HTTP ${path} failed with ${statusCode}: ${text}`));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(text) as unknown;
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
reject(new Error(`klo-daemon HTTP ${path} returned non-object JSON`));
|
||||
reject(new Error(`ktx-daemon HTTP ${path} returned non-object JSON`));
|
||||
return;
|
||||
}
|
||||
resolve(parsed as Record<string, unknown>);
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ function sdk(): LookerSdkPort {
|
|||
}
|
||||
|
||||
describe('DefaultLookerConnectionClientFactory', () => {
|
||||
it('resolves credentials by Looker connection id and creates a KLO Looker client', async () => {
|
||||
it('resolves credentials by Looker connection id and creates a KTX Looker client', async () => {
|
||||
const fakeSdk = sdk();
|
||||
const resolver: LookerCredentialResolver = {
|
||||
resolve: vi.fn().mockResolvedValue({
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KloLocalProject, KloProjectConnectionConfig } from '../../../project/index.js';
|
||||
import type { KtxLocalProject, KtxProjectConnectionConfig } from '../../../project/index.js';
|
||||
import {
|
||||
DefaultLookerClientFactory,
|
||||
DefaultLookerConnectionClientFactory,
|
||||
|
|
@ -19,7 +19,7 @@ function resolveEnvReference(ref: string, env: NodeJS.ProcessEnv): string | null
|
|||
|
||||
export function lookerCredentialsFromLocalConnection(
|
||||
connectionId: string,
|
||||
connection: KloProjectConnectionConfig | undefined,
|
||||
connection: KtxProjectConnectionConfig | undefined,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
) {
|
||||
if (!connection || String(connection.driver).toLowerCase() !== 'looker') {
|
||||
|
|
@ -46,7 +46,7 @@ export function lookerCredentialsFromLocalConnection(
|
|||
}
|
||||
|
||||
export function createLocalLookerCredentialResolver(
|
||||
project: KloLocalProject,
|
||||
project: KtxLocalProject,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): LookerCredentialResolver {
|
||||
return {
|
||||
|
|
@ -57,7 +57,7 @@ export function createLocalLookerCredentialResolver(
|
|||
}
|
||||
|
||||
export function createLocalLookerSourceAdapter(
|
||||
project: KloLocalProject,
|
||||
project: KtxLocalProject,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): LookerSourceAdapter {
|
||||
const connectionFactory = new DefaultLookerConnectionClientFactory(createLocalLookerCredentialResolver(project, env));
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import { LocalLookerRuntimeStore } from './local-runtime-store.js';
|
|||
|
||||
describe('LocalLookerRuntimeStore', () => {
|
||||
async function store() {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'klo-looker-store-'));
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-looker-store-'));
|
||||
return new LocalLookerRuntimeStore({
|
||||
dbPath: join(dir, 'db.sqlite'),
|
||||
now: () => new Date('2026-05-05T12:00:00.000Z'),
|
||||
|
|
@ -23,7 +23,7 @@ describe('LocalLookerRuntimeStore', () => {
|
|||
await local.upsertConnectionMapping({
|
||||
lookerConnectionId: 'prod-looker',
|
||||
lookerConnectionName: 'bq_reporting',
|
||||
kloConnectionId: 'prod-warehouse',
|
||||
ktxConnectionId: 'prod-warehouse',
|
||||
source: 'cli',
|
||||
});
|
||||
|
||||
|
|
@ -34,7 +34,7 @@ describe('LocalLookerRuntimeStore', () => {
|
|||
await expect(local.readMappings('prod-looker')).resolves.toEqual([
|
||||
{
|
||||
lookerConnectionName: 'bq_reporting',
|
||||
kloConnectionId: 'prod-warehouse',
|
||||
ktxConnectionId: 'prod-warehouse',
|
||||
lookerHost: null,
|
||||
lookerDatabase: null,
|
||||
lookerDialect: null,
|
||||
|
|
@ -47,7 +47,7 @@ describe('LocalLookerRuntimeStore', () => {
|
|||
await local.upsertConnectionMapping({
|
||||
lookerConnectionId: 'prod-looker',
|
||||
lookerConnectionName: 'bq_reporting',
|
||||
kloConnectionId: 'prod-warehouse',
|
||||
ktxConnectionId: 'prod-warehouse',
|
||||
source: 'cli',
|
||||
});
|
||||
|
||||
|
|
@ -67,7 +67,7 @@ describe('LocalLookerRuntimeStore', () => {
|
|||
await expect(local.listConnectionMappings('prod-looker')).resolves.toEqual([
|
||||
{
|
||||
lookerConnectionName: 'bq_reporting',
|
||||
kloConnectionId: 'prod-warehouse',
|
||||
ktxConnectionId: 'prod-warehouse',
|
||||
lookerHost: 'bigquery.googleapis.com',
|
||||
lookerDatabase: 'analytics',
|
||||
lookerDialect: 'bigquery_standard_sql',
|
||||
|
|
@ -85,30 +85,30 @@ describe('LocalLookerRuntimeStore', () => {
|
|||
await local.upsertConnectionMapping({
|
||||
lookerConnectionId: 'prod-looker',
|
||||
lookerConnectionName: 'manual',
|
||||
kloConnectionId: 'cli-warehouse',
|
||||
ktxConnectionId: 'cli-warehouse',
|
||||
source: 'cli',
|
||||
});
|
||||
|
||||
await local.applyYamlBootstrap({
|
||||
lookerConnectionId: 'prod-looker',
|
||||
mappings: [
|
||||
{ lookerConnectionName: 'analytics', kloConnectionId: 'yaml-warehouse' },
|
||||
{ lookerConnectionName: 'manual', kloConnectionId: 'yaml-warehouse' },
|
||||
{ lookerConnectionName: 'analytics', ktxConnectionId: 'yaml-warehouse' },
|
||||
{ lookerConnectionName: 'manual', ktxConnectionId: 'yaml-warehouse' },
|
||||
],
|
||||
});
|
||||
|
||||
await expect(local.listConnectionMappings('prod-looker')).resolves.toMatchObject([
|
||||
{
|
||||
lookerConnectionName: 'analytics',
|
||||
kloConnectionId: 'yaml-warehouse',
|
||||
ktxConnectionId: 'yaml-warehouse',
|
||||
lookerHost: 'looker-db.test',
|
||||
lookerDatabase: 'warehouse',
|
||||
lookerDialect: 'postgres',
|
||||
source: 'klo.yaml',
|
||||
source: 'ktx.yaml',
|
||||
},
|
||||
{
|
||||
lookerConnectionName: 'manual',
|
||||
kloConnectionId: 'cli-warehouse',
|
||||
ktxConnectionId: 'cli-warehouse',
|
||||
source: 'cli',
|
||||
},
|
||||
]);
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import type { LookerWarehouseConnectionInfo } from './client.js';
|
|||
import type { LookerConnectionMapping } from './mapping.js';
|
||||
import type { LookerRuntimeCursors } from './types.js';
|
||||
|
||||
export type LocalLookerMappingSource = 'klo.yaml' | 'cli' | 'refresh';
|
||||
export type LocalLookerMappingSource = 'ktx.yaml' | 'cli' | 'refresh';
|
||||
|
||||
interface LocalLookerRuntimeStoreOptions {
|
||||
dbPath: string;
|
||||
|
|
@ -19,7 +19,7 @@ export interface LocalLookerConnectionMappingListRow extends LookerConnectionMap
|
|||
export interface UpsertLocalLookerConnectionMappingInput {
|
||||
lookerConnectionId: string;
|
||||
lookerConnectionName: string;
|
||||
kloConnectionId: string | null;
|
||||
ktxConnectionId: string | null;
|
||||
source: LocalLookerMappingSource;
|
||||
}
|
||||
|
||||
|
|
@ -27,7 +27,7 @@ interface ApplyLocalLookerYamlBootstrapInput {
|
|||
lookerConnectionId: string;
|
||||
mappings: Array<{
|
||||
lookerConnectionName: string;
|
||||
kloConnectionId: string | null;
|
||||
ktxConnectionId: string | null;
|
||||
}>;
|
||||
}
|
||||
|
||||
|
|
@ -67,7 +67,7 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
CREATE TABLE IF NOT EXISTS local_looker_connection_mappings (
|
||||
looker_connection_id TEXT NOT NULL,
|
||||
looker_connection_name TEXT NOT NULL,
|
||||
klo_connection_id TEXT,
|
||||
ktx_connection_id TEXT,
|
||||
looker_host TEXT,
|
||||
looker_database TEXT,
|
||||
looker_dialect TEXT,
|
||||
|
|
@ -82,7 +82,7 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
const timestamp = this.now().toISOString();
|
||||
const apply = this.db.transaction(() => {
|
||||
const existing = this.db.prepare(`
|
||||
SELECT klo_connection_id, source
|
||||
SELECT ktx_connection_id, source
|
||||
FROM local_looker_connection_mappings
|
||||
WHERE looker_connection_id = ? AND looker_connection_name = ?
|
||||
`);
|
||||
|
|
@ -90,36 +90,36 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
INSERT INTO local_looker_connection_mappings (
|
||||
looker_connection_id,
|
||||
looker_connection_name,
|
||||
klo_connection_id,
|
||||
ktx_connection_id,
|
||||
looker_host,
|
||||
looker_database,
|
||||
looker_dialect,
|
||||
source,
|
||||
updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, NULL, NULL, NULL, 'klo.yaml', ?)
|
||||
VALUES (?, ?, ?, NULL, NULL, NULL, 'ktx.yaml', ?)
|
||||
`);
|
||||
const updateRefreshRow = this.db.prepare(`
|
||||
UPDATE local_looker_connection_mappings
|
||||
SET klo_connection_id = ?,
|
||||
source = 'klo.yaml',
|
||||
SET ktx_connection_id = ?,
|
||||
source = 'ktx.yaml',
|
||||
updated_at = ?
|
||||
WHERE looker_connection_id = ?
|
||||
AND looker_connection_name = ?
|
||||
AND source = 'refresh'
|
||||
AND klo_connection_id IS NULL
|
||||
AND ktx_connection_id IS NULL
|
||||
`);
|
||||
|
||||
for (const mapping of input.mappings) {
|
||||
const row = existing.get(input.lookerConnectionId, mapping.lookerConnectionName) as
|
||||
| { klo_connection_id: string | null; source: LocalLookerMappingSource }
|
||||
| { ktx_connection_id: string | null; source: LocalLookerMappingSource }
|
||||
| undefined;
|
||||
if (!row) {
|
||||
insert.run(input.lookerConnectionId, mapping.lookerConnectionName, mapping.kloConnectionId, timestamp);
|
||||
insert.run(input.lookerConnectionId, mapping.lookerConnectionName, mapping.ktxConnectionId, timestamp);
|
||||
continue;
|
||||
}
|
||||
if (row.source === 'refresh' && row.klo_connection_id === null) {
|
||||
updateRefreshRow.run(mapping.kloConnectionId, timestamp, input.lookerConnectionId, mapping.lookerConnectionName);
|
||||
if (row.source === 'refresh' && row.ktx_connection_id === null) {
|
||||
updateRefreshRow.run(mapping.ktxConnectionId, timestamp, input.lookerConnectionId, mapping.lookerConnectionName);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
@ -174,7 +174,7 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
`
|
||||
SELECT
|
||||
looker_connection_name,
|
||||
klo_connection_id,
|
||||
ktx_connection_id,
|
||||
looker_host,
|
||||
looker_database,
|
||||
looker_dialect,
|
||||
|
|
@ -186,7 +186,7 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
)
|
||||
.all(lookerConnectionId) as Array<{
|
||||
looker_connection_name: string;
|
||||
klo_connection_id: string | null;
|
||||
ktx_connection_id: string | null;
|
||||
looker_host: string | null;
|
||||
looker_database: string | null;
|
||||
looker_dialect: string | null;
|
||||
|
|
@ -195,7 +195,7 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
|
||||
return rows.map((row) => ({
|
||||
lookerConnectionName: row.looker_connection_name,
|
||||
kloConnectionId: row.klo_connection_id,
|
||||
ktxConnectionId: row.ktx_connection_id,
|
||||
lookerHost: row.looker_host,
|
||||
lookerDatabase: row.looker_database,
|
||||
lookerDialect: row.looker_dialect,
|
||||
|
|
@ -210,7 +210,7 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
INSERT INTO local_looker_connection_mappings (
|
||||
looker_connection_id,
|
||||
looker_connection_name,
|
||||
klo_connection_id,
|
||||
ktx_connection_id,
|
||||
looker_host,
|
||||
looker_database,
|
||||
looker_dialect,
|
||||
|
|
@ -219,12 +219,12 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
)
|
||||
VALUES (?, ?, ?, NULL, NULL, NULL, ?, ?)
|
||||
ON CONFLICT(looker_connection_id, looker_connection_name) DO UPDATE SET
|
||||
klo_connection_id = excluded.klo_connection_id,
|
||||
ktx_connection_id = excluded.ktx_connection_id,
|
||||
source = excluded.source,
|
||||
updated_at = excluded.updated_at
|
||||
`,
|
||||
)
|
||||
.run(input.lookerConnectionId, input.lookerConnectionName, input.kloConnectionId, input.source, this.now().toISOString());
|
||||
.run(input.lookerConnectionId, input.lookerConnectionName, input.ktxConnectionId, input.source, this.now().toISOString());
|
||||
}
|
||||
|
||||
async refreshDiscoveredConnections(input: RefreshLocalLookerDiscoveredConnectionsInput): Promise<void> {
|
||||
|
|
@ -234,7 +234,7 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
INSERT INTO local_looker_connection_mappings (
|
||||
looker_connection_id,
|
||||
looker_connection_name,
|
||||
klo_connection_id,
|
||||
ktx_connection_id,
|
||||
looker_host,
|
||||
looker_database,
|
||||
looker_dialect,
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import {
|
|||
projectParsedIdentifier,
|
||||
refreshLookerMappingPlaceholders,
|
||||
sqlglotDialectForConnectionType,
|
||||
suggestKloConnectionForLookerConnection,
|
||||
suggestKtxConnectionForLookerConnection,
|
||||
validateLookerMappings,
|
||||
validateLookerWarehouseTarget,
|
||||
} from './mapping.js';
|
||||
|
|
@ -69,7 +69,7 @@ describe('discoverLookerConnections', () => {
|
|||
});
|
||||
|
||||
describe('looker dialect and target validation helpers', () => {
|
||||
it('maps Looker dialect names to KLO connection types', () => {
|
||||
it('maps Looker dialect names to KTX connection types', () => {
|
||||
expect(lookerDialectToConnectionType('bigquery_standard_sql')).toBe('BIGQUERY');
|
||||
expect(lookerDialectToConnectionType('postgres')).toBe('POSTGRESQL');
|
||||
expect(lookerDialectToConnectionType('mssql')).toBe('SQLSERVER');
|
||||
|
|
@ -90,10 +90,10 @@ describe('looker dialect and target validation helpers', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('suggestKloConnectionForLookerConnection', () => {
|
||||
describe('suggestKtxConnectionForLookerConnection', () => {
|
||||
it('returns the single deterministic target with matching type, host, and database', () => {
|
||||
expect(
|
||||
suggestKloConnectionForLookerConnection({
|
||||
suggestKtxConnectionForLookerConnection({
|
||||
lookerConnection: liveConnections[1],
|
||||
candidateConnections: [
|
||||
{
|
||||
|
|
@ -113,7 +113,7 @@ describe('suggestKloConnectionForLookerConnection', () => {
|
|||
|
||||
it('returns null when more than one target matches', () => {
|
||||
expect(
|
||||
suggestKloConnectionForLookerConnection({
|
||||
suggestKtxConnectionForLookerConnection({
|
||||
lookerConnection: liveConnections[1],
|
||||
candidateConnections: [
|
||||
{
|
||||
|
|
@ -139,7 +139,7 @@ describe('refreshLookerMappingPlaceholders', () => {
|
|||
stored: [
|
||||
{
|
||||
lookerConnectionName: 'b2b_sandbox_bq',
|
||||
kloConnectionId: 'warehouse',
|
||||
ktxConnectionId: 'warehouse',
|
||||
lookerHost: null,
|
||||
lookerDatabase: null,
|
||||
lookerDialect: null,
|
||||
|
|
@ -152,14 +152,14 @@ describe('refreshLookerMappingPlaceholders', () => {
|
|||
mappings: [
|
||||
{
|
||||
lookerConnectionName: 'b2b_sandbox_bq',
|
||||
kloConnectionId: 'warehouse',
|
||||
ktxConnectionId: 'warehouse',
|
||||
lookerHost: 'warehouse.example.com',
|
||||
lookerDatabase: 'analytics',
|
||||
lookerDialect: 'bigquery_standard_sql',
|
||||
},
|
||||
{
|
||||
lookerConnectionName: 'pg_runtime',
|
||||
kloConnectionId: null,
|
||||
ktxConnectionId: null,
|
||||
lookerHost: 'pg.internal:5432',
|
||||
lookerDatabase: 'app',
|
||||
lookerDialect: 'postgres',
|
||||
|
|
@ -176,14 +176,14 @@ describe('computeLookerMappingDrift and validateLookerMappings', () => {
|
|||
storedMappings: [
|
||||
{
|
||||
lookerConnectionName: 'b2b_sandbox_bq',
|
||||
kloConnectionId: 'warehouse',
|
||||
ktxConnectionId: 'warehouse',
|
||||
lookerHost: null,
|
||||
lookerDatabase: null,
|
||||
lookerDialect: null,
|
||||
},
|
||||
{
|
||||
lookerConnectionName: 'stale_runtime',
|
||||
kloConnectionId: 'warehouse',
|
||||
ktxConnectionId: 'warehouse',
|
||||
lookerHost: null,
|
||||
lookerDatabase: null,
|
||||
lookerDialect: null,
|
||||
|
|
@ -194,7 +194,7 @@ describe('computeLookerMappingDrift and validateLookerMappings', () => {
|
|||
).toEqual({
|
||||
unmappedDiscovered: [liveConnections[1]],
|
||||
staleMappings: [{ lookerConnectionName: 'stale_runtime', reason: 'looker_connection_not_found' }],
|
||||
inSync: [{ lookerConnectionName: 'b2b_sandbox_bq', kloConnectionId: 'warehouse' }],
|
||||
inSync: [{ lookerConnectionName: 'b2b_sandbox_bq', ktxConnectionId: 'warehouse' }],
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -204,26 +204,26 @@ describe('computeLookerMappingDrift and validateLookerMappings', () => {
|
|||
mappings: [
|
||||
{
|
||||
lookerConnectionName: 'b2b_sandbox_bq',
|
||||
kloConnectionId: 'missing',
|
||||
ktxConnectionId: 'missing',
|
||||
lookerHost: null,
|
||||
lookerDatabase: null,
|
||||
lookerDialect: null,
|
||||
},
|
||||
{
|
||||
lookerConnectionName: 'pg_runtime',
|
||||
kloConnectionId: 'looker-target',
|
||||
ktxConnectionId: 'looker-target',
|
||||
lookerHost: null,
|
||||
lookerDatabase: null,
|
||||
lookerDialect: null,
|
||||
},
|
||||
],
|
||||
knownKloConnectionIds: new Set(['looker-target']),
|
||||
knownKtxConnectionIds: new Set(['looker-target']),
|
||||
knownConnectionTypes: new Map([['looker-target', 'LOOKER']]),
|
||||
}),
|
||||
).toEqual({
|
||||
ok: false,
|
||||
errors: [
|
||||
{ key: 'b2b_sandbox_bq', reason: 'KLO connection missing does not exist' },
|
||||
{ key: 'b2b_sandbox_bq', reason: 'KTX connection missing does not exist' },
|
||||
{
|
||||
key: 'pg_runtime',
|
||||
reason: 'Connection type LOOKER cannot be used as a Looker warehouse mapping target',
|
||||
|
|
@ -258,7 +258,7 @@ describe('collectExploreParseItems and projectParsedIdentifier', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('projects successful and failed parser rows into KLO parsed target tables', () => {
|
||||
it('projects successful and failed parser rows into KTX parsed target tables', () => {
|
||||
expect(
|
||||
projectParsedIdentifier({
|
||||
ok: true,
|
||||
|
|
@ -317,7 +317,7 @@ describe('buildLookerPullConfigFromInputs', () => {
|
|||
refreshedMappings: [
|
||||
{
|
||||
lookerConnectionName: 'b2b_sandbox_bq',
|
||||
kloConnectionId: 'warehouse',
|
||||
ktxConnectionId: 'warehouse',
|
||||
lookerHost: 'warehouse.example.com',
|
||||
lookerDatabase: 'analytics',
|
||||
lookerDialect: 'bigquery_standard_sql',
|
||||
|
|
@ -365,7 +365,7 @@ describe('buildLookerPullConfigFromInputs', () => {
|
|||
refreshedMappings: [
|
||||
{
|
||||
lookerConnectionName: 'b2b_sandbox_bq',
|
||||
kloConnectionId: 'warehouse',
|
||||
ktxConnectionId: 'warehouse',
|
||||
lookerHost: null,
|
||||
lookerDatabase: null,
|
||||
lookerDialect: null,
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ export type LookerWarehouseTargetConnectionType =
|
|||
|
||||
export interface LookerConnectionMapping {
|
||||
lookerConnectionName: string;
|
||||
kloConnectionId: string | null;
|
||||
ktxConnectionId: string | null;
|
||||
lookerHost: string | null;
|
||||
lookerDatabase: string | null;
|
||||
lookerDialect: string | null;
|
||||
|
|
@ -43,7 +43,7 @@ export interface LookerMappingCandidateConnection extends LookerTargetConnection
|
|||
export interface LookerMappingDrift {
|
||||
unmappedDiscovered: LookerWarehouseConnectionInfo[];
|
||||
staleMappings: Array<{ lookerConnectionName: string; reason: 'looker_connection_not_found' }>;
|
||||
inSync: Array<{ lookerConnectionName: string; kloConnectionId: string }>;
|
||||
inSync: Array<{ lookerConnectionName: string; ktxConnectionId: string }>;
|
||||
}
|
||||
|
||||
export type LookerMappingValidationResult =
|
||||
|
|
@ -155,7 +155,7 @@ export function normalizeName(value: string | null): string | null {
|
|||
return value ? value.toLowerCase() : null;
|
||||
}
|
||||
|
||||
export function suggestKloConnectionForLookerConnection(args: {
|
||||
export function suggestKtxConnectionForLookerConnection(args: {
|
||||
lookerConnection: LookerWarehouseConnectionInfo;
|
||||
candidateConnections: LookerMappingCandidateConnection[];
|
||||
}): string | null {
|
||||
|
|
@ -187,7 +187,7 @@ export function computeLookerMappingDrift(args: {
|
|||
const storedByName = new Map(args.storedMappings.map((mapping) => [mapping.lookerConnectionName, mapping]));
|
||||
|
||||
return {
|
||||
unmappedDiscovered: args.discovered.filter((connection) => !storedByName.get(connection.name)?.kloConnectionId),
|
||||
unmappedDiscovered: args.discovered.filter((connection) => !storedByName.get(connection.name)?.ktxConnectionId),
|
||||
staleMappings: args.storedMappings
|
||||
.filter((mapping) => !discoveredByName.has(mapping.lookerConnectionName))
|
||||
.map((mapping) => ({
|
||||
|
|
@ -195,32 +195,32 @@ export function computeLookerMappingDrift(args: {
|
|||
reason: 'looker_connection_not_found' as const,
|
||||
})),
|
||||
inSync: args.storedMappings
|
||||
.filter((mapping) => discoveredByName.has(mapping.lookerConnectionName) && mapping.kloConnectionId)
|
||||
.filter((mapping) => discoveredByName.has(mapping.lookerConnectionName) && mapping.ktxConnectionId)
|
||||
.map((mapping) => ({
|
||||
lookerConnectionName: mapping.lookerConnectionName,
|
||||
kloConnectionId: mapping.kloConnectionId as string,
|
||||
ktxConnectionId: mapping.ktxConnectionId as string,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
export function validateLookerMappings(args: {
|
||||
mappings: LookerConnectionMapping[];
|
||||
knownKloConnectionIds: Set<string>;
|
||||
knownKtxConnectionIds: Set<string>;
|
||||
knownConnectionTypes: ReadonlyMap<string, string>;
|
||||
}): LookerMappingValidationResult {
|
||||
const errors: Array<{ key: string; reason: string }> = [];
|
||||
for (const mapping of args.mappings) {
|
||||
if (!mapping.kloConnectionId) {
|
||||
if (!mapping.ktxConnectionId) {
|
||||
continue;
|
||||
}
|
||||
if (!args.knownKloConnectionIds.has(mapping.kloConnectionId)) {
|
||||
if (!args.knownKtxConnectionIds.has(mapping.ktxConnectionId)) {
|
||||
errors.push({
|
||||
key: mapping.lookerConnectionName,
|
||||
reason: `KLO connection ${mapping.kloConnectionId} does not exist`,
|
||||
reason: `KTX connection ${mapping.ktxConnectionId} does not exist`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const connectionType = args.knownConnectionTypes.get(mapping.kloConnectionId);
|
||||
const connectionType = args.knownConnectionTypes.get(mapping.ktxConnectionId);
|
||||
const validation = validateLookerWarehouseTarget(connectionType ?? 'unknown');
|
||||
if (!validation.ok) {
|
||||
errors.push({ key: mapping.lookerConnectionName, reason: validation.reason });
|
||||
|
|
@ -241,7 +241,7 @@ export function refreshLookerMappingPlaceholders(args: {
|
|||
if (!existing) {
|
||||
byName.set(live.name, {
|
||||
lookerConnectionName: live.name,
|
||||
kloConnectionId: null,
|
||||
ktxConnectionId: null,
|
||||
lookerHost: live.host,
|
||||
lookerDatabase: live.database,
|
||||
lookerDialect: live.dialect,
|
||||
|
|
@ -346,14 +346,14 @@ export async function buildLookerPullConfigFromInputs(args: {
|
|||
const connectionTypes: Record<string, LookerWarehouseTargetConnectionType> = {};
|
||||
|
||||
for (const mapping of args.refreshedMappings) {
|
||||
if (!mapping.kloConnectionId) {
|
||||
if (!mapping.ktxConnectionId) {
|
||||
continue;
|
||||
}
|
||||
const target = args.targetConnections.get(mapping.kloConnectionId);
|
||||
const target = args.targetConnections.get(mapping.ktxConnectionId);
|
||||
if (!target || !validateLookerWarehouseTarget(target.connection_type).ok) {
|
||||
continue;
|
||||
}
|
||||
connectionMappings[mapping.lookerConnectionName] = mapping.kloConnectionId;
|
||||
connectionMappings[mapping.lookerConnectionName] = mapping.ktxConnectionId;
|
||||
connectionTypes[mapping.lookerConnectionName] = target.connection_type as LookerWarehouseTargetConnectionType;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -255,7 +255,7 @@ describe('Looker staged runtime schemas', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('accepts slug-shaped connection ids inside KLO Looker runtime schemas', () => {
|
||||
it('accepts slug-shaped connection ids inside KTX Looker runtime schemas', () => {
|
||||
const parsedTargetTable = {
|
||||
ok: true as const,
|
||||
catalog: 'proj',
|
||||
|
|
@ -313,7 +313,7 @@ describe('Looker staged runtime schemas', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('rejects unsafe KLO Looker connection ids', () => {
|
||||
it('rejects unsafe KTX Looker connection ids', () => {
|
||||
expect(() =>
|
||||
parseLookerPullConfig({
|
||||
lookerConnectionId: '../prod-looker',
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ async function readMetabaseFile(name: string): Promise<string> {
|
|||
return readFile(join(metabaseDir, name), 'utf-8');
|
||||
}
|
||||
|
||||
describe('KLO Metabase client boundary', () => {
|
||||
it('keeps NestJS, server data-source base classes, and server-relative imports out of the KLO client', async () => {
|
||||
describe('KTX Metabase client boundary', () => {
|
||||
it('keeps NestJS, server data-source base classes, and server-relative imports out of the KTX client', async () => {
|
||||
const client = await readMetabaseFile('client.ts');
|
||||
expect(client).not.toContain(`@${'nestjs'}`);
|
||||
expect(client).not.toContain(`DataSource${'Client'}`);
|
||||
|
|
@ -19,7 +19,7 @@ describe('KLO Metabase client boundary', () => {
|
|||
expect(client).not.toContain('../../types/brand');
|
||||
});
|
||||
|
||||
it('keeps proxy implementation code out of the KLO v1 client', async () => {
|
||||
it('keeps proxy implementation code out of the KTX v1 client', async () => {
|
||||
const client = await readMetabaseFile('client.ts');
|
||||
expect(client).not.toContain(`network-${'proxy'}`);
|
||||
expect(client).not.toContain(`ssh${'2'}`);
|
||||
|
|
|
|||
|
|
@ -199,7 +199,7 @@ describe('MetabaseClient admin auth helpers', () => {
|
|||
);
|
||||
|
||||
await expect(client.getPermissionGroups()).resolves.toEqual([{ id: 2, name: 'Administrators' }]);
|
||||
await expect(client.createApiKey({ name: 'KLO CLI test', groupId: 2 })).resolves.toBe(mintedMetabaseCredential);
|
||||
await expect(client.createApiKey({ name: 'KTX CLI test', groupId: 2 })).resolves.toBe(mintedMetabaseCredential);
|
||||
|
||||
expect(fetchMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
|
|
@ -214,7 +214,7 @@ describe('MetabaseClient admin auth helpers', () => {
|
|||
'https://metabase.example.test/api/api-key',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ name: 'KLO CLI test', group_id: 2 }),
|
||||
body: JSON.stringify({ name: 'KTX CLI test', group_id: 2 }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
|
@ -343,7 +343,7 @@ describe('MetabaseClient.getResolvedSql', () => {
|
|||
expect(result?.resolutionStatus).toBe('resolved');
|
||||
const sql = result?.resolvedSql ?? '';
|
||||
expect(sql.startsWith('--')).toBe(true);
|
||||
expect(sql).toMatch(/KLO_PLACEHOLDER_WARNING/);
|
||||
expect(sql).toMatch(/KTX_PLACEHOLDER_WARNING/);
|
||||
expect(sql).toMatch(/\bid\b/);
|
||||
expect(sql).toMatch(/\bn\b/);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ class MetabaseApiError extends Error {
|
|||
* Strip Metabase `[[ ... {{ var }} ... ]]` optional-clause blocks from native SQL.
|
||||
*
|
||||
* The bracketed blocks are emitted only when the embedded `{{ var }}` is supplied at
|
||||
* Metabase query time. For KLO semantic-layer ingest there's no such runtime
|
||||
* Metabase query time. For KTX semantic-layer ingest there's no such runtime
|
||||
* parameter — chat-time filters are composed by the SL query planner — so the optional
|
||||
* block must be removed before the SQL becomes a permanent SL source. Substituting a
|
||||
* dummy value (the alternative) bakes a placeholder filter into the source and silently
|
||||
|
|
@ -425,7 +425,7 @@ export class MetabaseClient implements MetabaseRuntimeClient {
|
|||
|
||||
private buildPlaceholderWarningComment(tags: MetabaseTemplateTag[]): string {
|
||||
const lines = [
|
||||
'-- KLO_PLACEHOLDER_WARNING: this SQL was extracted from a Metabase card with',
|
||||
'-- KTX_PLACEHOLDER_WARNING: this SQL was extracted from a Metabase card with',
|
||||
'-- unbound template parameters. The placeholders below were substituted with DUMMY',
|
||||
"-- values to satisfy Metabase's parser — they DO NOT represent intended filters.",
|
||||
'-- Drop the corresponding clauses (or expose them as runtime SL filters) before',
|
||||
|
|
|
|||
|
|
@ -240,7 +240,7 @@ describe('fetchMetabaseBundle', () => {
|
|||
clientFactory,
|
||||
sourceStateReader,
|
||||
}),
|
||||
).rejects.toThrow(/unhydrated.*klo connection mapping refresh/);
|
||||
).rejects.toThrow(/unhydrated.*ktx connection mapping refresh/);
|
||||
});
|
||||
|
||||
it('skips cards whose getResolvedSql returns null and records them in unresolved-cards.json', async () => {
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ export async function fetchMetabaseBundle(params: FetchMetabaseBundleParams): Pr
|
|||
}
|
||||
if (mapping.metabaseDatabaseName === null) {
|
||||
throw new IngestInputError(
|
||||
`mapping for database ${pullConfig.metabaseDatabaseId} on Metabase connection ${pullConfig.metabaseConnectionId} is unhydrated; run \`klo connection mapping refresh ${pullConfig.metabaseConnectionId}\` to populate metabaseDatabaseName before ingest.`,
|
||||
`mapping for database ${pullConfig.metabaseDatabaseId} on Metabase connection ${pullConfig.metabaseConnectionId} is unhydrated; run \`ktx connection mapping refresh ${pullConfig.metabaseConnectionId}\` to populate metabaseDatabaseName before ingest.`,
|
||||
);
|
||||
}
|
||||
const mappingDatabaseName: string = mapping.metabaseDatabaseName;
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type { KloProjectConnectionConfig } from '../../../project/index.js';
|
||||
import type { KtxProjectConnectionConfig } from '../../../project/index.js';
|
||||
import { metabaseRuntimeConfigFromLocalConnection } from './local-metabase.adapter.js';
|
||||
|
||||
describe('metabaseRuntimeConfigFromLocalConnection', () => {
|
||||
it('resolves api_url and env-backed api_key_ref from a flat klo.yaml connection', () => {
|
||||
const connection: KloProjectConnectionConfig = {
|
||||
it('resolves api_url and env-backed api_key_ref from a flat ktx.yaml connection', () => {
|
||||
const connection: KtxProjectConnectionConfig = {
|
||||
driver: 'metabase',
|
||||
api_url: 'https://metabase.example.com',
|
||||
api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret
|
||||
|
|
@ -21,7 +21,7 @@ describe('metabaseRuntimeConfigFromLocalConnection', () => {
|
|||
});
|
||||
|
||||
it('accepts url as the local api URL alias', () => {
|
||||
const connection: KloProjectConnectionConfig = {
|
||||
const connection: KtxProjectConnectionConfig = {
|
||||
driver: 'metabase',
|
||||
url: 'https://metabase.example.com',
|
||||
api_key: 'literal-test-key', // pragma: allowlist secret
|
||||
|
|
@ -34,7 +34,7 @@ describe('metabaseRuntimeConfigFromLocalConnection', () => {
|
|||
});
|
||||
|
||||
it('rejects proxy-bearing local Metabase connections', () => {
|
||||
const connection: KloProjectConnectionConfig = {
|
||||
const connection: KtxProjectConnectionConfig = {
|
||||
driver: 'metabase',
|
||||
api_url: 'https://metabase.example.com',
|
||||
api_key: 'literal-test-key', // pragma: allowlist secret
|
||||
|
|
@ -42,12 +42,12 @@ describe('metabaseRuntimeConfigFromLocalConnection', () => {
|
|||
};
|
||||
|
||||
expect(() => metabaseRuntimeConfigFromLocalConnection('prod-metabase', connection)).toThrow(
|
||||
'Standalone KLO does not support proxy-bearing Metabase connections yet',
|
||||
'Standalone KTX does not support proxy-bearing Metabase connections yet',
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects non-Metabase source connections', () => {
|
||||
const connection: KloProjectConnectionConfig = {
|
||||
const connection: KtxProjectConnectionConfig = {
|
||||
driver: 'postgres',
|
||||
url: 'postgres://localhost/db',
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import type { KloLocalProject, KloProjectConnectionConfig } from '../../../project/index.js';
|
||||
import { kloLocalStateDbPath } from '../../../project/index.js';
|
||||
import type { KtxLocalProject, KtxProjectConnectionConfig } from '../../../project/index.js';
|
||||
import { ktxLocalStateDbPath } from '../../../project/index.js';
|
||||
import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory } from './client.js';
|
||||
import {
|
||||
IngestMetabaseClientFactory,
|
||||
|
|
@ -21,13 +21,13 @@ function resolveEnvReference(ref: string, env: NodeJS.ProcessEnv): string | null
|
|||
return stringField(env[name]);
|
||||
}
|
||||
|
||||
function hasNetworkProxy(connection: KloProjectConnectionConfig): boolean {
|
||||
function hasNetworkProxy(connection: KtxProjectConnectionConfig): boolean {
|
||||
return connection.networkProxy != null || connection.network_proxy != null;
|
||||
}
|
||||
|
||||
export function metabaseRuntimeConfigFromLocalConnection(
|
||||
connectionId: string,
|
||||
connection: KloProjectConnectionConfig | undefined,
|
||||
connection: KtxProjectConnectionConfig | undefined,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): MetabaseClientRuntimeConfig {
|
||||
if (!connection || String(connection.driver).toLowerCase() !== 'metabase') {
|
||||
|
|
@ -35,7 +35,7 @@ export function metabaseRuntimeConfigFromLocalConnection(
|
|||
}
|
||||
if (hasNetworkProxy(connection)) {
|
||||
throw new Error(
|
||||
`Standalone KLO does not support proxy-bearing Metabase connections yet. Use hosted Metabase ingest for "${connectionId}" until the KLO Metabase proxy support spec lands.`,
|
||||
`Standalone KTX does not support proxy-bearing Metabase connections yet. Use hosted Metabase ingest for "${connectionId}" until the KTX Metabase proxy support spec lands.`,
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -60,10 +60,10 @@ interface CreateLocalMetabaseSourceAdapterOptions {
|
|||
}
|
||||
|
||||
export function createLocalMetabaseSourceAdapter(
|
||||
project: KloLocalProject,
|
||||
project: KtxLocalProject,
|
||||
options: CreateLocalMetabaseSourceAdapterOptions = {},
|
||||
): MetabaseSourceAdapter {
|
||||
const sourceStateReader = new LocalMetabaseSourceStateReader({ dbPath: kloLocalStateDbPath(project) });
|
||||
const sourceStateReader = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) });
|
||||
const connectionFactory = new DefaultMetabaseConnectionClientFactory(
|
||||
(metabaseConnectionId) =>
|
||||
metabaseRuntimeConfigFromLocalConnection(
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ describe('LocalMetabaseSourceStateReader', () => {
|
|||
let store: LocalMetabaseSourceStateReader;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-metabase-local-state-'));
|
||||
store = new LocalMetabaseSourceStateReader({ dbPath: join(tempDir, '.klo', 'db.sqlite') });
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-metabase-local-state-'));
|
||||
store = new LocalMetabaseSourceStateReader({ dbPath: join(tempDir, '.ktx', 'db.sqlite') });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
@ -74,7 +74,7 @@ describe('LocalMetabaseSourceStateReader', () => {
|
|||
metabaseDbName: null,
|
||||
targetConnectionId: 'warehouse',
|
||||
syncEnabled: true,
|
||||
source: 'klo.yaml',
|
||||
source: 'ktx.yaml',
|
||||
},
|
||||
{
|
||||
metabaseDatabaseId: 2,
|
||||
|
|
@ -247,7 +247,7 @@ describe('LocalMetabaseSourceStateReader', () => {
|
|||
await store.applyYamlBootstrap({
|
||||
connectionId: 'prod-metabase',
|
||||
syncMode: 'ALL',
|
||||
defaultTagNames: ['klo'],
|
||||
defaultTagNames: ['ktx'],
|
||||
selections: [{ selectionType: 'collection', metabaseObjectId: 12 }],
|
||||
mappings: [{ metabaseDatabaseId: 1, targetConnectionId: 'prod-warehouse', syncEnabled: true }],
|
||||
});
|
||||
|
|
@ -255,7 +255,7 @@ describe('LocalMetabaseSourceStateReader', () => {
|
|||
await expect(store.getUnhydratedSyncEnabledMappingIds('prod-metabase')).resolves.toEqual([1]);
|
||||
await expect(store.getSourceState('prod-metabase')).resolves.toMatchObject({
|
||||
syncMode: 'ALL',
|
||||
defaultTagNames: ['klo'],
|
||||
defaultTagNames: ['ktx'],
|
||||
selections: [{ selectionType: 'collection', metabaseObjectId: 12 }],
|
||||
mappings: [],
|
||||
});
|
||||
|
|
@ -265,7 +265,7 @@ describe('LocalMetabaseSourceStateReader', () => {
|
|||
metabaseDatabaseName: null,
|
||||
targetConnectionId: 'prod-warehouse',
|
||||
syncEnabled: true,
|
||||
source: 'klo.yaml',
|
||||
source: 'ktx.yaml',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
|
@ -301,7 +301,7 @@ describe('LocalMetabaseSourceStateReader', () => {
|
|||
metabaseEngine: 'postgres',
|
||||
targetConnectionId: 'yaml-warehouse',
|
||||
syncEnabled: true,
|
||||
source: 'klo.yaml',
|
||||
source: 'ktx.yaml',
|
||||
},
|
||||
{
|
||||
metabaseDatabaseId: 2,
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import Database from 'better-sqlite3';
|
|||
import type { MetabaseSourceState, MetabaseSourceStateReader, MetabaseSourceStateSelection } from './source-state-port.js';
|
||||
import type { MetabaseSyncMode } from './types.js';
|
||||
|
||||
export type LocalMetabaseMappingSource = 'klo.yaml' | 'cli' | 'refresh';
|
||||
export type LocalMetabaseMappingSource = 'ktx.yaml' | 'cli' | 'refresh';
|
||||
|
||||
interface LocalMetabaseSourceStateStoreOptions {
|
||||
dbPath: string;
|
||||
|
|
@ -197,13 +197,13 @@ export class LocalMetabaseSourceStateReader implements MetabaseSourceStateReader
|
|||
source,
|
||||
updated_at
|
||||
)
|
||||
VALUES (?, ?, NULL, NULL, NULL, NULL, ?, ?, 'klo.yaml', ?)
|
||||
VALUES (?, ?, NULL, NULL, NULL, NULL, ?, ?, 'ktx.yaml', ?)
|
||||
`);
|
||||
const updateRefreshRow = this.db.prepare(`
|
||||
UPDATE local_metabase_database_mappings
|
||||
SET target_connection_id = ?,
|
||||
sync_enabled = ?,
|
||||
source = 'klo.yaml',
|
||||
source = 'ktx.yaml',
|
||||
updated_at = ?
|
||||
WHERE metabase_connection_id = ?
|
||||
AND metabase_database_id = ?
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ describe('computeMetabaseMappingDrift', () => {
|
|||
{ id: 3, name: 'Warehouse', engine: 'mysql', host: 'mysql.internal', dbName: 'warehouse' },
|
||||
],
|
||||
staleMappings: [{ id: '9', reason: 'database_not_found' }],
|
||||
inSync: [{ id: 2, kloConnectionId: 'target-postgres' }],
|
||||
inSync: [{ id: 2, ktxConnectionId: 'target-postgres' }],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -74,7 +74,7 @@ describe('validateMetabaseMappings', () => {
|
|||
expect(
|
||||
validateMetabaseMappings({
|
||||
mappings: { '2': 'target-postgres' },
|
||||
knownKloConnectionIds: new Set(['target-postgres']),
|
||||
knownKtxConnectionIds: new Set(['target-postgres']),
|
||||
}),
|
||||
).toEqual({ ok: true });
|
||||
});
|
||||
|
|
@ -83,11 +83,11 @@ describe('validateMetabaseMappings', () => {
|
|||
expect(
|
||||
validateMetabaseMappings({
|
||||
mappings: { '2': 'missing-target', '3': 'target-mysql' },
|
||||
knownKloConnectionIds: new Set(['target-mysql']),
|
||||
knownKtxConnectionIds: new Set(['target-mysql']),
|
||||
}),
|
||||
).toEqual({
|
||||
ok: false,
|
||||
errors: [{ key: '2', reason: 'KLO connection missing-target does not exist' }],
|
||||
errors: [{ key: '2', reason: 'KTX connection missing-target does not exist' }],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -149,7 +149,7 @@ describe('validateMappingPhysicalMatch', () => {
|
|||
).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for unknown engines because KLO cannot validate them', () => {
|
||||
it('returns null for unknown engines because KTX cannot validate them', () => {
|
||||
expect(
|
||||
validateMappingPhysicalMatch(
|
||||
{ metabaseEngine: 'unknown-engine', metabaseDbName: 'X', metabaseHost: 'host' },
|
||||
|
|
@ -177,7 +177,7 @@ describe('computeMetabaseMappingPhysicalMismatches', () => {
|
|||
).toEqual([
|
||||
{
|
||||
mappingId: 'mapping-bad',
|
||||
reason: "Metabase database 'app' does not match KLO connection database 'other_app'",
|
||||
reason: "Metabase database 'app' does not match KTX connection database 'other_app'",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
|
@ -201,7 +201,7 @@ describe('refreshMetabaseMapping', () => {
|
|||
refreshMetabaseMapping({
|
||||
client,
|
||||
currentMappings: { '2': 'target-postgres' },
|
||||
resolveKloConnectionPhysicalInfo: vi.fn().mockResolvedValue({
|
||||
resolveKtxConnectionPhysicalInfo: vi.fn().mockResolvedValue({
|
||||
connection_type: 'POSTGRESQL',
|
||||
host: 'pg.internal',
|
||||
database: 'wrong_database',
|
||||
|
|
@ -211,12 +211,12 @@ describe('refreshMetabaseMapping', () => {
|
|||
drift: {
|
||||
unmappedDiscovered: [],
|
||||
staleMappings: [],
|
||||
inSync: [{ id: 2, kloConnectionId: 'target-postgres' }],
|
||||
inSync: [{ id: 2, ktxConnectionId: 'target-postgres' }],
|
||||
},
|
||||
physicalMismatches: [
|
||||
{
|
||||
mappingId: '2',
|
||||
reason: "Metabase database 'analytics' does not match KLO connection database 'wrong_database'",
|
||||
reason: "Metabase database 'analytics' does not match KTX connection database 'wrong_database'",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
|
@ -282,7 +282,7 @@ describe('findBestMatch', () => {
|
|||
});
|
||||
|
||||
describe('METABASE_ENGINE_TO_CONNECTION_TYPE', () => {
|
||||
it('keeps the server-supported Metabase engine table in KLO', () => {
|
||||
it('keeps the server-supported Metabase engine table in KTX', () => {
|
||||
expect(METABASE_ENGINE_TO_CONNECTION_TYPE).toMatchObject({
|
||||
postgres: 'POSTGRESQL',
|
||||
bigquery: 'BIGQUERY',
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ export interface DiscoveredMetabaseDatabase {
|
|||
export interface MetabaseMappingDrift {
|
||||
unmappedDiscovered: DiscoveredMetabaseDatabase[];
|
||||
staleMappings: Array<{ id: string; reason: 'database_not_found' }>;
|
||||
inSync: Array<{ id: number; kloConnectionId: string }>;
|
||||
inSync: Array<{ id: number; ktxConnectionId: string }>;
|
||||
}
|
||||
|
||||
export interface MappingPhysicalInfo {
|
||||
|
|
@ -32,7 +32,7 @@ export interface MappingPhysicalInfo {
|
|||
metabaseHost: string | null;
|
||||
}
|
||||
|
||||
export interface KloConnectionPhysicalInfo {
|
||||
export interface KtxConnectionPhysicalInfo {
|
||||
connection_type: string;
|
||||
database?: unknown;
|
||||
host?: unknown;
|
||||
|
|
@ -45,7 +45,7 @@ export interface KloConnectionPhysicalInfo {
|
|||
export interface PhysicalMismatchInput {
|
||||
mappingId: string;
|
||||
metabase: MappingPhysicalInfo;
|
||||
target: KloConnectionPhysicalInfo;
|
||||
target: KtxConnectionPhysicalInfo;
|
||||
}
|
||||
|
||||
export interface PhysicalMismatch {
|
||||
|
|
@ -102,7 +102,7 @@ function displayValue(value: unknown): string {
|
|||
return typeof value === 'string' && value.length > 0 ? value : 'unknown';
|
||||
}
|
||||
|
||||
function getTargetDatabase(target: KloConnectionPhysicalInfo): unknown {
|
||||
function getTargetDatabase(target: KtxConnectionPhysicalInfo): unknown {
|
||||
if (target.connection_type === 'BIGQUERY') {
|
||||
return target.dataset_id ?? target.project_id ?? target.database;
|
||||
}
|
||||
|
|
@ -164,23 +164,23 @@ export function computeMetabaseMappingDrift(args: {
|
|||
.filter((id) => !discoveredById.has(id))
|
||||
.map((id) => ({ id, reason: 'database_not_found' as const }));
|
||||
const inSync = Object.entries(args.currentMappings)
|
||||
.filter(([id, kloConnectionId]) => discoveredById.has(id) && typeof kloConnectionId === 'string')
|
||||
.map(([id, kloConnectionId]) => ({ id: Number(id), kloConnectionId: kloConnectionId as string }));
|
||||
.filter(([id, ktxConnectionId]) => discoveredById.has(id) && typeof ktxConnectionId === 'string')
|
||||
.map(([id, ktxConnectionId]) => ({ id: Number(id), ktxConnectionId: ktxConnectionId as string }));
|
||||
|
||||
return { unmappedDiscovered, staleMappings, inSync };
|
||||
}
|
||||
|
||||
export function validateMetabaseMappings(args: {
|
||||
mappings: Record<string, string | null | undefined>;
|
||||
knownKloConnectionIds: Set<string>;
|
||||
knownKtxConnectionIds: Set<string>;
|
||||
}): MetabaseMappingValidationResult {
|
||||
const errors: Array<{ key: string; reason: string }> = [];
|
||||
for (const [key, connectionId] of Object.entries(args.mappings)) {
|
||||
if (!connectionId) {
|
||||
continue;
|
||||
}
|
||||
if (!args.knownKloConnectionIds.has(connectionId)) {
|
||||
errors.push({ key, reason: `KLO connection ${connectionId} does not exist` });
|
||||
if (!args.knownKtxConnectionIds.has(connectionId)) {
|
||||
errors.push({ key, reason: `KTX connection ${connectionId} does not exist` });
|
||||
}
|
||||
}
|
||||
return errors.length === 0 ? { ok: true } : { ok: false, errors };
|
||||
|
|
@ -188,7 +188,7 @@ export function validateMetabaseMappings(args: {
|
|||
|
||||
export function validateMappingPhysicalMatch(
|
||||
mapping: MappingPhysicalInfo,
|
||||
target: KloConnectionPhysicalInfo,
|
||||
target: KtxConnectionPhysicalInfo,
|
||||
): string | null {
|
||||
const engine = mapping.metabaseEngine?.toLowerCase();
|
||||
if (!engine) {
|
||||
|
|
@ -201,7 +201,7 @@ export function validateMappingPhysicalMatch(
|
|||
}
|
||||
|
||||
if (target.connection_type !== expectedType) {
|
||||
return `Metabase database engine '${engine}' does not match KLO connection type '${target.connection_type}'`;
|
||||
return `Metabase database engine '${engine}' does not match KTX connection type '${target.connection_type}'`;
|
||||
}
|
||||
|
||||
const metabaseDb = normalizeName(mapping.metabaseDbName);
|
||||
|
|
@ -209,7 +209,7 @@ export function validateMappingPhysicalMatch(
|
|||
|
||||
if (engine === 'snowflake' || engine === 'bigquery' || engine === 'bigquery-cloud-sdk') {
|
||||
if (metabaseDb && targetDb && metabaseDb !== targetDb) {
|
||||
return `Metabase database '${mapping.metabaseDbName}' does not match KLO connection database '${displayValue(
|
||||
return `Metabase database '${mapping.metabaseDbName}' does not match KTX connection database '${displayValue(
|
||||
getTargetDatabase(target),
|
||||
)}'`;
|
||||
}
|
||||
|
|
@ -221,12 +221,12 @@ export function validateMappingPhysicalMatch(
|
|||
const targetHost = normalizeHost(target.host);
|
||||
|
||||
if (metabaseHost && targetHost && metabaseHost !== targetHost) {
|
||||
return `Metabase host '${mapping.metabaseHost}' does not match KLO connection host '${displayValue(
|
||||
return `Metabase host '${mapping.metabaseHost}' does not match KTX connection host '${displayValue(
|
||||
target.host,
|
||||
)}'`;
|
||||
}
|
||||
if (metabaseDb && targetDb && metabaseDb !== targetDb) {
|
||||
return `Metabase database '${mapping.metabaseDbName}' does not match KLO connection database '${displayValue(
|
||||
return `Metabase database '${mapping.metabaseDbName}' does not match KTX connection database '${displayValue(
|
||||
getTargetDatabase(target),
|
||||
)}'`;
|
||||
}
|
||||
|
|
@ -250,7 +250,7 @@ export function computeMetabaseMappingPhysicalMismatches(inputs: PhysicalMismatc
|
|||
export async function refreshMetabaseMapping(args: {
|
||||
client: Pick<MetabaseRuntimeClient, 'getDatabases'>;
|
||||
currentMappings: Record<string, string | null | undefined>;
|
||||
resolveKloConnectionPhysicalInfo: (kloConnectionId: string) => Promise<KloConnectionPhysicalInfo | null>;
|
||||
resolveKtxConnectionPhysicalInfo: (ktxConnectionId: string) => Promise<KtxConnectionPhysicalInfo | null>;
|
||||
}): Promise<MappingRefreshReport> {
|
||||
const discovered = await discoverMetabaseDatabases(args.client);
|
||||
const drift = computeMetabaseMappingDrift({ currentMappings: args.currentMappings, discovered });
|
||||
|
|
@ -262,11 +262,11 @@ export async function refreshMetabaseMapping(args: {
|
|||
if (!discoveredDatabase) {
|
||||
continue;
|
||||
}
|
||||
const target = await args.resolveKloConnectionPhysicalInfo(mapping.kloConnectionId);
|
||||
const target = await args.resolveKtxConnectionPhysicalInfo(mapping.ktxConnectionId);
|
||||
if (!target) {
|
||||
physicalMismatches.push({
|
||||
mappingId: String(mapping.id),
|
||||
reason: `KLO connection ${mapping.kloConnectionId} does not exist`,
|
||||
reason: `KTX connection ${mapping.ktxConnectionId} does not exist`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { parse as parseYaml } from 'yaml';
|
||||
import { noopLogger, type KloLogger } from '../../../core/index.js';
|
||||
import { noopLogger, type KtxLogger } from '../../../core/index.js';
|
||||
|
||||
export interface DimensionDefinition {
|
||||
name: string;
|
||||
|
|
@ -42,7 +42,7 @@ export interface ParsedMetricflowRelationship {
|
|||
}
|
||||
|
||||
export interface MetricflowParseOptions {
|
||||
logger?: KloLogger;
|
||||
logger?: KtxLogger;
|
||||
}
|
||||
|
||||
// ============ MetricFlow YAML Interfaces ============
|
||||
|
|
@ -191,7 +191,7 @@ export function translateMetricflowJinjaFilter(filter: string): string {
|
|||
}
|
||||
|
||||
class MetricflowDeepParser {
|
||||
constructor(private readonly logger: KloLogger) {}
|
||||
constructor(private readonly logger: KtxLogger) {}
|
||||
|
||||
parseFiles(files: Array<{ content: string; path: string }>): MetricFlowParseResult {
|
||||
this.logger.log(`Parsing ${files.length} files for MetricFlow definitions`);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { mkdir, mkdtemp, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { describe, expect, test } from 'vitest';
|
||||
import type { KloEmbeddingPort } from '../../../core/embedding.js';
|
||||
import type { KtxEmbeddingPort } from '../../../core/embedding.js';
|
||||
import type { WorkUnit } from '../../types.js';
|
||||
import { clusterNotionWorkUnits, MIN_PAGES_TO_CLUSTER } from './cluster.js';
|
||||
|
||||
|
|
@ -14,7 +14,7 @@ function fakeEmbedding(text: string): number[] {
|
|||
return v;
|
||||
}
|
||||
|
||||
const mockEmbed: KloEmbeddingPort = {
|
||||
const mockEmbed: KtxEmbeddingPort = {
|
||||
maxBatchSize: 100,
|
||||
computeEmbedding: async (t: string) => fakeEmbedding(t),
|
||||
computeEmbeddingsBulk: async (texts: string[]) => texts.map(fakeEmbedding),
|
||||
|
|
@ -104,7 +104,7 @@ describe('clusterNotionWorkUnits', () => {
|
|||
}));
|
||||
const stagedDir = await makeStaged(pages);
|
||||
const wus = makeWorkUnits(pages);
|
||||
const failingEmbed: KloEmbeddingPort = {
|
||||
const failingEmbed: KtxEmbeddingPort = {
|
||||
maxBatchSize: 100,
|
||||
computeEmbedding: async () => {
|
||||
throw new Error('embedding down');
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { readFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import type { KloEmbeddingPort } from '../../../core/embedding.js';
|
||||
import type { KtxEmbeddingPort } from '../../../core/embedding.js';
|
||||
import { kmeans, pickK } from '../../clustering/kmeans.js';
|
||||
import type { WorkUnit } from '../../types.js';
|
||||
import { notionMetadataSchema } from './types.js';
|
||||
|
|
@ -12,7 +12,7 @@ const CLUSTER_SEED = 42;
|
|||
interface ClusterNotionWorkUnitsArgs {
|
||||
workUnits: WorkUnit[];
|
||||
stagedDir: string;
|
||||
embedding: KloEmbeddingPort;
|
||||
embedding: KtxEmbeddingPort;
|
||||
}
|
||||
|
||||
async function buildClusterText(wu: WorkUnit, stagedDir: string): Promise<string> {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { type KloLogger, noopLogger } from '../../core/index.js';
|
||||
import { type KtxLogger, noopLogger } from '../../core/index.js';
|
||||
import type { CandidateDedupResult, ContextCandidateForDedup, JsonValue } from '../ports.js';
|
||||
import { buildContextCandidateEmbeddingText } from './embedding-text.js';
|
||||
import type { ContextCandidateStorePort } from './store.js';
|
||||
|
|
@ -17,11 +17,11 @@ export interface CandidateDedupServiceDeps {
|
|||
store: ContextCandidateStorePort;
|
||||
embeddings: ContextCandidateEmbeddingPort;
|
||||
settings: CandidateDedupSettings;
|
||||
logger?: KloLogger;
|
||||
logger?: KtxLogger;
|
||||
}
|
||||
|
||||
export class CandidateDedupService {
|
||||
private readonly logger: KloLogger;
|
||||
private readonly logger: KtxLogger;
|
||||
|
||||
constructor(private readonly deps: CandidateDedupServiceDeps) {
|
||||
this.logger = deps.logger ?? noopLogger;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { type KloLogger, noopLogger } from '../../core/index.js';
|
||||
import { type KtxLogger, noopLogger } from '../../core/index.js';
|
||||
import type { JsonValue } from '../ports.js';
|
||||
import type { ContextCandidateStorePort } from './store.js';
|
||||
import type {
|
||||
|
|
@ -26,11 +26,11 @@ export interface ContextCandidateCarryforwardResult {
|
|||
export interface ContextCandidateCarryforwardServiceDeps {
|
||||
store: ContextCandidateStorePort;
|
||||
settings: ContextCandidateCarryforwardSettings;
|
||||
logger?: KloLogger;
|
||||
logger?: KtxLogger;
|
||||
}
|
||||
|
||||
export class ContextCandidateCarryforwardService {
|
||||
private readonly logger: KloLogger;
|
||||
private readonly logger: KtxLogger;
|
||||
|
||||
constructor(private readonly deps: ContextCandidateCarryforwardServiceDeps) {
|
||||
this.logger = deps.logger ?? noopLogger;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import type { KloModelRole } from '@klo/llm';
|
||||
import type { KtxModelRole } from '@ktx/llm';
|
||||
import type { ToolSet } from 'ai';
|
||||
import type { AgentRunnerService } from '../../agent/index.js';
|
||||
import { type KloLogger, noopLogger } from '../../core/index.js';
|
||||
import { type KtxLogger, noopLogger } from '../../core/index.js';
|
||||
import type { MemoryAction } from '../../memory/index.js';
|
||||
import type { ContextCandidateForDedup, CuratorPaginationPort, CuratorPaginationReport } from '../ports.js';
|
||||
import type {
|
||||
|
|
@ -35,7 +35,7 @@ export interface CuratorPaginationInput {
|
|||
evictionUnit: EvictionUnit | undefined;
|
||||
representatives: ContextCandidateForDedup[];
|
||||
initialBudget: CuratorPaginationBudget;
|
||||
modelRole: KloModelRole;
|
||||
modelRole: KtxModelRole;
|
||||
buildSystemPrompt: () => string;
|
||||
buildUserPrompt: (input: CuratorPaginationPromptInput) => string;
|
||||
buildToolSet: (passNumber: number) => ToolSet;
|
||||
|
|
@ -52,11 +52,11 @@ export interface CuratorPaginationServiceDeps {
|
|||
store: ContextCandidateStorePort;
|
||||
agentRunner: AgentRunnerService;
|
||||
settings: CuratorPaginationSettings;
|
||||
logger?: KloLogger;
|
||||
logger?: KtxLogger;
|
||||
}
|
||||
|
||||
export class CuratorPaginationService implements CuratorPaginationPort {
|
||||
private readonly logger: KloLogger;
|
||||
private readonly logger: KtxLogger;
|
||||
|
||||
constructor(private readonly deps: CuratorPaginationServiceDeps) {
|
||||
this.logger = deps.logger ?? noopLogger;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { readdir, readFile } from 'node:fs/promises';
|
||||
import { basename, dirname, join, relative } from 'node:path';
|
||||
import { noopLogger, type KloLogger } from '../../core/index.js';
|
||||
import { noopLogger, type KtxLogger } from '../../core/index.js';
|
||||
import type { JsonValue } from '../ports.js';
|
||||
import type { DiffSet } from '../types.js';
|
||||
import type { ContextEvidenceIndexStorePort } from './store.js';
|
||||
|
|
@ -32,7 +32,7 @@ interface PublishSyncArgs {
|
|||
interface ContextEvidenceIndexServiceDeps {
|
||||
store: ContextEvidenceIndexStorePort;
|
||||
embeddings: ContextEvidenceEmbeddingPort;
|
||||
logger?: Pick<KloLogger, 'warn'>;
|
||||
logger?: Pick<KtxLogger, 'warn'>;
|
||||
}
|
||||
|
||||
type JsonObject = { [key: string]: JsonValue | undefined };
|
||||
|
|
@ -66,7 +66,7 @@ interface MarkdownChunk {
|
|||
export class ContextEvidenceIndexService {
|
||||
private readonly store: ContextEvidenceIndexStorePort;
|
||||
private readonly embeddings: ContextEvidenceEmbeddingPort;
|
||||
private readonly logger: Pick<KloLogger, 'warn'>;
|
||||
private readonly logger: Pick<KtxLogger, 'warn'>;
|
||||
|
||||
constructor(deps: ContextEvidenceIndexServiceDeps) {
|
||||
this.store = deps.store;
|
||||
|
|
|
|||
|
|
@ -11,8 +11,8 @@ describe('SqliteContextEvidenceStore', () => {
|
|||
let dbPath: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-context-evidence-sqlite-'));
|
||||
dbPath = join(tempDir, '.klo', 'db.sqlite');
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-context-evidence-sqlite-'));
|
||||
dbPath = join(tempDir, '.ktx', 'db.sqlite');
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@ export { DbtSourceAdapter } from './adapters/dbt/dbt.adapter.js';
|
|||
export { FakeSourceAdapter } from './adapters/fake/fake.adapter.js';
|
||||
export type {
|
||||
DaemonLiveDatabaseIntrospectionOptions,
|
||||
KloDaemonDatabaseHttpJsonRunner,
|
||||
KloDaemonDatabaseIntrospectionCommand,
|
||||
KloDaemonDatabaseJsonRunner,
|
||||
KtxDaemonDatabaseHttpJsonRunner,
|
||||
KtxDaemonDatabaseIntrospectionCommand,
|
||||
KtxDaemonDatabaseJsonRunner,
|
||||
} from './adapters/live-database/daemon-introspection.js';
|
||||
export { createDaemonLiveDatabaseIntrospection } from './adapters/live-database/daemon-introspection.js';
|
||||
export type {
|
||||
|
|
@ -15,7 +15,7 @@ export type {
|
|||
} from './adapters/live-database/extracted-schema.js';
|
||||
export {
|
||||
buildLiveDatabaseTableNaturalKey,
|
||||
kloSchemaSnapshotToExtractedSchema,
|
||||
ktxSchemaSnapshotToExtractedSchema,
|
||||
} from './adapters/live-database/extracted-schema.js';
|
||||
export { LiveDatabaseSourceAdapter } from './adapters/live-database/live-database.adapter.js';
|
||||
export type {
|
||||
|
|
@ -68,7 +68,7 @@ export {
|
|||
export {
|
||||
createDaemonLookerTableIdentifierParser,
|
||||
type DaemonLookerTableIdentifierParserOptions,
|
||||
type KloDaemonTableIdentifierHttpJsonRunner,
|
||||
type KtxDaemonTableIdentifierHttpJsonRunner,
|
||||
} from './adapters/looker/daemon-table-identifier-parser.js';
|
||||
export type {
|
||||
LookerConnectionClientFactory,
|
||||
|
|
@ -102,12 +102,12 @@ export {
|
|||
projectParsedIdentifier,
|
||||
refreshLookerMappingPlaceholders,
|
||||
sqlglotDialectForConnectionType,
|
||||
suggestKloConnectionForLookerConnection,
|
||||
suggestKtxConnectionForLookerConnection,
|
||||
validateLookerMappings,
|
||||
validateLookerWarehouseTarget,
|
||||
} from './adapters/looker/mapping.js';
|
||||
export type {
|
||||
LookerConnectionMapping as KloLookerConnectionMapping,
|
||||
LookerConnectionMapping as KtxLookerConnectionMapping,
|
||||
LookerMappingCandidateConnection,
|
||||
LookerMappingClient,
|
||||
LookerMappingDrift,
|
||||
|
|
@ -220,7 +220,7 @@ export type {
|
|||
AutoMatchCandidate,
|
||||
AutoMatchResult as MetabaseAutoMatchResult,
|
||||
DiscoveredMetabaseDatabase,
|
||||
KloConnectionPhysicalInfo,
|
||||
KtxConnectionPhysicalInfo,
|
||||
MappingPhysicalInfo,
|
||||
MappingRefreshReport,
|
||||
MetabaseMappedConnectionType,
|
||||
|
|
@ -347,7 +347,7 @@ export type {
|
|||
HistoricSqlSourceAdapterDeps,
|
||||
HistoricSqlTimeWindow,
|
||||
HistoricSqlUsage,
|
||||
KloPostgresQueryClient,
|
||||
KtxPostgresQueryClient,
|
||||
PostgresPgssAggregateRow,
|
||||
PostgresPgssProbeResult,
|
||||
PostgresPgssReader,
|
||||
|
|
@ -424,7 +424,7 @@ export type {
|
|||
RunLocalMetabaseIngestOptions,
|
||||
} from './local-ingest.js';
|
||||
export { getLatestLocalIngestStatus, getLocalIngestStatus, runLocalIngest, runLocalMetabaseIngest } from './local-ingest.js';
|
||||
export { seedLocalMappingStateFromKloYaml } from './local-mapping-reconcile.js';
|
||||
export { seedLocalMappingStateFromKtxYaml } from './local-mapping-reconcile.js';
|
||||
export type {
|
||||
CreateLocalBundleIngestRuntimeOptions,
|
||||
LocalBundleIngestRuntime,
|
||||
|
|
@ -475,7 +475,7 @@ export type {
|
|||
DbtSchemaFile,
|
||||
DbtSchemaParseResult,
|
||||
} from './adapters/dbt-descriptions/parse-schema.js';
|
||||
export { findMatchingKloTable, matchDbtTables } from './adapters/dbt-descriptions/match-tables.js';
|
||||
export { findMatchingKtxTable, matchDbtTables } from './adapters/dbt-descriptions/match-tables.js';
|
||||
export type { DbtHostTableLite, DbtTableMatch } from './adapters/dbt-descriptions/match-tables.js';
|
||||
export { toDescriptionUpdates } from './adapters/dbt-descriptions/to-description-updates.js';
|
||||
export type { DbtDescriptionUpdates } from './adapters/dbt-descriptions/to-description-updates.js';
|
||||
|
|
@ -483,7 +483,7 @@ export { toRelationshipUpdates } from './adapters/dbt-descriptions/to-relationsh
|
|||
export type { DbtRelationshipUpdates } from './adapters/dbt-descriptions/to-relationship-updates.js';
|
||||
export { toMetadataUpdates } from './adapters/dbt-descriptions/to-metadata-updates.js';
|
||||
export { mergeSemanticModelTables } from './adapters/dbt-descriptions/merge-semantic-model-tables.js';
|
||||
export type { KloJoinUpdate, KloMetadataUpdate } from '../scan/enrichment-types.js';
|
||||
export type { KtxJoinUpdate, KtxMetadataUpdate } from '../scan/enrichment-types.js';
|
||||
export {
|
||||
createInitialMemoryFlowInteractionState,
|
||||
findMemoryFlowSearchMatches,
|
||||
|
|
|
|||
|
|
@ -243,11 +243,11 @@ const buildRunner = (deps: ReturnType<typeof makeDeps> = makeDeps(), overrides:
|
|||
gitService: deps.gitService as any,
|
||||
lockingService: deps.lockingService as any,
|
||||
storage: {
|
||||
homeDir: '/tmp/klo-test',
|
||||
systemGitAuthor: { name: 'KLO Test', email: 'system@klo.local' },
|
||||
resolveUploadDir: (uploadId) => `/tmp/klo-test/ingest-uploads/${uploadId}`,
|
||||
resolvePullDir: (jobId) => `/tmp/klo-test/ingest-pulls/${jobId}`,
|
||||
resolveTranscriptDir: (jobId) => `/tmp/klo-test/run/wu-transcripts/${jobId}`,
|
||||
homeDir: '/tmp/ktx-test',
|
||||
systemGitAuthor: { name: 'KTX Test', email: 'system@ktx.local' },
|
||||
resolveUploadDir: (uploadId) => `/tmp/ktx-test/ingest-uploads/${uploadId}`,
|
||||
resolvePullDir: (jobId) => `/tmp/ktx-test/ingest-pulls/${jobId}`,
|
||||
resolveTranscriptDir: (jobId) => `/tmp/ktx-test/run/wu-transcripts/${jobId}`,
|
||||
},
|
||||
settings: { probeRowCount: 1, memoryIngestionModel: 'test-model' },
|
||||
skillsRegistry: deps.skillsRegistry as any,
|
||||
|
|
@ -845,7 +845,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|||
toolTranscripts: [
|
||||
{
|
||||
unitKey: 'u1',
|
||||
path: '/tmp/klo-test/run/wu-transcripts/j1/u1.jsonl',
|
||||
path: '/tmp/ktx-test/run/wu-transcripts/j1/u1.jsonl',
|
||||
toolCallCount: 2,
|
||||
errorCount: 0,
|
||||
toolNames: ['read_raw_span', 'wiki_write'],
|
||||
|
|
@ -1065,7 +1065,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|||
});
|
||||
|
||||
it('runs manual override reconciliation from the prior report snapshot and marks the prior report superseded', async () => {
|
||||
const tempRoot = await mkdtemp(join(tmpdir(), 'klo-override-'));
|
||||
const tempRoot = await mkdtemp(join(tmpdir(), 'ktx-override-'));
|
||||
const deps = makeDeps();
|
||||
deps.reportsRepo.findByJobId.mockResolvedValue({
|
||||
id: 'report-old',
|
||||
|
|
@ -1134,7 +1134,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|||
...(buildRunner(deps) as any).deps,
|
||||
storage: {
|
||||
homeDir: tempRoot,
|
||||
systemGitAuthor: { name: 'KLO Test', email: 'system@klo.local' },
|
||||
systemGitAuthor: { name: 'KTX Test', email: 'system@ktx.local' },
|
||||
resolveUploadDir: (uploadId: string) => join(tempRoot, 'ingest-uploads', uploadId),
|
||||
resolvePullDir: (jobId: string) => join(tempRoot, 'ingest-pulls', jobId),
|
||||
resolveTranscriptDir: (jobId: string) => join(tempRoot, 'run', 'wu-transcripts', jobId),
|
||||
|
|
@ -1770,8 +1770,8 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|||
await currentToolSession.gitService.commitFiles(
|
||||
['semantic-layer/c1/good.yaml'],
|
||||
'test: add good source',
|
||||
'KLO Test',
|
||||
'system@klo.local',
|
||||
'KTX Test',
|
||||
'system@ktx.local',
|
||||
);
|
||||
}
|
||||
if (unitKey === 'wu-bad') {
|
||||
|
|
@ -1782,8 +1782,8 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|||
await currentToolSession.gitService.commitFiles(
|
||||
['semantic-layer/c1/bad.yaml'],
|
||||
'test: add bad source',
|
||||
'KLO Test',
|
||||
'system@klo.local',
|
||||
'KTX Test',
|
||||
'system@ktx.local',
|
||||
);
|
||||
}
|
||||
return { stopReason: 'natural' };
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { dirname, join } from 'node:path';
|
|||
import { type Tool, tool } from 'ai';
|
||||
import pLimit from 'p-limit';
|
||||
import { z } from 'zod';
|
||||
import { type KloLogger, noopLogger } from '../core/index.js';
|
||||
import { type KtxLogger, noopLogger } from '../core/index.js';
|
||||
import type { CaptureSession, MemoryAction } from '../memory/index.js';
|
||||
import type { SlValidationDeps } from '../sl/index.js';
|
||||
import { createTouchedSlSources, type ToolContext, type ToolSession } from '../tools/index.js';
|
||||
|
|
@ -88,7 +88,7 @@ function reportIdFromCreateResult(result: unknown): string | undefined {
|
|||
}
|
||||
|
||||
export class IngestBundleRunner {
|
||||
private readonly logger: KloLogger;
|
||||
private readonly logger: KtxLogger;
|
||||
private readonly chainByConnection = new Map<string, Promise<unknown>>();
|
||||
|
||||
constructor(private readonly deps: IngestBundleRunnerDeps) {
|
||||
|
|
|
|||
|
|
@ -18,14 +18,14 @@ describe('ingest prompt assets', () => {
|
|||
expect(prompt).toContain('Do not create a duplicate contested artifact');
|
||||
});
|
||||
|
||||
it('uses product-neutral KLO runtime wording', async () => {
|
||||
it('uses product-neutral KTX runtime wording', async () => {
|
||||
const prompt = await readFile(
|
||||
new URL('../../prompts/memory_agent_bundle_ingest_work_unit.md', import.meta.url),
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
expect(prompt).toContain('KLO semantic-layer sources and/or knowledge wiki pages');
|
||||
expect(prompt).toContain('maps cleanly to KLO');
|
||||
expect(prompt).toContain('KTX semantic-layer sources and/or knowledge wiki pages');
|
||||
expect(prompt).toContain('maps cleanly to KTX');
|
||||
expect(prompt).not.toMatch(forbiddenProductPattern());
|
||||
});
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue