mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-19 11:41:02 +02:00
refactor(sl): split overlay columns from column_overrides and enforce TS/Python wire contract
Overlay sources now have two distinct collections: `columns:` for computed columns (requiring `expr` + `type`) and `column_overrides:` for metadata patches to inherited manifest columns. Composing or loading an overlay that mixes the two — or references an unknown column — fails with a typed error. Introduce `ResolvedSemanticLayerSource` / `resolvedSourceSchema` / `toResolvedWire` as the strict shape sent to the Python engine, and add a schema contract test that diffs Zod against the Pydantic JSON schema dumped by `python -m semantic_layer dump-schema`. `SourceDefinition` is now `extra="forbid"` on the Python side. `loadAllSources` surfaces per-file load errors instead of swallowing them, so validation/query paths can report manifest shard parse failures.
This commit is contained in:
parent
3e12a9fef4
commit
f561bfa850
42 changed files with 847 additions and 193 deletions
|
|
@ -213,7 +213,11 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx
|
||||||
if (!source) {
|
if (!source) {
|
||||||
throw new Error(`Semantic-layer source "${args.connectionId}/${args.sourceName}" was not found`);
|
throw new Error(`Semantic-layer source "${args.connectionId}/${args.sourceName}" was not found`);
|
||||||
}
|
}
|
||||||
const result = await validateLocalSlSource(source.yaml, { project, connectionId: args.connectionId });
|
const result = await validateLocalSlSource(source.yaml, {
|
||||||
|
project,
|
||||||
|
connectionId: args.connectionId,
|
||||||
|
sourceName: args.sourceName,
|
||||||
|
});
|
||||||
if (!result.valid) {
|
if (!result.valid) {
|
||||||
for (const error of result.errors) {
|
for (const error of result.errors) {
|
||||||
io.stderr.write(`${error}\n`);
|
io.stderr.write(`${error}\n`);
|
||||||
|
|
|
||||||
|
|
@ -153,6 +153,7 @@
|
||||||
"@types/node": "^25.7.0",
|
"@types/node": "^25.7.0",
|
||||||
"@types/pg": "^8.20.0",
|
"@types/pg": "^8.20.0",
|
||||||
"@vitest/coverage-v8": "^4.1.6",
|
"@vitest/coverage-v8": "^4.1.6",
|
||||||
|
"ajv": "8.20.0",
|
||||||
"typescript": "^6.0.3",
|
"typescript": "^6.0.3",
|
||||||
"vitest": "^4.1.6"
|
"vitest": "^4.1.6"
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -14,14 +14,14 @@ Use this skill for **uploaded** dbt projects (`dbt_project.yml` at stage root, `
|
||||||
|-----|--------|--------|
|
|-----|--------|--------|
|
||||||
| `models:` entry with `columns:` | **Overlay** on the manifest table with the same name (after `discover_data` / `entity_details`) | One SL source per physical table; model name may differ from DB name — resolve with `read_raw_file` + warehouse context. |
|
| `models:` entry with `columns:` | **Overlay** on the manifest table with the same name (after `discover_data` / `entity_details`) | One SL source per physical table; model name may differ from DB name — resolve with `read_raw_file` + warehouse context. |
|
||||||
| `sources:` → `tables:` | Same as models; use `identifier` when present instead of logical `name`. | Schema + name must match how the connection sees tables. |
|
| `sources:` → `tables:` | Same as models; use `identifier` when present instead of logical `name`. | Schema + name must match how the connection sees tables. |
|
||||||
| Column `description` | `descriptions.user` or merged `descriptions` map on the column | Do not overwrite `dbt` description keys from sync. |
|
| Column `description` | `column_overrides[].descriptions.user` on the overlay | Do not overwrite `dbt` description keys from sync. |
|
||||||
| `data_tests: not_null` / `unique` | Short hint in column `descriptions` or notes: “dbt: not null”, “dbt: unique” | Full structured metadata lands in manifest via **sync**; the skill keeps bundle-time SL text useful for the agent. |
|
| `data_tests: not_null` / `unique` | Short hint in column `descriptions` or notes: “dbt: not null”, “dbt: unique” | Full structured metadata lands in manifest via **sync**; the skill keeps bundle-time SL text useful for the agent. |
|
||||||
| `accepted_values` | Add a **brief** line in the column description: allowed values (truncate long lists) | Also mention enum-like use in `discover_data` / filters. |
|
| `accepted_values` | Add a **brief** line in the column description: allowed values (truncate long lists) | Also mention enum-like use in `discover_data` / filters. |
|
||||||
| `relationships` | Add or confirm `joins:` on the overlay **only** when `to` resolves to a real table via `read_raw_file` + `discover_data` / `entity_details` | If the ref cannot be resolved, capture the intent in a wiki page instead. |
|
| `relationships` | Add or confirm `joins:` on the overlay **only** when `to` resolves to a real table via `read_raw_file` + `discover_data` / `entity_details` | If the ref cannot be resolved, capture the intent in a wiki page instead. |
|
||||||
|
|
||||||
## Physical schema grounding
|
## Physical schema grounding
|
||||||
|
|
||||||
dbt YAML is documentation and test metadata; it is not permission to invent physical columns. Before writing any table-backed SL source, confirm the real warehouse shape with `discover_data`, `sl_discover`, or `entity_details` and use only confirmed column names in `columns:`, `grain:`, `joins:`, `segments:`, and `measures[].expr`.
|
dbt YAML is documentation and test metadata; it is not permission to invent physical columns. Before writing any table-backed SL source, confirm the real warehouse shape with `discover_data`, `sl_discover`, or `entity_details` and use only confirmed column names in `column_overrides:`, computed-only `columns:`, `grain:`, `joins:`, `segments:`, and `measures[].expr`.
|
||||||
|
|
||||||
For dbt context-source ingest, the dbt connection is usually not the warehouse connection. Call `sl_discover` without `connectionId` first, then write overlays to the connection that owns the matching manifest-backed source (for example `postgres-warehouse`), not to the dbt connection (for example `dbt-main`). If no matching manifest-backed source is visible on any warehouse connection, do not call `sl_write_source`; record `emit_unmapped_fallback` and keep the fact wiki-only.
|
For dbt context-source ingest, the dbt connection is usually not the warehouse connection. Call `sl_discover` without `connectionId` first, then write overlays to the connection that owns the matching manifest-backed source (for example `postgres-warehouse`), not to the dbt connection (for example `dbt-main`). If no matching manifest-backed source is visible on any warehouse connection, do not call `sl_write_source`; record `emit_unmapped_fallback` and keep the fact wiki-only.
|
||||||
|
|
||||||
|
|
@ -61,7 +61,7 @@ SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
|
||||||
|
|
||||||
## 1.1 test hints (descriptions / meta)
|
## 1.1 test hints (descriptions / meta)
|
||||||
|
|
||||||
When YAML shows `accepted_values` or `not_null`, add **short** hints into `columns[].descriptions` (e.g. under `user`) or freeform column notes so chat and validation see intent before the next git sync refreshes `constraints` / `enum_values` in `_schema`. Keep hints under a few words when possible.
|
When YAML shows `accepted_values` or `not_null`, add **short** hints into `column_overrides[].descriptions` (for example under `user`) or freeform column notes so chat and validation see intent before the next git sync refreshes `constraints` / `enum_values` in `_schema`. Keep hints under a few words when possible.
|
||||||
|
|
||||||
## Overlap with MetricFlow
|
## Overlap with MetricFlow
|
||||||
|
|
||||||
|
|
@ -71,6 +71,6 @@ If the same bundle also has MetricFlow `semantic_models:` / `metrics:`, the **`m
|
||||||
|
|
||||||
- Do not run `dbt` CLI or assume `target/` / `manifest.json` exists in the upload.
|
- Do not run `dbt` CLI or assume `target/` / `manifest.json` exists in the upload.
|
||||||
- Do not invent column names, grain keys, or measure expressions from dbt model names, descriptions, tests, or common naming patterns.
|
- Do not invent column names, grain keys, or measure expressions from dbt model names, descriptions, tests, or common naming patterns.
|
||||||
- Do not write `columns:`, `grain:`, or `measures:` for a dbt model unless those exact column names are confirmed by dbt YAML columns or warehouse schema discovery.
|
- Do not write computed `columns:`, `column_overrides:`, `grain:`, or `measures:` for a dbt model unless those exact column names are confirmed by dbt YAML columns or warehouse schema discovery.
|
||||||
- Do not invent joins from `relationships` tests if the target model/table is not found in SL or the warehouse.
|
- Do not invent joins from `relationships` tests if the target model/table is not found in SL or the warehouse.
|
||||||
- Do not read `peerFileIndex` paths — use `read_raw_file` only on `rawFiles` and `dependencyPaths` from the WorkUnit.
|
- Do not read `peerFileIndex` paths — use `read_raw_file` only on `rawFiles` and `dependencyPaths` from the WorkUnit.
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ LookML views map to SL sources, `measure:` to measures, `explore: { join: }` to
|
||||||
|
|
||||||
| LookML | KTX form | Notes |
|
| LookML | KTX form | Notes |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `view: X { sql_table_name: …; measure:/dimension:/join: }` | **Overlay** at `<connId>/X.yaml` with `measures`, `columns` (computed), `joins`, `segments` | Manifest-backed; inherit grain/columns |
|
| `view: X { sql_table_name: …; measure:/dimension:/join: }` | **Overlay** at `<connId>/X.yaml` with `measures`, computed-only `columns`, `column_overrides`, `joins`, `segments` | Manifest-backed; inherit grain/columns |
|
||||||
| `view: X { derived_table: { sql: … } }` | **Standalone** with top-level `sql:`, explicit `grain:` + `columns:` | No manifest entry exists |
|
| `view: X { derived_table: { sql: … } }` | **Standalone** with top-level `sql:`, explicit `grain:` + `columns:` | No manifest entry exists |
|
||||||
| `view: X { sql_always_where: <p> }` | **Standalone** with `sql: SELECT * FROM <base> WHERE <p>` | Enforcement, not opt-in |
|
| `view: X { sql_always_where: <p> }` | **Standalone** with `sql: SELECT * FROM <base> WHERE <p>` | Enforcement, not opt-in |
|
||||||
| `explore: { join: Y { sql_on: …; relationship: … } }` | `joins:` entry `{ to: Y, on: "<local> = Y.<col>", relationship: … }` | On the overlay or standalone |
|
| `explore: { join: Y { sql_on: …; relationship: … } }` | `joins:` entry `{ to: Y, on: "<local> = Y.<col>", relationship: … }` | On the overlay or standalone |
|
||||||
|
|
@ -136,7 +136,8 @@ KTX overlay at `<connId>/fct_labs.yaml`:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
name: fct_labs
|
name: fct_labs
|
||||||
description: "Lab-order fact table. One row per lab order event."
|
descriptions:
|
||||||
|
user: "Lab-order fact table. One row per lab order event."
|
||||||
columns:
|
columns:
|
||||||
- name: is_byol
|
- name: is_byol
|
||||||
type: boolean
|
type: boolean
|
||||||
|
|
|
||||||
|
|
@ -79,7 +79,7 @@ SL source, `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
|
||||||
|
|
||||||
For each card:
|
For each card:
|
||||||
1. Analyze `resolvedSql` + `resultMetadata`: identify base tables, aggregations, joins, filters, column types.
|
1. Analyze `resolvedSql` + `resultMetadata`: identify base tables, aggregations, joins, filters, column types.
|
||||||
2. **REQUIRED before any write**: call `sl_discover` for every candidate target source name. The response tells you whether the name is manifest-backed (`Type: table` or `Type: sql`). For manifest-backed names you MUST use the overlay shape (`name:` + `measures:`/`segments:`/`description:` only — no `sql:`, `table:`, `grain:`, or `columns:`); the tool will reject a standalone write and you'll have wasted the call. If `sl_discover` returns nothing for the name, you can write a standalone source. Also call `sl_read_source` on existing sources you intend to extend so you don't duplicate measures.
|
2. **REQUIRED before any write**: call `sl_discover` for every candidate target source name. The response tells you whether the name is manifest-backed (`Type: table` or `Type: sql`). For manifest-backed names you MUST use the overlay shape (`name:` plus overlay fields such as `measures:`, `segments:`, `descriptions:`, `joins:`, `disable_joins:`, `column_overrides:`, and computed-only `columns:` entries with `expr` + `type`; no `sql:`, `table:`, `grain:`, or base-table `columns:`); the tool will reject a standalone write and you'll have wasted the call. If `sl_discover` returns nothing for the name, you can write a standalone source. Also call `sl_read_source` on existing sources you intend to extend so you don't duplicate measures.
|
||||||
3. Include `rawPaths: ["cards/<id>.json"]` on every `sl_write_source`, `sl_edit_source`, and `wiki_write` call. If one artifact generalizes multiple near-duplicate cards, include each contributing card path and no unrelated cards.
|
3. Include `rawPaths: ["cards/<id>.json"]` on every `sl_write_source`, `sl_edit_source`, and `wiki_write` call. If one artifact generalizes multiple near-duplicate cards, include each contributing card path and no unrelated cards.
|
||||||
4. Decide:
|
4. Decide:
|
||||||
- Simple aggregation on a table that already has a source → `sl_edit_source` to add a measure.
|
- Simple aggregation on a table that already has a source → `sl_edit_source` to add a measure.
|
||||||
|
|
@ -98,7 +98,7 @@ measures:
|
||||||
expr: "<expression>"
|
expr: "<expression>"
|
||||||
```
|
```
|
||||||
|
|
||||||
Overlay shape: `name:` plus any of `measures:`, `segments:`, `descriptions:`, `joins:`, `disable_joins:`. Never include `sql:`, `table:`, `grain:`, or `columns:` on a manifest-backed name — those would shadow the manifest's schema and drop its joins. Overlay `joins:` are merged additively with the manifest's joins (deduped by `to` + `on`); use `disable_joins: ["<on-clause>"]` to suppress a specific manifest join. After the overlay exists, use `sl_edit_source` for further tweaks. See `sl_capture` skill for the canonical overlay rule.
|
Overlay shape: `name:` plus any of `measures:`, `segments:`, `descriptions:`, `joins:`, `disable_joins:`, `exclude_columns:`, `column_overrides:`, or computed-only `columns:` entries with `expr` + `type`. Never include `sql:`, `table:`, `grain:`, or base-table `columns:` on a manifest-backed name — those would shadow the manifest's schema and drop its joins. Use `column_overrides:` for inherited column descriptions. Overlay `joins:` are merged additively with the manifest's joins (deduped by `to` + `on`); use `disable_joins: ["<on-clause>"]` to suppress a specific manifest join. After the overlay exists, use `sl_edit_source` for further tweaks. See `sl_capture` skill for the canonical overlay rule.
|
||||||
|
|
||||||
**Join discovery:** When your card's SQL references warehouse tables (e.g. in `FROM` or `JOIN` clauses), call `sl_discover({ query: '<table>' })` before writing. The matching manifest entry's `name` is the value you use in `joins: [- to: <name>]` only when the card output exposes a local key that matches the target source grain (for example `account_id = mart_account_segments.account_id`). Do not declare a KTX join just because the card SQL joins that table internally. If the output only exposes display fields such as `account_name`, keep the SQL source self-contained or project the key before adding the join. Use `many_to_one` for FK-to-dimension joins, `one_to_many` for the reverse.
|
**Join discovery:** When your card's SQL references warehouse tables (e.g. in `FROM` or `JOIN` clauses), call `sl_discover({ query: '<table>' })` before writing. The matching manifest entry's `name` is the value you use in `joins: [- to: <name>]` only when the card output exposes a local key that matches the target source grain (for example `account_id = mart_account_segments.account_id`). Do not declare a KTX join just because the card SQL joins that table internally. If the output only exposes display fields such as `account_name`, keep the SQL source self-contained or project the key before adding the join. Use `many_to_one` for FK-to-dimension joins, `one_to_many` for the reverse.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ A MetricFlow `semantic_model` maps to an SL source; MetricFlow `measures` map to
|
||||||
|
|
||||||
| MetricFlow | KTX form | Notes |
|
| MetricFlow | KTX form | Notes |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `semantic_model: X { model: ref('t') }` with measures + dimensions | **Overlay** at `<connId>/X.yaml` with `measures`, `columns` (computed), `joins` | The `model:` ref resolves to a manifest table. |
|
| `semantic_model: X { model: ref('t') }` with measures + dimensions | **Overlay** at `<connId>/X.yaml` with `measures`, computed-only `columns`, `column_overrides`, `joins` | The `model:` ref resolves to a manifest table. |
|
||||||
| `semantic_model: X { model: source('s','t') }` | **Overlay** at `<connId>/X.yaml` over table `t`. | Same shape; `source()` still resolves to a physical table. |
|
| `semantic_model: X { model: source('s','t') }` | **Overlay** at `<connId>/X.yaml` over table `t`. | Same shape; `source()` still resolves to a physical table. |
|
||||||
| `semantic_model: X { model: <literal> }` with no manifest entry | **Standalone** with explicit `sql:`, `grain:`, `columns:` | Happens when the dbt manifest isn't available. |
|
| `semantic_model: X { model: <literal> }` with no manifest entry | **Standalone** with explicit `sql:`, `grain:`, `columns:` | Happens when the dbt manifest isn't available. |
|
||||||
| `semantic_model: Y { extends: X }` | **Merge** Y's measures/dimensions/entities into X's overlay, or write a single overlay named for the most-derived child (Y) containing both X's and Y's primitives | Do not emit a second overlay for X — flatten. |
|
| `semantic_model: Y { extends: X }` | **Merge** Y's measures/dimensions/entities into X's overlay, or write a single overlay named for the most-derived child (Y) containing both X's and Y's primitives | Do not emit a second overlay for X — flatten. |
|
||||||
|
|
@ -84,7 +84,7 @@ If `sl_discover` errors because no such table exists, use `discover_data` and
|
||||||
`entity_details` to find the warehouse target. If a SQL probe is still needed,
|
`entity_details` to find the warehouse target. If a SQL probe is still needed,
|
||||||
call `sql_execution` with the same warehouse connection name, for example:
|
call `sql_execution` with the same warehouse connection name, for example:
|
||||||
`sql_execution({connectionName: "warehouse", sql: "SELECT 1 FROM analytics.orders LIMIT 0"})`.
|
`sql_execution({connectionName: "warehouse", sql: "SELECT 1 FROM analytics.orders LIMIT 0"})`.
|
||||||
**Never invent column names** - every column in `columns:`, `grain:`, and
|
**Never invent column names** - every column in computed `columns:`, `column_overrides:`, `grain:`, and
|
||||||
`sql:` must be sourced from raw files, `entity_details`, or a successful SQL
|
`sql:` must be sourced from raw files, `entity_details`, or a successful SQL
|
||||||
probe.
|
probe.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -39,6 +39,10 @@ columns: # computed dimensions only
|
||||||
- name: is_large_order
|
- name: is_large_order
|
||||||
type: boolean
|
type: boolean
|
||||||
expr: "amount > 1000"
|
expr: "amount > 1000"
|
||||||
|
column_overrides: # metadata patches for inherited columns
|
||||||
|
- name: status
|
||||||
|
descriptions:
|
||||||
|
user: "Order lifecycle status."
|
||||||
segments:
|
segments:
|
||||||
- name: paid_non_refunded
|
- name: paid_non_refunded
|
||||||
expr: "is_paid = true AND is_refunded = false"
|
expr: "is_paid = true AND is_refunded = false"
|
||||||
|
|
@ -51,6 +55,7 @@ joins:
|
||||||
Rules:
|
Rules:
|
||||||
- Do **not** repeat base-table columns, grain, `table`, or `source_type` in an overlay — those are inherited.
|
- Do **not** repeat base-table columns, grain, `table`, or `source_type` in an overlay — those are inherited.
|
||||||
- Overlay columns MUST be computed (`expr` + `type`).
|
- Overlay columns MUST be computed (`expr` + `type`).
|
||||||
|
- Use `column_overrides` to add descriptions or metadata to inherited manifest columns. Do not put `type` or `expr` in `column_overrides`.
|
||||||
- `exclude_columns` hides specific manifest columns; `disable_joins` suppresses specific auto-detected joins.
|
- `exclude_columns` hides specific manifest columns; `disable_joins` suppresses specific auto-detected joins.
|
||||||
|
|
||||||
### Standalone table sources
|
### Standalone table sources
|
||||||
|
|
@ -110,7 +115,7 @@ An SQL source is a one-shot answer: the aggregation is frozen, callers cannot re
|
||||||
|
|
||||||
### Columns
|
### Columns
|
||||||
|
|
||||||
Every standalone column requires `name` and `type`. Overlays have computed columns only.
|
Every standalone column requires `name` and `type`. Overlays have computed columns in `columns:` and manifest column metadata patches in `column_overrides:`.
|
||||||
|
|
||||||
- `type`: one of `string`, `number`, `boolean`, `time`. Map LookML `date`/`datetime`/`timestamp` → `time`. Map LookML `yesno` → `boolean`.
|
- `type`: one of `string`, `number`, `boolean`, `time`. Map LookML `date`/`datetime`/`timestamp` → `time`. Map LookML `yesno` → `boolean`.
|
||||||
- `role` (optional): `time` enables time-granularity queries (month, week, day). `default` is the implicit fallback.
|
- `role` (optional): `time` enables time-granularity queries (month, week, day). `default` is the implicit fallback.
|
||||||
|
|
|
||||||
|
|
@ -100,7 +100,33 @@ measures:
|
||||||
|
|
||||||
**Extract repeated filter bundles into named segments.** If the same predicate appears on multiple measures of the same source, lift it to a `segments[]` entry and have each measure reference it. One edit updates every measure that depends on it.
|
**Extract repeated filter bundles into named segments.** If the same predicate appears on multiple measures of the same source, lift it to a `segments[]` entry and have each measure reference it. One edit updates every measure that depends on it.
|
||||||
|
|
||||||
**Never write a standalone file on a manifest-backed name.** If `sl_discover({ query: "<table-or-source-name>" })` finds an existing schema for that name, you MUST write an overlay (`name:` + `measures:`/`segments:`/`descriptions:` only — no `sql:`, `table:`, `grain:`, `columns:`, `joins:`). A standalone with `sql:` or `table:` on a manifest-backed name clobbers the inherited columns and joins; `sl_write_source` and `sl_validate` both reject this shape with a clear fix hint. Always run `sl_discover` before your first write on any existing name.
|
**Never write a standalone file on a manifest-backed name.** If `sl_discover({ query: "<table-or-source-name>" })` finds an existing schema for that name, you MUST write an overlay. A standalone with `sql:` or `table:` on a manifest-backed name clobbers the inherited columns and joins; `sl_write_source` and `sl_validate` both reject this shape with a clear fix hint. Always run `sl_discover` before your first write on any existing name.
|
||||||
|
|
||||||
|
Overlay before/after examples:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# Wrong: patches an inherited manifest column through columns:
|
||||||
|
name: fct_orders
|
||||||
|
columns:
|
||||||
|
- name: status
|
||||||
|
descriptions:
|
||||||
|
user: "Order lifecycle status."
|
||||||
|
```
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# Right: patch inherited columns with column_overrides:
|
||||||
|
name: fct_orders
|
||||||
|
column_overrides:
|
||||||
|
- name: status
|
||||||
|
descriptions:
|
||||||
|
user: "Order lifecycle status."
|
||||||
|
columns:
|
||||||
|
- name: is_large_order
|
||||||
|
type: boolean
|
||||||
|
expr: "amount > 1000"
|
||||||
|
```
|
||||||
|
|
||||||
|
Overlay YAML may include `measures:`, `segments:`, `descriptions:`, `joins:`, `disable_joins:`, `exclude_columns:`, `column_overrides:`, and computed-only `columns:` entries with `expr` and `type`. Do not include `sql:`, `table:`, `grain:`, or base-table `columns:`.
|
||||||
|
|
||||||
**Prefer overlay decomposition over standalone SQL sources.** Before reaching for `source_type: sql`, check whether the metric decomposes into measures on existing overlays (including cross-source derived measures). Use `source_type: sql` only when:
|
**Prefer overlay decomposition over standalone SQL sources.** Before reaching for `source_type: sql`, check whether the metric decomposes into measures on existing overlays (including cross-source derived measures). Use `source_type: sql` only when:
|
||||||
- The metric requires per-user/per-entity derivation that cannot be expressed as a single `expr` (e.g., `EXISTS` over a time-windowed subset), OR
|
- The metric requires per-user/per-entity derivation that cannot be expressed as a single `expr` (e.g., `EXISTS` over a time-windowed subset), OR
|
||||||
|
|
|
||||||
|
|
@ -38,7 +38,7 @@ describe('importMetricflowSemanticModels', () => {
|
||||||
const scoped = {
|
const scoped = {
|
||||||
getManifestEntry: vi.fn().mockResolvedValue(null),
|
getManifestEntry: vi.fn().mockResolvedValue(null),
|
||||||
isManifestBacked: vi.fn().mockResolvedValue(false),
|
isManifestBacked: vi.fn().mockResolvedValue(false),
|
||||||
loadAllSources: vi.fn().mockResolvedValue([]),
|
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
|
||||||
loadSource: vi.fn().mockResolvedValue(null),
|
loadSource: vi.fn().mockResolvedValue(null),
|
||||||
writeSource: vi.fn().mockResolvedValue({ warnings: [] }),
|
writeSource: vi.fn().mockResolvedValue({ warnings: [] }),
|
||||||
};
|
};
|
||||||
|
|
@ -104,7 +104,7 @@ describe('importMetricflowSemanticModels', () => {
|
||||||
const scoped = {
|
const scoped = {
|
||||||
getManifestEntry: vi.fn().mockResolvedValue(null),
|
getManifestEntry: vi.fn().mockResolvedValue(null),
|
||||||
isManifestBacked: vi.fn().mockResolvedValue(false),
|
isManifestBacked: vi.fn().mockResolvedValue(false),
|
||||||
loadAllSources: vi.fn().mockResolvedValue([]),
|
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
|
||||||
loadSource: vi.fn().mockImplementation((connectionId: string, sourceName: string) =>
|
loadSource: vi.fn().mockImplementation((connectionId: string, sourceName: string) =>
|
||||||
Promise.resolve(sourceName === 'orders' ? { name: 'orders' } : null),
|
Promise.resolve(sourceName === 'orders' ? { name: 'orders' } : null),
|
||||||
),
|
),
|
||||||
|
|
@ -139,7 +139,7 @@ describe('importMetricflowSemanticModels', () => {
|
||||||
const scoped = {
|
const scoped = {
|
||||||
getManifestEntry: vi.fn().mockResolvedValue(null),
|
getManifestEntry: vi.fn().mockResolvedValue(null),
|
||||||
isManifestBacked: vi.fn().mockResolvedValue(false),
|
isManifestBacked: vi.fn().mockResolvedValue(false),
|
||||||
loadAllSources: vi.fn().mockResolvedValue([]),
|
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
|
||||||
loadSource: vi.fn().mockResolvedValue(null),
|
loadSource: vi.fn().mockResolvedValue(null),
|
||||||
writeSource: vi.fn().mockRejectedValueOnce(new Error('cannot write orders')).mockResolvedValue({ warnings: [] }),
|
writeSource: vi.fn().mockRejectedValueOnce(new Error('cannot write orders')).mockResolvedValue({ warnings: [] }),
|
||||||
};
|
};
|
||||||
|
|
@ -190,7 +190,7 @@ describe('importMetricflowSemanticModels', () => {
|
||||||
isManifestBacked: vi.fn().mockImplementation(async (_connectionId: string, sourceName: string) => {
|
isManifestBacked: vi.fn().mockImplementation(async (_connectionId: string, sourceName: string) => {
|
||||||
return sourceName === 'orders';
|
return sourceName === 'orders';
|
||||||
}),
|
}),
|
||||||
loadAllSources: vi.fn().mockResolvedValue([]),
|
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
|
||||||
loadSource: vi.fn().mockResolvedValue(null),
|
loadSource: vi.fn().mockResolvedValue(null),
|
||||||
writeSource: vi.fn().mockImplementation(async (_connectionId: string, source: (typeof written)[number]) => {
|
writeSource: vi.fn().mockImplementation(async (_connectionId: string, source: (typeof written)[number]) => {
|
||||||
written.push(source);
|
written.push(source);
|
||||||
|
|
@ -268,7 +268,7 @@ describe('importMetricflowSemanticModels', () => {
|
||||||
isManifestBacked: vi.fn().mockImplementation(async (_connectionId: string, sourceName: string) => {
|
isManifestBacked: vi.fn().mockImplementation(async (_connectionId: string, sourceName: string) => {
|
||||||
return sourceName === 'orders';
|
return sourceName === 'orders';
|
||||||
}),
|
}),
|
||||||
loadAllSources: vi.fn().mockResolvedValue([]),
|
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
|
||||||
loadSource: vi.fn().mockResolvedValue(null),
|
loadSource: vi.fn().mockResolvedValue(null),
|
||||||
writeSource: vi.fn().mockResolvedValue({ warnings: [] }),
|
writeSource: vi.fn().mockResolvedValue({ warnings: [] }),
|
||||||
};
|
};
|
||||||
|
|
@ -311,7 +311,7 @@ describe('importMetricflowSemanticModels', () => {
|
||||||
const scoped = {
|
const scoped = {
|
||||||
getManifestEntry: vi.fn().mockResolvedValue(null),
|
getManifestEntry: vi.fn().mockResolvedValue(null),
|
||||||
isManifestBacked: vi.fn().mockResolvedValue(false),
|
isManifestBacked: vi.fn().mockResolvedValue(false),
|
||||||
loadAllSources: vi.fn().mockResolvedValue([]),
|
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
|
||||||
loadSource: vi.fn().mockResolvedValue(null),
|
loadSource: vi.fn().mockResolvedValue(null),
|
||||||
writeSource: vi
|
writeSource: vi
|
||||||
.fn()
|
.fn()
|
||||||
|
|
|
||||||
|
|
@ -71,7 +71,7 @@ export async function importMetricflowSemanticModels(
|
||||||
let crossModelSourcesCreated = 0;
|
let crossModelSourcesCreated = 0;
|
||||||
|
|
||||||
const preexistingSourceNames = new Set(
|
const preexistingSourceNames = new Set(
|
||||||
(await semanticLayerService.loadAllSources(input.connectionId)).map((source) => source.name),
|
(await semanticLayerService.loadAllSources(input.connectionId)).sources.map((source) => source.name),
|
||||||
);
|
);
|
||||||
const modelContexts: MetricflowSemanticModelImportContext[] = [];
|
const modelContexts: MetricflowSemanticModelImportContext[] = [];
|
||||||
const sourceNameByModelRef = new Map<string, string>();
|
const sourceNameByModelRef = new Map<string, string>();
|
||||||
|
|
|
||||||
|
|
@ -187,7 +187,10 @@ const makeDeps = () => {
|
||||||
loadAllSources: vi
|
loadAllSources: vi
|
||||||
.fn()
|
.fn()
|
||||||
.mockImplementation((connectionId: string) =>
|
.mockImplementation((connectionId: string) =>
|
||||||
Promise.resolve(connectionId === 'warehouse-2' ? [{ name: 'looker__orders' }] : []),
|
Promise.resolve({
|
||||||
|
sources: connectionId === 'warehouse-2' ? [{ name: 'looker__orders' }] : [],
|
||||||
|
loadErrors: [],
|
||||||
|
}),
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
const slSearchService = {
|
const slSearchService = {
|
||||||
|
|
@ -1347,7 +1350,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
||||||
frontmatter: { sl_refs: ['looker__b2b__sales_pipeline.arr'] },
|
frontmatter: { sl_refs: ['looker__b2b__sales_pipeline.arr'] },
|
||||||
});
|
});
|
||||||
deps.semanticLayerService.loadAllSources.mockImplementation((connectionId: string) =>
|
deps.semanticLayerService.loadAllSources.mockImplementation((connectionId: string) =>
|
||||||
Promise.resolve([{ name: `${connectionId}_source` }]),
|
Promise.resolve({ sources: [{ name: `${connectionId}_source` }], loadErrors: [] }),
|
||||||
);
|
);
|
||||||
deps.agentRunner.runLoop.mockImplementation(async (params: any) => {
|
deps.agentRunner.runLoop.mockImplementation(async (params: any) => {
|
||||||
if (params.telemetryTags.operationName === 'ingest-bundle-wu') {
|
if (params.telemetryTags.operationName === 'ingest-bundle-wu') {
|
||||||
|
|
@ -1447,7 +1450,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
||||||
parseArtifacts: { semanticModels: [{ name: 'orders' }] },
|
parseArtifacts: { semanticModels: [{ name: 'orders' }] },
|
||||||
});
|
});
|
||||||
deps.semanticLayerService.loadAllSources.mockImplementation((connectionId: string) =>
|
deps.semanticLayerService.loadAllSources.mockImplementation((connectionId: string) =>
|
||||||
Promise.resolve([{ name: `${connectionId}_source` }]),
|
Promise.resolve({ sources: [{ name: `${connectionId}_source` }], loadErrors: [] }),
|
||||||
);
|
);
|
||||||
const postProcessor = {
|
const postProcessor = {
|
||||||
run: vi.fn().mockResolvedValue({
|
run: vi.fn().mockResolvedValue({
|
||||||
|
|
@ -1631,7 +1634,10 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
||||||
const deps = makeDeps();
|
const deps = makeDeps();
|
||||||
deps.adapter.listTargetConnectionIds = vi.fn().mockResolvedValue(['postgres-warehouse']);
|
deps.adapter.listTargetConnectionIds = vi.fn().mockResolvedValue(['postgres-warehouse']);
|
||||||
deps.semanticLayerService.loadAllSources.mockImplementation((connectionId: string) =>
|
deps.semanticLayerService.loadAllSources.mockImplementation((connectionId: string) =>
|
||||||
Promise.resolve(connectionId === 'postgres-warehouse' ? [{ name: 'stg_accounts' }] : []),
|
Promise.resolve({
|
||||||
|
sources: connectionId === 'postgres-warehouse' ? [{ name: 'stg_accounts' }] : [],
|
||||||
|
loadErrors: [],
|
||||||
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
const runner = buildRunner(deps);
|
const runner = buildRunner(deps);
|
||||||
|
|
@ -1659,7 +1665,10 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
||||||
|
|
||||||
it('does not resolve qualified fallback table refs by source name alone', async () => {
|
it('does not resolve qualified fallback table refs by source name alone', async () => {
|
||||||
const deps = makeDeps();
|
const deps = makeDeps();
|
||||||
deps.semanticLayerService.loadAllSources.mockResolvedValue([{ name: 'orders', table: 'sales.orders' }]);
|
deps.semanticLayerService.loadAllSources.mockResolvedValue({
|
||||||
|
sources: [{ name: 'orders', table: 'sales.orders' }],
|
||||||
|
loadErrors: [],
|
||||||
|
});
|
||||||
const runner = buildRunner(deps);
|
const runner = buildRunner(deps);
|
||||||
|
|
||||||
await expect(
|
await expect(
|
||||||
|
|
|
||||||
|
|
@ -300,7 +300,7 @@ export class IngestBundleRunner {
|
||||||
const blocks = await Promise.all(
|
const blocks = await Promise.all(
|
||||||
connectionIds.map(async (connectionId) => {
|
connectionIds.map(async (connectionId) => {
|
||||||
try {
|
try {
|
||||||
const sources = await this.deps.semanticLayerService.loadAllSources(connectionId);
|
const { sources } = await this.deps.semanticLayerService.loadAllSources(connectionId);
|
||||||
const names = sources.map((source) => source.name).sort((left, right) => left.localeCompare(right));
|
const names = sources.map((source) => source.name).sort((left, right) => left.localeCompare(right));
|
||||||
const body = names.length > 0 ? names.join('\n') : '(no sources yet)';
|
const body = names.length > 0 ? names.join('\n') : '(no sources yet)';
|
||||||
return `## ${connectionId}\n${body}`;
|
return `## ${connectionId}\n${body}`;
|
||||||
|
|
@ -329,7 +329,7 @@ export class IngestBundleRunner {
|
||||||
): Promise<boolean> {
|
): Promise<boolean> {
|
||||||
for (const connectionId of connectionIds) {
|
for (const connectionId of connectionIds) {
|
||||||
try {
|
try {
|
||||||
const sources = await semanticLayerService.loadAllSources(connectionId);
|
const { sources } = await semanticLayerService.loadAllSources(connectionId);
|
||||||
if (sources.some((source) => semanticSourceMatchesTableRef(source, tableRef))) {
|
if (sources.some((source) => semanticSourceMatchesTableRef(source, tableRef))) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
@ -1211,7 +1211,7 @@ export class IngestBundleRunner {
|
||||||
].sort();
|
].sort();
|
||||||
for (const connectionId of touchedConnections) {
|
for (const connectionId of touchedConnections) {
|
||||||
try {
|
try {
|
||||||
const allSources = await this.deps.semanticLayerService.loadAllSources(connectionId);
|
const { sources: allSources } = await this.deps.semanticLayerService.loadAllSources(connectionId);
|
||||||
await this.deps.slSearchService.indexSources(connectionId, allSources);
|
await this.deps.slSearchService.indexSources(connectionId, allSources);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
this.logger.warn(
|
this.logger.warn(
|
||||||
|
|
|
||||||
|
|
@ -44,23 +44,26 @@ describe('repairWikiSlRefs', () => {
|
||||||
})),
|
})),
|
||||||
};
|
};
|
||||||
const semanticLayerService = {
|
const semanticLayerService = {
|
||||||
loadAllSources: vi.fn(async () => [
|
loadAllSources: vi.fn(async () => ({
|
||||||
{
|
sources: [
|
||||||
name: 'mart_customer_health',
|
{
|
||||||
grain: [],
|
name: 'mart_customer_health',
|
||||||
columns: [],
|
grain: [],
|
||||||
joins: [],
|
columns: [],
|
||||||
measures: [{ name: 'high_risk_account_count', expr: 'count(*)' }],
|
joins: [],
|
||||||
segments: [{ name: 'high_risk', expr: "risk_level = 'high'" }],
|
measures: [{ name: 'high_risk_account_count', expr: 'count(*)' }],
|
||||||
},
|
segments: [{ name: 'high_risk', expr: "risk_level = 'high'" }],
|
||||||
{
|
},
|
||||||
name: 'int_procurement_qualifying_actions',
|
{
|
||||||
grain: [],
|
name: 'int_procurement_qualifying_actions',
|
||||||
columns: [],
|
grain: [],
|
||||||
joins: [],
|
columns: [],
|
||||||
measures: [],
|
joins: [],
|
||||||
},
|
measures: [],
|
||||||
]),
|
},
|
||||||
|
],
|
||||||
|
loadErrors: [],
|
||||||
|
})),
|
||||||
};
|
};
|
||||||
|
|
||||||
const result = await repairWikiSlRefs({
|
const result = await repairWikiSlRefs({
|
||||||
|
|
|
||||||
|
|
@ -56,7 +56,8 @@ async function loadVisibleSlRefs(
|
||||||
const warnings: string[] = [];
|
const warnings: string[] = [];
|
||||||
for (const connectionId of connectionIds) {
|
for (const connectionId of connectionIds) {
|
||||||
try {
|
try {
|
||||||
for (const source of await semanticLayerService.loadAllSources(connectionId)) {
|
const { sources } = await semanticLayerService.loadAllSources(connectionId);
|
||||||
|
for (const source of sources) {
|
||||||
for (const ref of entityRefsForSource(source)) {
|
for (const ref of entityRefsForSource(source)) {
|
||||||
refs.add(ref);
|
refs.add(ref);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -89,7 +89,7 @@ const buildMocks = (overrides: Partial<BuiltMocks> = {}): BuiltMocks => {
|
||||||
embeddingService: { computeEmbedding: vi.fn() },
|
embeddingService: { computeEmbedding: vi.fn() },
|
||||||
semanticLayerService: {
|
semanticLayerService: {
|
||||||
forWorktree: vi.fn().mockReturnThis(),
|
forWorktree: vi.fn().mockReturnThis(),
|
||||||
loadAllSources: vi.fn().mockResolvedValue([]),
|
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
|
||||||
readSourceFile: vi.fn(),
|
readSourceFile: vi.fn(),
|
||||||
},
|
},
|
||||||
slSearchService: { indexSources: vi.fn(), buildSearchText: vi.fn() },
|
slSearchService: { indexSources: vi.fn(), buildSearchText: vi.fn() },
|
||||||
|
|
|
||||||
|
|
@ -308,7 +308,7 @@ export class MemoryAgentService {
|
||||||
// Reindex SL search if any SL actions actually landed on main.
|
// Reindex SL search if any SL actions actually landed on main.
|
||||||
if (hasSL && finalActions.some((a) => a.target === 'sl')) {
|
if (hasSL && finalActions.some((a) => a.target === 'sl')) {
|
||||||
try {
|
try {
|
||||||
const allSources = await this.deps.semanticLayerService.loadAllSources(input.connectionId!);
|
const { sources: allSources } = await this.deps.semanticLayerService.loadAllSources(input.connectionId!);
|
||||||
await this.deps.slSearchService.indexSources(input.connectionId!, allSources);
|
await this.deps.slSearchService.indexSources(input.connectionId!, allSources);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
this.logger.warn(
|
this.logger.warn(
|
||||||
|
|
@ -610,7 +610,7 @@ export class MemoryAgentService {
|
||||||
|
|
||||||
private async buildSlIndex(connectionId: string): Promise<string> {
|
private async buildSlIndex(connectionId: string): Promise<string> {
|
||||||
const [sources, warehouseLine] = await Promise.all([
|
const [sources, warehouseLine] = await Promise.all([
|
||||||
this.deps.semanticLayerService.loadAllSources(connectionId),
|
this.deps.semanticLayerService.loadAllSources(connectionId).then((result) => result.sources),
|
||||||
this.buildWarehouseLine(connectionId),
|
this.buildWarehouseLine(connectionId),
|
||||||
]);
|
]);
|
||||||
const indexLines =
|
const indexLines =
|
||||||
|
|
|
||||||
|
|
@ -371,7 +371,7 @@ export class KtxDescriptionGenerator {
|
||||||
connectorId: input.connector.id,
|
connectorId: input.connector.id,
|
||||||
table: input.table.name,
|
table: input.table.name,
|
||||||
});
|
});
|
||||||
return 'Table not found';
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
@ -397,7 +397,7 @@ export class KtxDescriptionGenerator {
|
||||||
return description;
|
return description;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger?.error(`Error generating table description: ${errorMessage(error)}`);
|
this.logger?.error(`Error generating table description: ${errorMessage(error)}`);
|
||||||
return 'Table not found';
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -392,6 +392,26 @@ describe('local semantic-layer helpers', () => {
|
||||||
).rejects.toThrow('Invalid semantic-layer source');
|
).rejects.toThrow('Invalid semantic-layer source');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('reports legacy overlay column patches with a file-attributed migration hint', async () => {
|
||||||
|
const invalidYaml = [
|
||||||
|
'name: orders',
|
||||||
|
'columns:',
|
||||||
|
' - name: status',
|
||||||
|
' descriptions:',
|
||||||
|
' user: Order status.',
|
||||||
|
'',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
validateLocalSlSource(invalidYaml, { project, connectionId: 'warehouse', sourceName: 'orders' }),
|
||||||
|
).resolves.toEqual({
|
||||||
|
valid: false,
|
||||||
|
errors: [
|
||||||
|
"semantic-layer/warehouse/orders.yaml: column 'status' patches a manifest column but is in 'columns:' — move it to 'column_overrides:'",
|
||||||
|
],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('rejects unsafe source paths', async () => {
|
it('rejects unsafe source paths', async () => {
|
||||||
await expect(
|
await expect(
|
||||||
readLocalSlSource(project, {
|
readLocalSlSource(project, {
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ import {
|
||||||
type ManifestTableEntry,
|
type ManifestTableEntry,
|
||||||
projectManifestEntry,
|
projectManifestEntry,
|
||||||
SemanticLayerService,
|
SemanticLayerService,
|
||||||
|
toResolvedWire,
|
||||||
} from './semantic-layer.service.js';
|
} from './semantic-layer.service.js';
|
||||||
import type { PgliteSlSearchPrototypeOwnerOptions } from './pglite-sl-search-prototype.js';
|
import type { PgliteSlSearchPrototypeOwnerOptions } from './pglite-sl-search-prototype.js';
|
||||||
import { loadLatestSlDictionaryEntries } from './sl-dictionary-profile.js';
|
import { loadLatestSlDictionaryEntries } from './sl-dictionary-profile.js';
|
||||||
|
|
@ -240,7 +241,12 @@ export async function loadLocalSlSourceRecords(
|
||||||
if (!base) {
|
if (!base) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
const source = composeOverlay(base.source, parsed);
|
let source: SemanticLayerSource;
|
||||||
|
try {
|
||||||
|
source = composeOverlay(base.source, parsed);
|
||||||
|
} catch (error) {
|
||||||
|
throw new Error(`${path}: ${error instanceof Error ? error.message : String(error)}`);
|
||||||
|
}
|
||||||
sources.set(name, {
|
sources.set(name, {
|
||||||
...summarizeSemanticSource({ connectionId, path, source }),
|
...summarizeSemanticSource({ connectionId, path, source }),
|
||||||
yaml: sourceToYaml(source),
|
yaml: sourceToYaml(source),
|
||||||
|
|
@ -253,11 +259,28 @@ export async function loadLocalSlSourceRecords(
|
||||||
|
|
||||||
export async function validateLocalSlSource(
|
export async function validateLocalSlSource(
|
||||||
rawYaml: string,
|
rawYaml: string,
|
||||||
options?: { project?: KtxLocalProject; connectionId?: string },
|
options?: { project?: KtxLocalProject; connectionId?: string; sourceName?: string },
|
||||||
): Promise<LocalSlValidationResult> {
|
): Promise<LocalSlValidationResult> {
|
||||||
try {
|
try {
|
||||||
const parsed = parseYamlRecord(rawYaml);
|
const parsed = parseYamlRecord(rawYaml);
|
||||||
const schema = parsed.table || parsed.sql ? sourceDefinitionSchema : sourceOverlaySchema;
|
const schema = parsed.table || parsed.sql ? sourceDefinitionSchema : sourceOverlaySchema;
|
||||||
|
if (schema === sourceOverlaySchema && Array.isArray(parsed.columns)) {
|
||||||
|
const sourceName = options?.sourceName ?? (typeof parsed.name === 'string' ? parsed.name : 'source');
|
||||||
|
const path =
|
||||||
|
options?.connectionId && isSafeConnectionId(options.connectionId)
|
||||||
|
? `semantic-layer/${options.connectionId}/${sourceName}.yaml`
|
||||||
|
: `${sourceName}.yaml`;
|
||||||
|
const legacyColumnPatchErrors = parsed.columns
|
||||||
|
.filter((column): column is Record<string, unknown> => isRecord(column))
|
||||||
|
.filter((column) => typeof column.name === 'string' && (!column.expr || !column.type))
|
||||||
|
.map(
|
||||||
|
(column) =>
|
||||||
|
`${path}: column '${column.name}' patches a manifest column but is in 'columns:' — move it to 'column_overrides:'`,
|
||||||
|
);
|
||||||
|
if (legacyColumnPatchErrors.length > 0) {
|
||||||
|
return { valid: false, errors: legacyColumnPatchErrors };
|
||||||
|
}
|
||||||
|
}
|
||||||
const result = schema.parse(parsed);
|
const result = schema.parse(parsed);
|
||||||
const errors: string[] = [];
|
const errors: string[] = [];
|
||||||
|
|
||||||
|
|
@ -268,6 +291,10 @@ export async function validateLocalSlSource(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if ('table' in result || 'sql' in result) {
|
||||||
|
toResolvedWire(result as SemanticLayerSource);
|
||||||
|
}
|
||||||
|
|
||||||
return { valid: errors.length === 0, errors };
|
return { valid: errors.length === 0, errors };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
return { valid: false, errors: validationErrors(error) };
|
return { valid: false, errors: validationErrors(error) };
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import type { SemanticLayerQueryInput, SemanticLayerSource } from './types.js';
|
import type { ResolvedSemanticLayerSource, SemanticLayerQueryInput } from './types.js';
|
||||||
|
|
||||||
export interface KtxConnectionInfo {
|
export interface KtxConnectionInfo {
|
||||||
id: string;
|
id: string;
|
||||||
|
|
@ -20,7 +20,7 @@ export interface SlConnectionCatalogPort {
|
||||||
|
|
||||||
export interface SlPythonPort {
|
export interface SlPythonPort {
|
||||||
validateSources(input: {
|
validateSources(input: {
|
||||||
sources: SemanticLayerSource[];
|
sources: ResolvedSemanticLayerSource[];
|
||||||
dialect: string;
|
dialect: string;
|
||||||
recently_touched?: string[];
|
recently_touched?: string[];
|
||||||
}): Promise<{
|
}): Promise<{
|
||||||
|
|
@ -28,7 +28,7 @@ export interface SlPythonPort {
|
||||||
error?: unknown;
|
error?: unknown;
|
||||||
}>;
|
}>;
|
||||||
query(input: {
|
query(input: {
|
||||||
sources: SemanticLayerSource[];
|
sources: ResolvedSemanticLayerSource[];
|
||||||
query: SemanticLayerQueryInput;
|
query: SemanticLayerQueryInput;
|
||||||
dialect: string;
|
dialect: string;
|
||||||
}): Promise<{ data?: { sql?: string; plan?: Record<string, unknown> } | null; error?: unknown }>;
|
}): Promise<{ data?: { sql?: string; plan?: Record<string, unknown> } | null; error?: unknown }>;
|
||||||
|
|
|
||||||
60
packages/context/src/sl/schemas.contract.test.ts
Normal file
60
packages/context/src/sl/schemas.contract.test.ts
Normal file
|
|
@ -0,0 +1,60 @@
|
||||||
|
import { execFileSync } from 'node:child_process';
|
||||||
|
import { Ajv2020 } from 'ajv/dist/2020.js';
|
||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { resolvedSourceSchema } from './schemas.js';
|
||||||
|
import { toResolvedWire } from './semantic-layer.service.js';
|
||||||
|
import type { SemanticLayerSource } from './types.js';
|
||||||
|
|
||||||
|
const sourceDefinitionJsonSchema = JSON.parse(
|
||||||
|
execFileSync('uv', ['run', 'python', '-m', 'semantic_layer', 'dump-schema'], {
|
||||||
|
cwd: new URL('../../../../', import.meta.url),
|
||||||
|
encoding: 'utf8',
|
||||||
|
}),
|
||||||
|
) as Record<string, unknown>;
|
||||||
|
|
||||||
|
const fixtures: SemanticLayerSource[] = [
|
||||||
|
{
|
||||||
|
name: 'orders',
|
||||||
|
table: 'public.orders',
|
||||||
|
grain: ['id'],
|
||||||
|
columns: [
|
||||||
|
{ name: 'id', type: 'number' },
|
||||||
|
{
|
||||||
|
name: 'status',
|
||||||
|
type: 'string',
|
||||||
|
descriptions: { dbt: 'Order lifecycle status.' },
|
||||||
|
constraints: { dbt: { not_null: true } },
|
||||||
|
enum_values: { dbt: ['placed', 'shipped'] },
|
||||||
|
tests: { dbt: [{ name: 'accepted_values', package: 'dbt' }] },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
joins: [{ to: 'customers', on: 'orders.customer_id = customers.id', relationship: 'many_to_one' }],
|
||||||
|
measures: [{ name: 'order_count', expr: 'count(id)' }],
|
||||||
|
segments: [{ name: 'paid', expr: "status = 'paid'" }],
|
||||||
|
default_time_dimension: { dbt: 'created_at' },
|
||||||
|
tags: { dbt: ['mart'] },
|
||||||
|
freshness: { dbt: { loaded_at_field: 'updated_at' } },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'aav_orders',
|
||||||
|
sql: 'select id, status from public.orders where status = paid',
|
||||||
|
grain: ['id'],
|
||||||
|
columns: [{ name: 'id', type: 'number' }],
|
||||||
|
joins: [],
|
||||||
|
measures: [],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
describe('resolved source JSON Schema contract', () => {
|
||||||
|
it('keeps TS resolved-source fixtures accepted by the Python SourceDefinition schema', () => {
|
||||||
|
const ajv = new Ajv2020({ allErrors: true, strict: false });
|
||||||
|
const validate = ajv.compile(sourceDefinitionJsonSchema);
|
||||||
|
|
||||||
|
for (const fixture of fixtures) {
|
||||||
|
const wire = toResolvedWire(fixture);
|
||||||
|
expect(resolvedSourceSchema.safeParse(wire).success).toBe(true);
|
||||||
|
expect(validate(wire), JSON.stringify(validate.errors, null, 2)).toBe(true);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -78,6 +78,8 @@ const joinDeclarationSchema = z.object({
|
||||||
alias: z.string().optional(),
|
alias: z.string().optional(),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const resolvedJoinDeclarationSchema = joinDeclarationSchema.strict();
|
||||||
|
|
||||||
const sourceColumnSchema = z.object({
|
const sourceColumnSchema = z.object({
|
||||||
name: unqualifiedNameSchema,
|
name: unqualifiedNameSchema,
|
||||||
// type/descriptions optional on standalone sources: compose-time enrichment fills them
|
// type/descriptions optional on standalone sources: compose-time enrichment fills them
|
||||||
|
|
@ -89,24 +91,39 @@ const sourceColumnSchema = z.object({
|
||||||
visibility: z.enum(columnVisibilityValues).optional(),
|
visibility: z.enum(columnVisibilityValues).optional(),
|
||||||
descriptions: descriptionsSchema.optional(),
|
descriptions: descriptionsSchema.optional(),
|
||||||
expr: z.string().optional(),
|
expr: z.string().optional(),
|
||||||
|
natural_granularity: z.string().optional(),
|
||||||
constraints: sourceKeyedColumnConstraintsSchema.optional(),
|
constraints: sourceKeyedColumnConstraintsSchema.optional(),
|
||||||
enum_values: sourceKeyedStringArraySchema.optional(),
|
enum_values: sourceKeyedStringArraySchema.optional(),
|
||||||
tests: dbtColumnTestsSchema.optional(),
|
tests: dbtColumnTestsSchema.optional(),
|
||||||
});
|
});
|
||||||
|
|
||||||
/** Overlay column: type requires expr (structural types are inherited from manifest). */
|
const resolvedSourceColumnSchema = sourceColumnSchema.extend({
|
||||||
|
type: z.enum(columnTypeValues),
|
||||||
|
}).strict();
|
||||||
|
|
||||||
|
/** Overlay column: computed columns only. Structural columns live in the manifest. */
|
||||||
const overlayColumnSchema = z
|
const overlayColumnSchema = z
|
||||||
.object({
|
.object({
|
||||||
name: unqualifiedNameSchema,
|
name: unqualifiedNameSchema,
|
||||||
type: z.enum(columnTypeValues).optional(),
|
type: z.enum(columnTypeValues),
|
||||||
role: z.enum(columnRoleValues).optional(),
|
role: z.enum(columnRoleValues).optional(),
|
||||||
visibility: z.enum(columnVisibilityValues).optional(),
|
visibility: z.enum(columnVisibilityValues).optional(),
|
||||||
descriptions: descriptionsSchema.optional(),
|
descriptions: descriptionsSchema.optional(),
|
||||||
expr: z.string().optional(),
|
expr: z.string().min(1),
|
||||||
})
|
})
|
||||||
.refine((col) => !col.type || col.expr, {
|
.strict();
|
||||||
message: "Overlay column with 'type' must also have 'expr' (only computed columns may specify a type)",
|
|
||||||
});
|
const columnOverrideSchema = z
|
||||||
|
.object({
|
||||||
|
name: unqualifiedNameSchema,
|
||||||
|
role: z.enum(columnRoleValues).optional(),
|
||||||
|
visibility: z.enum(columnVisibilityValues).optional(),
|
||||||
|
descriptions: descriptionsSchema.optional(),
|
||||||
|
constraints: sourceKeyedColumnConstraintsSchema.optional(),
|
||||||
|
enum_values: sourceKeyedStringArraySchema.optional(),
|
||||||
|
tests: dbtColumnTestsSchema.optional(),
|
||||||
|
})
|
||||||
|
.strict();
|
||||||
|
|
||||||
/** Standalone source: has `table` or `sql`, requires grain + columns. */
|
/** Standalone source: has `table` or `sql`, requires grain + columns. */
|
||||||
export const sourceDefinitionSchema = z
|
export const sourceDefinitionSchema = z
|
||||||
|
|
@ -143,6 +160,26 @@ export const sourceDefinitionSchema = z
|
||||||
message: "Standalone source must have exactly one of 'table' or 'sql' (not both)",
|
message: "Standalone source must have exactly one of 'table' or 'sql' (not both)",
|
||||||
});
|
});
|
||||||
|
|
||||||
|
export const resolvedSourceSchema = z
|
||||||
|
.object({
|
||||||
|
name: z.string().min(1),
|
||||||
|
descriptions: descriptionsSchema.optional(),
|
||||||
|
table: z.string().optional(),
|
||||||
|
sql: z.string().optional(),
|
||||||
|
grain: z.array(unqualifiedNameSchema).min(1),
|
||||||
|
columns: z.array(resolvedSourceColumnSchema).min(1),
|
||||||
|
joins: z.array(resolvedJoinDeclarationSchema).default([]),
|
||||||
|
measures: z.array(slMeasureDefinitionSchema).default([]),
|
||||||
|
segments: z.array(segmentDefinitionSchema).optional(),
|
||||||
|
default_time_dimension: defaultTimeDimensionDbtSchema.optional(),
|
||||||
|
tags: sourceKeyedStringArraySchema.optional(),
|
||||||
|
freshness: sourceFreshnessSchema.optional(),
|
||||||
|
})
|
||||||
|
.strict()
|
||||||
|
.refine((s) => (s.table || s.sql) && !(s.table && s.sql), {
|
||||||
|
message: "Resolved source must have exactly one of 'table' or 'sql' (not both)",
|
||||||
|
});
|
||||||
|
|
||||||
/** Overlay source: no table/sql, all fields optional except name. */
|
/** Overlay source: no table/sql, all fields optional except name. */
|
||||||
export const sourceOverlaySchema = z
|
export const sourceOverlaySchema = z
|
||||||
.object({
|
.object({
|
||||||
|
|
@ -150,6 +187,7 @@ export const sourceOverlaySchema = z
|
||||||
descriptions: z.record(z.string(), z.string()).optional(),
|
descriptions: z.record(z.string(), z.string()).optional(),
|
||||||
grain: z.array(unqualifiedNameSchema).optional(),
|
grain: z.array(unqualifiedNameSchema).optional(),
|
||||||
columns: z.array(overlayColumnSchema).optional(),
|
columns: z.array(overlayColumnSchema).optional(),
|
||||||
|
column_overrides: z.array(columnOverrideSchema).optional(),
|
||||||
joins: z.array(joinDeclarationSchema).optional(),
|
joins: z.array(joinDeclarationSchema).optional(),
|
||||||
measures: z.array(slMeasureDefinitionSchema).optional(),
|
measures: z.array(slMeasureDefinitionSchema).optional(),
|
||||||
segments: z.array(segmentDefinitionSchema).optional(),
|
segments: z.array(segmentDefinitionSchema).optional(),
|
||||||
|
|
|
||||||
|
|
@ -2,13 +2,17 @@ import type { Mock } from 'vitest';
|
||||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
ColumnNameCollisionError,
|
||||||
composeOverlay,
|
composeOverlay,
|
||||||
|
ConflictingExcludeAndOverrideError,
|
||||||
enrichColumnsFromManifest,
|
enrichColumnsFromManifest,
|
||||||
findDanglingSegmentRefs,
|
findDanglingSegmentRefs,
|
||||||
projectManifestEntry,
|
projectManifestEntry,
|
||||||
SemanticLayerService,
|
SemanticLayerService,
|
||||||
|
toResolvedWire,
|
||||||
|
UnknownColumnOverrideError,
|
||||||
} from './semantic-layer.service.js';
|
} from './semantic-layer.service.js';
|
||||||
import { sourceDefinitionSchema } from './schemas.js';
|
import { resolvedSourceSchema, sourceDefinitionSchema, sourceOverlaySchema } from './schemas.js';
|
||||||
import type { SemanticLayerSource } from './types.js';
|
import type { SemanticLayerSource } from './types.js';
|
||||||
|
|
||||||
const pythonPort = {
|
const pythonPort = {
|
||||||
|
|
@ -139,10 +143,10 @@ describe('composeOverlay', () => {
|
||||||
expect(composed.measures).toHaveLength(1);
|
expect(composed.measures).toHaveLength(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('merges overlay columns onto same-named manifest columns, preserving manifest type when overlay omits it', () => {
|
it('applies column_overrides to same-named manifest columns', () => {
|
||||||
const overlay = {
|
const overlay = {
|
||||||
name: 'fct_labs',
|
name: 'fct_labs',
|
||||||
columns: [
|
column_overrides: [
|
||||||
{ name: 'lab_order_id', descriptions: { user: 'Primary key' } },
|
{ name: 'lab_order_id', descriptions: { user: 'Primary key' } },
|
||||||
{ name: 'admin_user_id', descriptions: { user: 'FK to admin_users' } },
|
{ name: 'admin_user_id', descriptions: { user: 'FK to admin_users' } },
|
||||||
],
|
],
|
||||||
|
|
@ -158,11 +162,13 @@ describe('composeOverlay', () => {
|
||||||
expect(adminUser?.descriptions).toEqual({ user: 'FK to admin_users' });
|
expect(adminUser?.descriptions).toEqual({ user: 'FK to admin_users' });
|
||||||
});
|
});
|
||||||
|
|
||||||
it('still appends new overlay computed columns alongside merged same-name columns', () => {
|
it('appends computed columns alongside column overrides', () => {
|
||||||
const overlay = {
|
const overlay = {
|
||||||
name: 'fct_labs',
|
name: 'fct_labs',
|
||||||
columns: [
|
column_overrides: [
|
||||||
{ name: 'lab_order_id', descriptions: { user: 'PK doc' } },
|
{ name: 'lab_order_id', descriptions: { user: 'PK doc' } },
|
||||||
|
],
|
||||||
|
columns: [
|
||||||
{ name: 'is_byol', type: 'boolean', expr: "lab_type = 'byol'" },
|
{ name: 'is_byol', type: 'boolean', expr: "lab_type = 'byol'" },
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
|
|
@ -172,6 +178,34 @@ describe('composeOverlay', () => {
|
||||||
expect(composed.columns.find((c) => c.name === 'lab_order_id')?.type).toBe('string');
|
expect(composed.columns.find((c) => c.name === 'lab_order_id')?.type).toBe('string');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('rejects column_overrides that target unknown manifest columns', () => {
|
||||||
|
expect(() =>
|
||||||
|
composeOverlay(baseTable, {
|
||||||
|
name: 'fct_labs',
|
||||||
|
column_overrides: [{ name: 'missing', descriptions: { user: 'Nope' } }],
|
||||||
|
}),
|
||||||
|
).toThrow(UnknownColumnOverrideError);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects computed columns whose names collide with manifest columns', () => {
|
||||||
|
expect(() =>
|
||||||
|
composeOverlay(baseTable, {
|
||||||
|
name: 'fct_labs',
|
||||||
|
columns: [{ name: 'lab_order_id', type: 'string', expr: 'lab_order_id' }],
|
||||||
|
}),
|
||||||
|
).toThrow(ColumnNameCollisionError);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects exclude/override conflicts before applying exclusions', () => {
|
||||||
|
expect(() =>
|
||||||
|
composeOverlay(baseTable, {
|
||||||
|
name: 'fct_labs',
|
||||||
|
exclude_columns: ['lab_order_id'],
|
||||||
|
column_overrides: [{ name: 'lab_order_id', descriptions: { user: 'Hidden PK' } }],
|
||||||
|
}),
|
||||||
|
).toThrow(ConflictingExcludeAndOverrideError);
|
||||||
|
});
|
||||||
|
|
||||||
it('merges overlay descriptions (plural) with base descriptions keyed by source', () => {
|
it('merges overlay descriptions (plural) with base descriptions keyed by source', () => {
|
||||||
const baseWithDescriptions: SemanticLayerSource = {
|
const baseWithDescriptions: SemanticLayerSource = {
|
||||||
...baseTable,
|
...baseTable,
|
||||||
|
|
@ -451,6 +485,62 @@ describe('sourceDefinitionSchema', () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('sourceOverlaySchema', () => {
|
||||||
|
it('accepts column_overrides and keeps columns computed-only', () => {
|
||||||
|
const result = sourceOverlaySchema.safeParse({
|
||||||
|
name: 'orders',
|
||||||
|
column_overrides: [{ name: 'status', descriptions: { user: 'Lifecycle status' } }],
|
||||||
|
columns: [{ name: 'is_paid', type: 'boolean', expr: "status = 'paid'" }],
|
||||||
|
});
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects typeless overlay columns and singular description on overrides', () => {
|
||||||
|
const result = sourceOverlaySchema.safeParse({
|
||||||
|
name: 'orders',
|
||||||
|
column_overrides: [{ name: 'status', description: 'Lifecycle status' }],
|
||||||
|
columns: [{ name: 'status', descriptions: { user: 'Lifecycle status' } }],
|
||||||
|
});
|
||||||
|
expect(result.success).toBe(false);
|
||||||
|
if (!result.success) {
|
||||||
|
const paths = result.error.issues.map((issue) => issue.path.join('.'));
|
||||||
|
expect(paths).toContain('column_overrides.0');
|
||||||
|
expect(paths).toContain('columns.0.type');
|
||||||
|
expect(paths).toContain('columns.0.expr');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('toResolvedWire', () => {
|
||||||
|
it('strips TS-only authoring and provenance fields before the Python boundary', () => {
|
||||||
|
const wire = toResolvedWire({
|
||||||
|
name: 'orders',
|
||||||
|
table: 'public.orders',
|
||||||
|
inherits_columns_from: 'orders',
|
||||||
|
grain: ['id'],
|
||||||
|
columns: [{ name: 'id', type: 'string' }],
|
||||||
|
joins: [{ to: 'customers', on: 'orders.customer_id = customers.id', relationship: 'many_to_one', source: 'formal' }],
|
||||||
|
measures: [],
|
||||||
|
usage: {
|
||||||
|
narrative: 'Frequently queried orders.',
|
||||||
|
frequencyTier: 'high',
|
||||||
|
commonFilters: ['status'],
|
||||||
|
commonJoins: [],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(wire).toEqual({
|
||||||
|
name: 'orders',
|
||||||
|
table: 'public.orders',
|
||||||
|
grain: ['id'],
|
||||||
|
columns: [{ name: 'id', type: 'string' }],
|
||||||
|
joins: [{ to: 'customers', on: 'orders.customer_id = customers.id', relationship: 'many_to_one' }],
|
||||||
|
measures: [],
|
||||||
|
});
|
||||||
|
expect(resolvedSourceSchema.parse(wire)).toEqual(wire);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('projectManifestEntry', () => {
|
describe('projectManifestEntry', () => {
|
||||||
it('projects manifest usage onto the semantic-layer source', () => {
|
it('projects manifest usage onto the semantic-layer source', () => {
|
||||||
const source = projectManifestEntry('orders', {
|
const source = projectManifestEntry('orders', {
|
||||||
|
|
@ -570,7 +660,8 @@ describe('loadAllSources — standalone enrichment via inherits_columns_from', (
|
||||||
].join('\n'),
|
].join('\n'),
|
||||||
});
|
});
|
||||||
|
|
||||||
const sources = await service.loadAllSources('conn-1');
|
const { sources, loadErrors } = await service.loadAllSources('conn-1');
|
||||||
|
expect(loadErrors).toEqual([]);
|
||||||
|
|
||||||
expect(sources[0]).toMatchObject({
|
expect(sources[0]).toMatchObject({
|
||||||
name: 'orders',
|
name: 'orders',
|
||||||
|
|
@ -634,7 +725,8 @@ describe('loadAllSources — standalone enrichment via inherits_columns_from', (
|
||||||
return Promise.reject(new Error(`Unexpected readFile: ${path}`));
|
return Promise.reject(new Error(`Unexpected readFile: ${path}`));
|
||||||
});
|
});
|
||||||
|
|
||||||
const sources = await service.loadAllSources('conn-1');
|
const { sources, loadErrors } = await service.loadAllSources('conn-1');
|
||||||
|
expect(loadErrors).toEqual([]);
|
||||||
const aav = sources.find((s) => s.name === 'aav_consignments');
|
const aav = sources.find((s) => s.name === 'aav_consignments');
|
||||||
expect(aav).toBeDefined();
|
expect(aav).toBeDefined();
|
||||||
expect(aav?.columns).toEqual([
|
expect(aav?.columns).toEqual([
|
||||||
|
|
@ -679,7 +771,8 @@ describe('loadAllSources — standalone enrichment via inherits_columns_from', (
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
const sources = await service.loadAllSources('conn-1');
|
const { sources, loadErrors } = await service.loadAllSources('conn-1');
|
||||||
|
expect(loadErrors).toEqual([]);
|
||||||
const aav = sources.find((s) => s.name === 'aav_consignments');
|
const aav = sources.find((s) => s.name === 'aav_consignments');
|
||||||
expect(aav?.columns[0].type).toBe('string');
|
expect(aav?.columns[0].type).toBe('string');
|
||||||
});
|
});
|
||||||
|
|
@ -703,7 +796,8 @@ describe('loadAllSources — standalone enrichment via inherits_columns_from', (
|
||||||
].join('\n'),
|
].join('\n'),
|
||||||
});
|
});
|
||||||
|
|
||||||
const sources = await service.loadAllSources('conn-1');
|
const { sources, loadErrors } = await service.loadAllSources('conn-1');
|
||||||
|
expect(loadErrors).toEqual([]);
|
||||||
const aav = sources.find((s) => s.name === 'aav_consignments');
|
const aav = sources.find((s) => s.name === 'aav_consignments');
|
||||||
expect(aav?.columns).toEqual([{ name: 'FOO', type: 'string' }]);
|
expect(aav?.columns).toEqual([{ name: 'FOO', type: 'string' }]);
|
||||||
});
|
});
|
||||||
|
|
@ -726,7 +820,8 @@ describe('loadAllSources — standalone enrichment via inherits_columns_from', (
|
||||||
].join('\n'),
|
].join('\n'),
|
||||||
});
|
});
|
||||||
|
|
||||||
const sources = await service.loadAllSources('conn-1');
|
const { sources, loadErrors } = await service.loadAllSources('conn-1');
|
||||||
|
expect(loadErrors).toEqual([]);
|
||||||
|
|
||||||
expect(sources[0]).toMatchObject({
|
expect(sources[0]).toMatchObject({
|
||||||
name: 'orders',
|
name: 'orders',
|
||||||
|
|
@ -734,6 +829,33 @@ describe('loadAllSources — standalone enrichment via inherits_columns_from', (
|
||||||
columns: [{ name: 'id', type: 'string', descriptions: { user: 'Stable order identifier.' } }],
|
columns: [{ name: 'id', type: 'string', descriptions: { user: 'Stable order identifier.' } }],
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('reports file-attributed errors for legacy overlay column patches', async () => {
|
||||||
|
const schemaPath = 'semantic-layer/conn-1/_schema/marts.yaml';
|
||||||
|
const overlayPath = 'semantic-layer/conn-1/orders.yaml';
|
||||||
|
configService.listFiles.mockResolvedValue({ files: [schemaPath, overlayPath] });
|
||||||
|
configService.readFile.mockImplementation((path: string) => {
|
||||||
|
if (path === schemaPath) {
|
||||||
|
return Promise.resolve({
|
||||||
|
content: [
|
||||||
|
'tables:',
|
||||||
|
' orders:',
|
||||||
|
' table: public.orders',
|
||||||
|
' columns:',
|
||||||
|
' - { name: id, type: string, pk: true }',
|
||||||
|
].join('\n'),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return Promise.resolve({
|
||||||
|
content: ['name: orders', 'columns:', ' - name: id', ' descriptions: { user: "Stable id." }'].join('\n'),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const { loadErrors } = await service.loadAllSources('conn-1');
|
||||||
|
|
||||||
|
expect(loadErrors.join('\n')).toContain(overlayPath);
|
||||||
|
expect(loadErrors.join('\n')).toContain("move it to 'column_overrides:'");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('validateWithProposedSource', () => {
|
describe('validateWithProposedSource', () => {
|
||||||
|
|
|
||||||
|
|
@ -4,8 +4,14 @@ import { noopLogger } from '../core/index.js';
|
||||||
import type { TableUsageOutput } from '../ingest/adapters/historic-sql/skill-schemas.js';
|
import type { TableUsageOutput } from '../ingest/adapters/historic-sql/skill-schemas.js';
|
||||||
import type { SlConnectionCatalogPort, SlPythonPort } from './ports.js';
|
import type { SlConnectionCatalogPort, SlPythonPort } from './ports.js';
|
||||||
import { normalizeSemanticLayerDescriptions } from './description-normalization.js';
|
import { normalizeSemanticLayerDescriptions } from './description-normalization.js';
|
||||||
import { isOverlaySource, sourceDefinitionSchema, sourceOverlaySchema } from './schemas.js';
|
import { isOverlaySource, resolvedSourceSchema, sourceDefinitionSchema, sourceOverlaySchema } from './schemas.js';
|
||||||
import type { SemanticLayerQueryExecutionResult, SemanticLayerQueryInput, SemanticLayerSource } from './types.js';
|
import type {
|
||||||
|
ResolvedSemanticLayerSource,
|
||||||
|
SemanticLayerColumnOverride,
|
||||||
|
SemanticLayerQueryExecutionResult,
|
||||||
|
SemanticLayerQueryInput,
|
||||||
|
SemanticLayerSource,
|
||||||
|
} from './types.js';
|
||||||
|
|
||||||
interface WriteSourceOptions {
|
interface WriteSourceOptions {
|
||||||
skipValidation?: boolean;
|
skipValidation?: boolean;
|
||||||
|
|
@ -14,6 +20,30 @@ interface WriteSourceOptions {
|
||||||
const SL_DIR_PREFIX = 'semantic-layer';
|
const SL_DIR_PREFIX = 'semantic-layer';
|
||||||
const CONNECTION_ID_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/;
|
const CONNECTION_ID_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/;
|
||||||
|
|
||||||
|
export interface LoadAllSourcesResult {
|
||||||
|
sources: SemanticLayerSource[];
|
||||||
|
loadErrors: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export class UnknownColumnOverrideError extends Error {}
|
||||||
|
export class ColumnNameCollisionError extends Error {}
|
||||||
|
export class ConflictingExcludeAndOverrideError extends Error {}
|
||||||
|
class ComposeContractError extends Error {}
|
||||||
|
|
||||||
|
function isComposeError(error: unknown): boolean {
|
||||||
|
return (
|
||||||
|
error instanceof UnknownColumnOverrideError ||
|
||||||
|
error instanceof ColumnNameCollisionError ||
|
||||||
|
error instanceof ConflictingExcludeAndOverrideError ||
|
||||||
|
error instanceof ComposeContractError
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatComposeError(filePath: string, error: unknown): string {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
return `${filePath}: ${message}`;
|
||||||
|
}
|
||||||
|
|
||||||
function formatPortError(error: unknown, fallback: string): string {
|
function formatPortError(error: unknown, fallback: string): string {
|
||||||
if (typeof error === 'string') {
|
if (typeof error === 'string') {
|
||||||
return error;
|
return error;
|
||||||
|
|
@ -37,6 +67,24 @@ function formatPortError(error: unknown, fallback: string): string {
|
||||||
return fallback;
|
return fallback;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function toResolvedWire(source: SemanticLayerSource): ResolvedSemanticLayerSource {
|
||||||
|
const stripped = {
|
||||||
|
...source,
|
||||||
|
columns: source.columns.map((column) => ({ ...column })),
|
||||||
|
joins: source.joins.map(({ source: _source, ...join }) => join),
|
||||||
|
} as Record<string, unknown>;
|
||||||
|
delete stripped.inherits_columns_from;
|
||||||
|
delete stripped.usage;
|
||||||
|
delete stripped.source_type;
|
||||||
|
|
||||||
|
const parsed = resolvedSourceSchema.safeParse(stripped);
|
||||||
|
if (!parsed.success) {
|
||||||
|
const issues = parsed.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`).join('; ');
|
||||||
|
throw new ComposeContractError(`resolved source '${source.name}' violates the TS/Python contract: ${issues}`);
|
||||||
|
}
|
||||||
|
return parsed.data as ResolvedSemanticLayerSource;
|
||||||
|
}
|
||||||
|
|
||||||
export class SemanticLayerService {
|
export class SemanticLayerService {
|
||||||
constructor(
|
constructor(
|
||||||
private readonly configService: KtxFileStorePort,
|
private readonly configService: KtxFileStorePort,
|
||||||
|
|
@ -158,16 +206,17 @@ export class SemanticLayerService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async loadAllSources(connectionId: string): Promise<SemanticLayerSource[]> {
|
async loadAllSources(connectionId: string): Promise<LoadAllSourcesResult> {
|
||||||
const dir = `${SL_DIR_PREFIX}/${connectionId}`;
|
const dir = `${SL_DIR_PREFIX}/${connectionId}`;
|
||||||
const schemaDir = `${dir}/_schema`;
|
const schemaDir = `${dir}/_schema`;
|
||||||
|
const loadErrors: string[] = [];
|
||||||
|
|
||||||
let allFiles: string[];
|
let allFiles: string[];
|
||||||
try {
|
try {
|
||||||
const result = await this.configService.listFiles(dir);
|
const result = await this.configService.listFiles(dir);
|
||||||
allFiles = result.files.filter((f) => f.endsWith('.yaml'));
|
allFiles = result.files.filter((f) => f.endsWith('.yaml'));
|
||||||
} catch {
|
} catch {
|
||||||
return [];
|
return { sources: [], loadErrors };
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1. Load manifest shards from _schema/*.yaml → project to sources
|
// 1. Load manifest shards from _schema/*.yaml → project to sources
|
||||||
|
|
@ -184,7 +233,9 @@ export class SemanticLayerService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
this.logger.warn(`Failed to parse manifest shard ${filePath}: ${e}`);
|
const message = `Failed to parse manifest shard ${filePath}: ${e instanceof Error ? e.message : String(e)}`;
|
||||||
|
loadErrors.push(message);
|
||||||
|
this.logger.warn(message);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -227,6 +278,7 @@ export class SemanticLayerService {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
toResolvedWire(standalone);
|
||||||
sources.set(name, standalone);
|
sources.set(name, standalone);
|
||||||
} else {
|
} else {
|
||||||
// Overlay — compose with manifest entry if present
|
// Overlay — compose with manifest entry if present
|
||||||
|
|
@ -238,11 +290,15 @@ export class SemanticLayerService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
this.logger.warn(`Failed to parse YAML file ${filePath}: ${e}`);
|
const message = isComposeError(e)
|
||||||
|
? formatComposeError(filePath, e)
|
||||||
|
: `Failed to parse YAML file ${filePath}: ${e instanceof Error ? e.message : String(e)}`;
|
||||||
|
loadErrors.push(message);
|
||||||
|
this.logger.warn(message);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return Array.from(sources.values());
|
return { sources: Array.from(sources.values()), loadErrors };
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -622,8 +678,10 @@ export class SemanticLayerService {
|
||||||
connectionId: string,
|
connectionId: string,
|
||||||
proposedSource: SemanticLayerSource,
|
proposedSource: SemanticLayerSource,
|
||||||
): Promise<{ errors: string[]; warnings: string[]; perSourceWarnings: Record<string, string[]> }> {
|
): Promise<{ errors: string[]; warnings: string[]; perSourceWarnings: Record<string, string[]> }> {
|
||||||
const existing = await this.loadAllSources(connectionId);
|
const loaded = await this.loadAllSources(connectionId);
|
||||||
|
const existing = loaded.sources;
|
||||||
const merged = existing.filter((s) => s.name !== proposedSource.name);
|
const merged = existing.filter((s) => s.name !== proposedSource.name);
|
||||||
|
const loadErrors = [...loaded.loadErrors];
|
||||||
|
|
||||||
// Overlays (no table/sql) must be composed with their manifest base before
|
// Overlays (no table/sql) must be composed with their manifest base before
|
||||||
// validation, otherwise the filter below drops them and the edited source
|
// validation, otherwise the filter below drops them and the edited source
|
||||||
|
|
@ -641,11 +699,27 @@ export class SemanticLayerService {
|
||||||
perSourceWarnings: {},
|
perSourceWarnings: {},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
toPush = composeOverlay(base, { ...proposedSource });
|
try {
|
||||||
|
toPush = composeOverlay(base, { ...proposedSource });
|
||||||
|
} catch (error) {
|
||||||
|
return {
|
||||||
|
errors: [...loadErrors, formatComposeError(`${proposedSource.name}.yaml`, error)],
|
||||||
|
warnings: [],
|
||||||
|
perSourceWarnings: {},
|
||||||
|
};
|
||||||
|
}
|
||||||
} else if (proposedSource.inherits_columns_from) {
|
} else if (proposedSource.inherits_columns_from) {
|
||||||
const base = await this.findManifestEntryByTableRef(connectionId, proposedSource.inherits_columns_from);
|
const base = await this.findManifestEntryByTableRef(connectionId, proposedSource.inherits_columns_from);
|
||||||
if (base) {
|
if (base) {
|
||||||
toPush = enrichColumnsFromManifest(proposedSource, base);
|
try {
|
||||||
|
toPush = enrichColumnsFromManifest(proposedSource, base);
|
||||||
|
} catch (error) {
|
||||||
|
return {
|
||||||
|
errors: [...loadErrors, formatComposeError(`${proposedSource.name}.yaml`, error)],
|
||||||
|
warnings: [],
|
||||||
|
perSourceWarnings: {},
|
||||||
|
};
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Miss is non-fatal — the source ships unenriched, validator will surface
|
// Miss is non-fatal — the source ships unenriched, validator will surface
|
||||||
// any column-without-type errors via the warehouse probe.
|
// any column-without-type errors via the warehouse probe.
|
||||||
|
|
@ -654,37 +728,37 @@ export class SemanticLayerService {
|
||||||
|
|
||||||
const validatable = merged.filter((s) => s.table != null || s.sql != null);
|
const validatable = merged.filter((s) => s.table != null || s.sql != null);
|
||||||
if (validatable.length === 0) {
|
if (validatable.length === 0) {
|
||||||
return { errors: [], warnings: [], perSourceWarnings: {} };
|
return { errors: loadErrors, warnings: [], perSourceWarnings: {} };
|
||||||
}
|
}
|
||||||
|
|
||||||
const dialect = await this.getDialectForConnection(connectionId);
|
const dialect = await this.getDialectForConnection(connectionId);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const { data, error } = await this.python.validateSources({
|
const { data, error } = await this.python.validateSources({
|
||||||
sources: validatable,
|
sources: validatable.map(toResolvedWire),
|
||||||
dialect,
|
dialect,
|
||||||
recently_touched: [proposedSource.name],
|
recently_touched: [proposedSource.name],
|
||||||
});
|
});
|
||||||
if (error) {
|
if (error) {
|
||||||
const errorMsg = formatPortError(error, 'Unknown validation error');
|
const errorMsg = formatPortError(error, 'Unknown validation error');
|
||||||
return { errors: [errorMsg], warnings: [], perSourceWarnings: {} };
|
return { errors: [...loadErrors, errorMsg], warnings: [], perSourceWarnings: {} };
|
||||||
}
|
}
|
||||||
if (!data) {
|
if (!data) {
|
||||||
return {
|
return {
|
||||||
errors: await this.validatePhysicalTableReferences(connectionId, validatable),
|
errors: [...loadErrors, ...(await this.validatePhysicalTableReferences(connectionId, validatable))],
|
||||||
warnings: [],
|
warnings: [],
|
||||||
perSourceWarnings: {},
|
perSourceWarnings: {},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
const physicalErrors = await this.validatePhysicalTableReferences(connectionId, validatable);
|
const physicalErrors = await this.validatePhysicalTableReferences(connectionId, validatable);
|
||||||
return {
|
return {
|
||||||
errors: [...(data.errors ?? []), ...physicalErrors],
|
errors: [...loadErrors, ...(data.errors ?? []), ...physicalErrors],
|
||||||
warnings: data.warnings ?? [],
|
warnings: data.warnings ?? [],
|
||||||
perSourceWarnings: data.per_source_warnings ?? {},
|
perSourceWarnings: data.per_source_warnings ?? {},
|
||||||
};
|
};
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return {
|
return {
|
||||||
errors: [`Validation call failed: ${e instanceof Error ? e.message : String(e)}`],
|
errors: [...loadErrors, `Validation call failed: ${e instanceof Error ? e.message : String(e)}`],
|
||||||
warnings: [],
|
warnings: [],
|
||||||
perSourceWarnings: {},
|
perSourceWarnings: {},
|
||||||
};
|
};
|
||||||
|
|
@ -692,23 +766,23 @@ export class SemanticLayerService {
|
||||||
}
|
}
|
||||||
|
|
||||||
async validateSourcesForConnection(connectionId: string): Promise<{ errors: string[]; warnings: string[] }> {
|
async validateSourcesForConnection(connectionId: string): Promise<{ errors: string[]; warnings: string[] }> {
|
||||||
const allSources = await this.loadAllSources(connectionId);
|
const { sources: allSources, loadErrors } = await this.loadAllSources(connectionId);
|
||||||
const sources = allSources.filter((source) => source.table != null || source.sql != null);
|
const sources = allSources.filter((source) => source.table != null || source.sql != null);
|
||||||
if (sources.length === 0) {
|
if (sources.length === 0) {
|
||||||
return { errors: [], warnings: [] };
|
return { errors: loadErrors, warnings: [] };
|
||||||
}
|
}
|
||||||
|
|
||||||
const dialect = await this.getDialectForConnection(connectionId);
|
const dialect = await this.getDialectForConnection(connectionId);
|
||||||
const { data, error } = await this.python.validateSources({ sources, dialect });
|
const { data, error } = await this.python.validateSources({ sources: sources.map(toResolvedWire), dialect });
|
||||||
if (error) {
|
if (error) {
|
||||||
return { errors: [formatPortError(error, 'Unknown validation error')], warnings: [] };
|
return { errors: [...loadErrors, formatPortError(error, 'Unknown validation error')], warnings: [] };
|
||||||
}
|
}
|
||||||
if (!data) {
|
if (!data) {
|
||||||
return { errors: await this.validatePhysicalTableReferences(connectionId, sources), warnings: [] };
|
return { errors: [...loadErrors, ...(await this.validatePhysicalTableReferences(connectionId, sources))], warnings: [] };
|
||||||
}
|
}
|
||||||
const physicalErrors = await this.validatePhysicalTableReferences(connectionId, sources);
|
const physicalErrors = await this.validatePhysicalTableReferences(connectionId, sources);
|
||||||
return {
|
return {
|
||||||
errors: [...(data.errors ?? []), ...physicalErrors],
|
errors: [...loadErrors, ...(data.errors ?? []), ...physicalErrors],
|
||||||
warnings: data.warnings ?? [],
|
warnings: data.warnings ?? [],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
@ -802,6 +876,7 @@ export class SemanticLayerService {
|
||||||
} else {
|
} else {
|
||||||
// Overlay — check references against manifest
|
// Overlay — check references against manifest
|
||||||
const excludeColumns = (data.exclude_columns as string[]) ?? [];
|
const excludeColumns = (data.exclude_columns as string[]) ?? [];
|
||||||
|
const columnOverrides = (data.column_overrides as Array<{ name: string }> | undefined) ?? [];
|
||||||
const disableJoins = (data.disable_joins as string[]) ?? [];
|
const disableJoins = (data.disable_joins as string[]) ?? [];
|
||||||
const cols = manifestColumns.get(name);
|
const cols = manifestColumns.get(name);
|
||||||
const joins = manifestJoins.get(name);
|
const joins = manifestJoins.get(name);
|
||||||
|
|
@ -817,6 +892,16 @@ export class SemanticLayerService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const excluded = new Set(excludeColumns);
|
||||||
|
for (const override of columnOverrides) {
|
||||||
|
if (!cols.has(override.name)) {
|
||||||
|
warnings.push(`${name}: column_overrides references non-existent column '${override.name}'`);
|
||||||
|
}
|
||||||
|
if (excluded.has(override.name)) {
|
||||||
|
warnings.push(`${name}: column '${override.name}' appears in both exclude_columns and column_overrides`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for (const joinOn of disableJoins) {
|
for (const joinOn of disableJoins) {
|
||||||
const normalized = joinOn.replace(/\s+/g, ' ').trim();
|
const normalized = joinOn.replace(/\s+/g, ' ').trim();
|
||||||
if (!joins?.has(normalized)) {
|
if (!joins?.has(normalized)) {
|
||||||
|
|
@ -999,7 +1084,10 @@ export class SemanticLayerService {
|
||||||
*/
|
*/
|
||||||
async executeQuery(connectionId: string, query: SemanticLayerQueryInput): Promise<SemanticLayerQueryExecutionResult> {
|
async executeQuery(connectionId: string, query: SemanticLayerQueryInput): Promise<SemanticLayerQueryExecutionResult> {
|
||||||
// 1. Load sources, filtering out sources with no table or sql
|
// 1. Load sources, filtering out sources with no table or sql
|
||||||
const allSources = await this.loadAllSources(connectionId);
|
const { sources: allSources, loadErrors } = await this.loadAllSources(connectionId);
|
||||||
|
if (loadErrors.length > 0) {
|
||||||
|
throw new Error(`Semantic layer source load failed: ${loadErrors.join('; ')}`);
|
||||||
|
}
|
||||||
const sources = allSources.filter((s) => {
|
const sources = allSources.filter((s) => {
|
||||||
if (!s.table && !s.sql) {
|
if (!s.table && !s.sql) {
|
||||||
this.logger.warn(`Skipping source "${s.name}" with no table or sql defined`);
|
this.logger.warn(`Skipping source "${s.name}" with no table or sql defined`);
|
||||||
|
|
@ -1021,7 +1109,7 @@ export class SemanticLayerService {
|
||||||
|
|
||||||
// 3. Generate SQL via python SL engine
|
// 3. Generate SQL via python SL engine
|
||||||
const { data: slResult, error: slError } = await this.python.query({
|
const { data: slResult, error: slError } = await this.python.query({
|
||||||
sources,
|
sources: sources.map(toResolvedWire),
|
||||||
query,
|
query,
|
||||||
dialect,
|
dialect,
|
||||||
});
|
});
|
||||||
|
|
@ -1092,18 +1180,20 @@ export function projectManifestEntry(name: string, entry: ManifestTableEntry): S
|
||||||
const grain = pkColumns.length > 0 ? pkColumns : entry.columns.map((c) => c.name);
|
const grain = pkColumns.length > 0 ? pkColumns : entry.columns.map((c) => c.name);
|
||||||
|
|
||||||
// Table-level dbt config from manifest shards is surfaced on the source for search / tools.
|
// Table-level dbt config from manifest shards is surfaced on the source for search / tools.
|
||||||
return {
|
const source: SemanticLayerSource = {
|
||||||
name,
|
name,
|
||||||
table: entry.table,
|
table: entry.table,
|
||||||
descriptions: entry.descriptions,
|
descriptions: entry.descriptions,
|
||||||
grain,
|
grain,
|
||||||
columns,
|
columns,
|
||||||
joins: (entry.joins ?? []).map((j) => ({ to: j.to, on: j.on, relationship: j.relationship, source: j.source })),
|
joins: (entry.joins ?? []).map((j) => ({ to: j.to, on: j.on, relationship: j.relationship })),
|
||||||
measures: [],
|
measures: [],
|
||||||
...(entry.tags?.dbt?.length ? { tags: entry.tags } : {}),
|
...(entry.tags?.dbt?.length ? { tags: entry.tags } : {}),
|
||||||
...(entry.freshness?.dbt ? { freshness: entry.freshness } : {}),
|
...(entry.freshness?.dbt ? { freshness: entry.freshness } : {}),
|
||||||
...(entry.usage ? { usage: entry.usage } : {}),
|
...(entry.usage ? { usage: entry.usage } : {}),
|
||||||
};
|
};
|
||||||
|
toResolvedWire(source);
|
||||||
|
return source;
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeWs(s: string): string {
|
function normalizeWs(s: string): string {
|
||||||
|
|
@ -1331,6 +1421,7 @@ const COMPOSE_KNOWN_KEYS = new Set([
|
||||||
'descriptions',
|
'descriptions',
|
||||||
'grain',
|
'grain',
|
||||||
'columns',
|
'columns',
|
||||||
|
'column_overrides',
|
||||||
'joins',
|
'joins',
|
||||||
'measures',
|
'measures',
|
||||||
'segments',
|
'segments',
|
||||||
|
|
@ -1365,27 +1456,48 @@ export function composeOverlay(base: SemanticLayerSource, overlay: Record<string
|
||||||
result.usage = normalizedOverlay.usage as SemanticLayerSource['usage'];
|
result.usage = normalizedOverlay.usage as SemanticLayerSource['usage'];
|
||||||
}
|
}
|
||||||
|
|
||||||
// Filter out excluded columns
|
|
||||||
const excluded = new Set((normalizedOverlay.exclude_columns as string[] | undefined) ?? []);
|
const excluded = new Set((normalizedOverlay.exclude_columns as string[] | undefined) ?? []);
|
||||||
const baseColumns = result.columns.filter((c) => !excluded.has(c.name));
|
const columnOverrides = (normalizedOverlay.column_overrides as SemanticLayerColumnOverride[] | undefined) ?? [];
|
||||||
|
const overrideNames = columnOverrides.map((column) => column.name);
|
||||||
// Overlay columns matched by name merge onto the base column (overlay fields win, but
|
const conflictingOverrides = overrideNames.filter((name) => excluded.has(name));
|
||||||
// the base column's type/role/etc are preserved when the overlay omits them — dbt-style
|
if (conflictingOverrides.length > 0) {
|
||||||
// overlays often declare a column only to attach descriptions). New names append.
|
throw new ConflictingExcludeAndOverrideError(
|
||||||
const overlayColumns = (normalizedOverlay.columns as SemanticLayerSource['columns'] | undefined) ?? [];
|
`column_overrides conflict with exclude_columns for '${base.name}': ${conflictingOverrides.join(', ')}`,
|
||||||
const baseByName = new Map(baseColumns.map((c) => [c.name.toLowerCase(), c]));
|
);
|
||||||
const mergedAppended: SemanticLayerSource['columns'] = [];
|
|
||||||
const mergedByName = new Map<string, SemanticLayerSource['columns'][number]>();
|
|
||||||
for (const overlay of overlayColumns) {
|
|
||||||
const key = overlay.name.toLowerCase();
|
|
||||||
const base = baseByName.get(key);
|
|
||||||
if (base) {
|
|
||||||
mergedByName.set(key, mergeOverlayColumn(base, overlay));
|
|
||||||
} else {
|
|
||||||
mergedAppended.push(overlay);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
result.columns = [...baseColumns.map((c) => mergedByName.get(c.name.toLowerCase()) ?? c), ...mergedAppended];
|
|
||||||
|
const baseByLowerName = new Map(base.columns.map((column) => [column.name.toLowerCase(), column]));
|
||||||
|
const columnsByLowerName = new Map(
|
||||||
|
result.columns.filter((column) => !excluded.has(column.name)).map((column) => [column.name.toLowerCase(), column]),
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const override of columnOverrides) {
|
||||||
|
const key = override.name.toLowerCase();
|
||||||
|
const baseColumn = baseByLowerName.get(key);
|
||||||
|
if (!baseColumn) {
|
||||||
|
throw new UnknownColumnOverrideError(
|
||||||
|
`column '${override.name}' in column_overrides does not exist on manifest source '${base.name}'`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const baseDescriptions = baseColumn.descriptions ?? {};
|
||||||
|
const overrideDescriptions = override.descriptions ?? {};
|
||||||
|
const merged = { ...baseColumn, ...override };
|
||||||
|
if (Object.keys(baseDescriptions).length > 0 || Object.keys(overrideDescriptions).length > 0) {
|
||||||
|
merged.descriptions = { ...baseDescriptions, ...overrideDescriptions };
|
||||||
|
}
|
||||||
|
columnsByLowerName.set(key, merged);
|
||||||
|
}
|
||||||
|
|
||||||
|
const computedColumns = (normalizedOverlay.columns as SemanticLayerSource['columns'] | undefined) ?? [];
|
||||||
|
for (const column of computedColumns) {
|
||||||
|
if (baseByLowerName.has(column.name.toLowerCase())) {
|
||||||
|
throw new ColumnNameCollisionError(
|
||||||
|
`column '${column.name}' in columns patches a manifest column on '${base.name}' — move it to 'column_overrides:'`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
columnsByLowerName.set(column.name.toLowerCase(), column);
|
||||||
|
}
|
||||||
|
result.columns = [...columnsByLowerName.values()];
|
||||||
|
|
||||||
// Measures from overlay only
|
// Measures from overlay only
|
||||||
result.measures = (normalizedOverlay.measures as SemanticLayerSource['measures'] | undefined) ?? [];
|
result.measures = (normalizedOverlay.measures as SemanticLayerSource['measures'] | undefined) ?? [];
|
||||||
|
|
@ -1414,6 +1526,12 @@ export function composeOverlay(base: SemanticLayerSource, overlay: Record<string
|
||||||
const newJoins = overlayJoins.filter((j) => !existingKeys.has(`${j.to}::${normalizeWs(j.on)}`));
|
const newJoins = overlayJoins.filter((j) => !existingKeys.has(`${j.to}::${normalizeWs(j.on)}`));
|
||||||
result.joins = [...manifestJoins, ...newJoins];
|
result.joins = [...manifestJoins, ...newJoins];
|
||||||
|
|
||||||
|
const overlayParse = sourceOverlaySchema.safeParse(normalizedOverlay);
|
||||||
|
if (!overlayParse.success) {
|
||||||
|
const issues = overlayParse.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`).join('; ');
|
||||||
|
throw new ComposeContractError(`overlay for '${base.name}' violates the authoring schema: ${issues}`);
|
||||||
|
}
|
||||||
|
toResolvedWire(result);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1445,32 +1563,6 @@ function parseJoinOn(
|
||||||
return { fromColumn: leftCol, toColumn: rightCol };
|
return { fromColumn: leftCol, toColumn: rightCol };
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Merge an overlay column declaration onto a matching manifest column. Overlay fields
|
|
||||||
* win, except descriptions (plural) which merge per source key. Manifest values are
|
|
||||||
* preserved when the overlay omits them — this lets dbt/metabase emit description-only
|
|
||||||
* overlay column entries without redeclaring `type:` (which would have to mirror the
|
|
||||||
* scan column and rot when the schema changes).
|
|
||||||
*/
|
|
||||||
function mergeOverlayColumn(
|
|
||||||
base: SemanticLayerSource['columns'][number],
|
|
||||||
overlay: SemanticLayerSource['columns'][number],
|
|
||||||
): SemanticLayerSource['columns'][number] {
|
|
||||||
const merged: SemanticLayerSource['columns'][number] = { ...base, ...overlay };
|
|
||||||
if (!overlay.type && base.type) {
|
|
||||||
merged.type = base.type;
|
|
||||||
}
|
|
||||||
if (!overlay.role && base.role) {
|
|
||||||
merged.role = base.role;
|
|
||||||
}
|
|
||||||
const baseDescriptions = base.descriptions ?? {};
|
|
||||||
const overlayDescriptions = overlay.descriptions ?? {};
|
|
||||||
if (Object.keys(baseDescriptions).length > 0 || Object.keys(overlayDescriptions).length > 0) {
|
|
||||||
merged.descriptions = { ...baseDescriptions, ...overlayDescriptions };
|
|
||||||
}
|
|
||||||
return merged;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fill any blank `type`, `descriptions`, or `role` on the source's columns from the
|
* Fill any blank `type`, `descriptions`, or `role` on the source's columns from the
|
||||||
* matching manifest column (by name). Local values always win. Columns absent from
|
* matching manifest column (by name). Local values always win. Columns absent from
|
||||||
|
|
@ -1503,5 +1595,7 @@ export function enrichColumnsFromManifest(
|
||||||
}
|
}
|
||||||
return merged;
|
return merged;
|
||||||
});
|
});
|
||||||
return { ...source, columns: enrichedColumns };
|
const enriched = { ...source, columns: enrichedColumns };
|
||||||
|
toResolvedWire(enriched);
|
||||||
|
return enriched;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ import { SlDiscoverTool } from './sl-discover.tool.js';
|
||||||
function makeTool() {
|
function makeTool() {
|
||||||
const semanticLayerService = {
|
const semanticLayerService = {
|
||||||
listConnectionIdsWithNames: vi.fn(async () => [] as Array<{ id: string; name: string; connectionType: string }>),
|
listConnectionIdsWithNames: vi.fn(async () => [] as Array<{ id: string; name: string; connectionType: string }>),
|
||||||
loadAllSources: vi.fn(async () => [] as SemanticLayerSource[]),
|
loadAllSources: vi.fn(async () => ({ sources: [] as SemanticLayerSource[], loadErrors: [] })),
|
||||||
};
|
};
|
||||||
const slSearchService = {
|
const slSearchService = {
|
||||||
search: vi.fn(async () => []),
|
search: vi.fn(async () => []),
|
||||||
|
|
@ -53,7 +53,8 @@ describe('SlDiscoverTool - session-scoped reads', () => {
|
||||||
listConnectionIdsWithNames: vi.fn().mockResolvedValue([
|
listConnectionIdsWithNames: vi.fn().mockResolvedValue([
|
||||||
{ id: 'warehouse', name: 'warehouse', connectionType: 'postgres' },
|
{ id: 'warehouse', name: 'warehouse', connectionType: 'postgres' },
|
||||||
]),
|
]),
|
||||||
loadAllSources: vi.fn().mockResolvedValue([
|
loadAllSources: vi.fn().mockResolvedValue({
|
||||||
|
sources: [
|
||||||
{
|
{
|
||||||
name: 'orders',
|
name: 'orders',
|
||||||
table: 'public.orders',
|
table: 'public.orders',
|
||||||
|
|
@ -62,7 +63,9 @@ describe('SlDiscoverTool - session-scoped reads', () => {
|
||||||
measures: [],
|
measures: [],
|
||||||
joins: [],
|
joins: [],
|
||||||
},
|
},
|
||||||
]),
|
],
|
||||||
|
loadErrors: [],
|
||||||
|
}),
|
||||||
};
|
};
|
||||||
|
|
||||||
const result = await tool.call({}, makeContext({ session: makeSession(sessionSemanticLayerService) }));
|
const result = await tool.call({}, makeContext({ session: makeSession(sessionSemanticLayerService) }));
|
||||||
|
|
|
||||||
|
|
@ -101,7 +101,7 @@ Use this to understand what data is available before querying through the semant
|
||||||
// If inspecting a specific source — show the SL interface (columns, measures, joins)
|
// If inspecting a specific source — show the SL interface (columns, measures, joins)
|
||||||
// without the raw SQL. Use `sl_read_source` to see the full YAML including SQL.
|
// without the raw SQL. Use `sl_read_source` to see the full YAML including SQL.
|
||||||
if (sourceName) {
|
if (sourceName) {
|
||||||
const sources = await semanticLayerService.loadAllSources(connectionId);
|
const { sources } = await semanticLayerService.loadAllSources(connectionId);
|
||||||
const source = sources.find((s) => s.name === sourceName);
|
const source = sources.find((s) => s.name === sourceName);
|
||||||
if (!source) {
|
if (!source) {
|
||||||
return {
|
return {
|
||||||
|
|
@ -151,7 +151,7 @@ Use this to understand what data is available before querying through the semant
|
||||||
// Load sources from all connections in parallel
|
// Load sources from all connections in parallel
|
||||||
const results = await Promise.all(
|
const results = await Promise.all(
|
||||||
connections.map(async (conn) => {
|
connections.map(async (conn) => {
|
||||||
const sources = await semanticLayerService.loadAllSources(conn.id);
|
const { sources } = await semanticLayerService.loadAllSources(conn.id);
|
||||||
let filtered = sources;
|
let filtered = sources;
|
||||||
if (query) {
|
if (query) {
|
||||||
filtered = await this.filterByQuery(conn.id, sources, query);
|
filtered = await this.filterByQuery(conn.id, sources, query);
|
||||||
|
|
@ -213,7 +213,7 @@ Use this to understand what data is available before querying through the semant
|
||||||
connectionName: string,
|
connectionName: string,
|
||||||
query?: string,
|
query?: string,
|
||||||
): Promise<ToolOutput<SlDiscoverStructured>> {
|
): Promise<ToolOutput<SlDiscoverStructured>> {
|
||||||
const sources = await semanticLayerService.loadAllSources(connectionId);
|
const { sources } = await semanticLayerService.loadAllSources(connectionId);
|
||||||
|
|
||||||
if (sources.length === 0) {
|
if (sources.length === 0) {
|
||||||
return {
|
return {
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ function makeTool(overrides: any = {}) {
|
||||||
}),
|
}),
|
||||||
validateWithProposedSource: vi.fn().mockResolvedValue({ errors: [], warnings: [] }),
|
validateWithProposedSource: vi.fn().mockResolvedValue({ errors: [], warnings: [] }),
|
||||||
writeSource: vi.fn().mockResolvedValue({ commitHash: 'c1' }),
|
writeSource: vi.fn().mockResolvedValue({ commitHash: 'c1' }),
|
||||||
loadAllSources: vi.fn().mockResolvedValue([]),
|
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
|
||||||
deleteSource: vi.fn().mockResolvedValue(undefined),
|
deleteSource: vi.fn().mockResolvedValue(undefined),
|
||||||
isManifestBacked: vi.fn().mockResolvedValue(false),
|
isManifestBacked: vi.fn().mockResolvedValue(false),
|
||||||
...overrides.semanticLayerService,
|
...overrides.semanticLayerService,
|
||||||
|
|
@ -44,7 +44,7 @@ function makeSession(overrides: Partial<ToolSession> = {}): ToolSession {
|
||||||
}),
|
}),
|
||||||
validateWithProposedSource: vi.fn().mockResolvedValue({ errors: [], warnings: [] }),
|
validateWithProposedSource: vi.fn().mockResolvedValue({ errors: [], warnings: [] }),
|
||||||
writeSource: vi.fn().mockResolvedValue({ commitHash: 'c1' }),
|
writeSource: vi.fn().mockResolvedValue({ commitHash: 'c1' }),
|
||||||
loadAllSources: vi.fn().mockResolvedValue([]),
|
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
|
||||||
} as any,
|
} as any,
|
||||||
wikiService: {} as any,
|
wikiService: {} as any,
|
||||||
configService: {} as any,
|
configService: {} as any,
|
||||||
|
|
@ -191,9 +191,10 @@ describe('SlEditSourceTool — manifest-backed source without overlay', () => {
|
||||||
expect(joinedErrors).toContain('manifest');
|
expect(joinedErrors).toContain('manifest');
|
||||||
expect(joinedErrors).toContain('sl_write_source');
|
expect(joinedErrors).toContain('sl_write_source');
|
||||||
expect(joinedErrors).toContain('overlay');
|
expect(joinedErrors).toContain('overlay');
|
||||||
// Overlay shape: only name + measures/segments/description
|
// Overlay shape: name plus overlay-only fields.
|
||||||
expect(joinedErrors).toContain('measures');
|
expect(joinedErrors).toContain('measures');
|
||||||
expect(joinedErrors).toContain('segments');
|
expect(joinedErrors).toContain('segments');
|
||||||
|
expect(joinedErrors).toContain('column_overrides');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('still returns the plain "Source not found" error for truly-missing names', async () => {
|
it('still returns the plain "Source not found" error for truly-missing names', async () => {
|
||||||
|
|
|
||||||
|
|
@ -127,7 +127,8 @@ If no source exists yet, use sl_write_source instead — this tool will reject t
|
||||||
` - name: <measure_name>`,
|
` - name: <measure_name>`,
|
||||||
` expr: "<expression>"`,
|
` expr: "<expression>"`,
|
||||||
` description: "<what it measures>"`,
|
` description: "<what it measures>"`,
|
||||||
`Overlay shape: "name:" plus any of "measures:", "segments:", "descriptions:". Do NOT include "sql:", "table:", "grain:", "columns:", or "joins:" — those are inherited from the manifest.`,
|
`Overlay shape: "name:" plus any of "measures:", "segments:", "descriptions:", "joins:", "disable_joins:", "exclude_columns:", "column_overrides:", or computed-only "columns:" entries with expr + type.`,
|
||||||
|
`Do NOT include "sql:", "table:", "grain:", or base-table "columns:" — those are inherited from the manifest.`,
|
||||||
].join('\n'),
|
].join('\n'),
|
||||||
],
|
],
|
||||||
sourceName,
|
sourceName,
|
||||||
|
|
@ -181,7 +182,7 @@ If no source exists yet, use sl_write_source instead — this tool will reject t
|
||||||
const result = await semanticLayerService.writeSource(connectionId, source, author, authorEmail, commitMessage);
|
const result = await semanticLayerService.writeSource(connectionId, source, author, authorEmail, commitMessage);
|
||||||
|
|
||||||
if (!skipIndex) {
|
if (!skipIndex) {
|
||||||
const allSources = await semanticLayerService.loadAllSources(connectionId);
|
const { sources: allSources } = await semanticLayerService.loadAllSources(connectionId);
|
||||||
await this.slSearchService.indexSources(connectionId, allSources).catch(() => {});
|
await this.slSearchService.indexSources(connectionId, allSources).catch(() => {});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -34,7 +34,7 @@ describe('SlValidateTool — session-aware touched-set filtering', () => {
|
||||||
{ name: 'customers', table: 'x.customers', grain: ['id'], columns: [], joins: [], measures: [] },
|
{ name: 'customers', table: 'x.customers', grain: ['id'], columns: [], joins: [], measures: [] },
|
||||||
];
|
];
|
||||||
const serviceMock = {
|
const serviceMock = {
|
||||||
loadAllSources: vi.fn().mockResolvedValue(sources),
|
loadAllSources: vi.fn().mockResolvedValue({ sources, loadErrors: [] }),
|
||||||
validateSourcesForConnection: vi.fn().mockResolvedValue({
|
validateSourcesForConnection: vi.fn().mockResolvedValue({
|
||||||
errors: ['orders: missing join target', 'customers: invalid grain'],
|
errors: ['orders: missing join target', 'customers: invalid grain'],
|
||||||
warnings: ['orders: disconnected-components warning'],
|
warnings: ['orders: disconnected-components warning'],
|
||||||
|
|
|
||||||
|
|
@ -62,7 +62,7 @@ Checks: all join targets exist, grain is valid, no missing references.
|
||||||
|
|
||||||
const semanticLayerService = context.session?.semanticLayerService ?? this.semanticLayerService;
|
const semanticLayerService = context.session?.semanticLayerService ?? this.semanticLayerService;
|
||||||
|
|
||||||
const sources = await semanticLayerService.loadAllSources(connectionId);
|
const { sources } = await semanticLayerService.loadAllSources(connectionId);
|
||||||
if (sources.length === 0) {
|
if (sources.length === 0) {
|
||||||
return this.buildOutput(true, [], '(all)', {
|
return this.buildOutput(true, [], '(all)', {
|
||||||
validationErrors: ['No sources found for this connection.'],
|
validationErrors: ['No sources found for this connection.'],
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ function makeDeps(opts: { sourceYaml: string; executeQuery: ReturnType<typeof vi
|
||||||
isManifestBacked: vi.fn().mockResolvedValue(false),
|
isManifestBacked: vi.fn().mockResolvedValue(false),
|
||||||
listManifestSourceNames: vi.fn().mockResolvedValue([]),
|
listManifestSourceNames: vi.fn().mockResolvedValue([]),
|
||||||
loadSource: vi.fn().mockResolvedValue(null),
|
loadSource: vi.fn().mockResolvedValue(null),
|
||||||
loadAllSources: vi.fn().mockResolvedValue([]),
|
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
|
||||||
validatePhysicalTableReferences: vi.fn().mockResolvedValue([]),
|
validatePhysicalTableReferences: vi.fn().mockResolvedValue([]),
|
||||||
} as never,
|
} as never,
|
||||||
connections: {
|
connections: {
|
||||||
|
|
|
||||||
|
|
@ -88,8 +88,9 @@ export async function validateSingleSource(
|
||||||
errors.push(
|
errors.push(
|
||||||
`${sourceName}.yaml: standalone source shadows an existing manifest entry — ` +
|
`${sourceName}.yaml: standalone source shadows an existing manifest entry — ` +
|
||||||
`writing it as-is drops the manifest's columns and joins. ` +
|
`writing it as-is drops the manifest's columns and joins. ` +
|
||||||
`Remove "sql:", "table:", "grain:", "columns:", and "joins:" and keep only ` +
|
`Remove "sql:", "table:", "grain:", and base-table "columns:" and keep only ` +
|
||||||
`"name:" plus "measures:"/"segments:"/"descriptions:" to write an overlay ` +
|
`"name:" plus overlay fields such as "measures:", "segments:", "descriptions:", ` +
|
||||||
|
`"joins:", "column_overrides:", or computed-only "columns:" to write an overlay ` +
|
||||||
`that inherits the manifest schema. Call sl_read_source to inspect the existing source first.`,
|
`that inherits the manifest schema. Call sl_read_source to inspect the existing source first.`,
|
||||||
);
|
);
|
||||||
return { errors, warnings };
|
return { errors, warnings };
|
||||||
|
|
@ -108,7 +109,7 @@ export async function validateSingleSource(
|
||||||
}
|
}
|
||||||
if (errorPaths.has('columns')) {
|
if (errorPaths.has('columns')) {
|
||||||
warnings.push(
|
warnings.push(
|
||||||
`${sourceName}.yaml: hint — overlay columns must be computed: {name, expr, type}. Do NOT include base table columns.`,
|
`${sourceName}.yaml: hint — overlay columns must be computed: {name, expr, type}. Use column_overrides for manifest column descriptions or metadata.`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (errorPaths.has('measures')) {
|
if (errorPaths.has('measures')) {
|
||||||
|
|
@ -240,7 +241,8 @@ async function probeOverlayMeasures(
|
||||||
}
|
}
|
||||||
| undefined;
|
| undefined;
|
||||||
try {
|
try {
|
||||||
const all = await deps.semanticLayerService.loadAllSources(connectionId);
|
const { sources: all, loadErrors } = await deps.semanticLayerService.loadAllSources(connectionId);
|
||||||
|
errors.push(...loadErrors);
|
||||||
composed = all.find((s) => s.name === sourceName);
|
composed = all.find((s) => s.name === sourceName);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
errors.push(
|
errors.push(
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ function makeTool(overrides: Partial<Record<string, any>> = {}) {
|
||||||
listManifestSourceNames: vi.fn().mockResolvedValue(['ACCOUNTS', 'ORDERS']),
|
listManifestSourceNames: vi.fn().mockResolvedValue(['ACCOUNTS', 'ORDERS']),
|
||||||
isManifestBacked: vi.fn().mockResolvedValue(false),
|
isManifestBacked: vi.fn().mockResolvedValue(false),
|
||||||
loadSource: vi.fn().mockResolvedValue(null),
|
loadSource: vi.fn().mockResolvedValue(null),
|
||||||
loadAllSources: vi.fn().mockResolvedValue([]),
|
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
|
||||||
validateWithProposedSource: vi.fn().mockResolvedValue({ errors: [], warnings: [] }),
|
validateWithProposedSource: vi.fn().mockResolvedValue({ errors: [], warnings: [] }),
|
||||||
writeSource: vi.fn().mockResolvedValue({ commitHash: 'c1' }),
|
writeSource: vi.fn().mockResolvedValue({ commitHash: 'c1' }),
|
||||||
deleteSource: vi.fn().mockResolvedValue(undefined),
|
deleteSource: vi.fn().mockResolvedValue(undefined),
|
||||||
|
|
@ -59,7 +59,7 @@ describe('SlWriteSourceTool — session gating', () => {
|
||||||
actions: [],
|
actions: [],
|
||||||
semanticLayerService: {
|
semanticLayerService: {
|
||||||
loadSource: vi.fn().mockResolvedValue(null),
|
loadSource: vi.fn().mockResolvedValue(null),
|
||||||
loadAllSources: vi.fn().mockResolvedValue([]),
|
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
|
||||||
validateWithProposedSource: vi.fn().mockResolvedValue({ errors: [], warnings: [] }),
|
validateWithProposedSource: vi.fn().mockResolvedValue({ errors: [], warnings: [] }),
|
||||||
writeSource: vi.fn().mockResolvedValue({ commitHash: 'c1' }),
|
writeSource: vi.fn().mockResolvedValue({ commitHash: 'c1' }),
|
||||||
deleteSource: vi.fn().mockResolvedValue(undefined),
|
deleteSource: vi.fn().mockResolvedValue(undefined),
|
||||||
|
|
@ -213,7 +213,7 @@ describe('SlWriteSourceTool — session gating', () => {
|
||||||
ingest: { runId: 'run-1', jobId: 'job-1', syncId: 'sync-1', sourceKey: 'metabase' },
|
ingest: { runId: 'run-1', jobId: 'job-1', syncId: 'sync-1', sourceKey: 'metabase' },
|
||||||
semanticLayerService: {
|
semanticLayerService: {
|
||||||
loadSource: vi.fn().mockResolvedValue(null),
|
loadSource: vi.fn().mockResolvedValue(null),
|
||||||
loadAllSources: vi.fn().mockResolvedValue([]),
|
loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
|
||||||
validateWithProposedSource: vi.fn().mockResolvedValue({ errors: [], warnings: [] }),
|
validateWithProposedSource: vi.fn().mockResolvedValue({ errors: [], warnings: [] }),
|
||||||
writeSource: vi.fn().mockResolvedValue({ commitHash: 'c1' }),
|
writeSource: vi.fn().mockResolvedValue({ commitHash: 'c1' }),
|
||||||
deleteSource: vi.fn().mockResolvedValue(undefined),
|
deleteSource: vi.fn().mockResolvedValue(undefined),
|
||||||
|
|
|
||||||
|
|
@ -23,7 +23,9 @@ const slWriteSourceInputSchema = z.object({
|
||||||
.describe('Name of the source to create, edit, or delete'),
|
.describe('Name of the source to create, edit, or delete'),
|
||||||
source: sourceInputSchema
|
source: sourceInputSchema
|
||||||
.optional()
|
.optional()
|
||||||
.describe('Source definition (standalone with table/sql) or overlay (measures, computed columns, etc.)'),
|
.describe(
|
||||||
|
'Source definition (standalone with table/sql) or overlay (measures, column_overrides, computed columns, etc.)',
|
||||||
|
),
|
||||||
delete: z.boolean().optional().describe('Set to true to delete this source entirely'),
|
delete: z.boolean().optional().describe('Set to true to delete this source entirely'),
|
||||||
rawPaths: z
|
rawPaths: z
|
||||||
.array(z.string().min(1))
|
.array(z.string().min(1))
|
||||||
|
|
@ -73,7 +75,8 @@ If the source already exists, this tool will overwrite it with the new definitio
|
||||||
- table: For physical table/view sources (e.g., "public.orders"). Mutually exclusive with sql.
|
- table: For physical table/view sources (e.g., "public.orders"). Mutually exclusive with sql.
|
||||||
- sql: For SQL-based sources (the SQL query). Mutually exclusive with table.
|
- sql: For SQL-based sources (the SQL query). Mutually exclusive with table.
|
||||||
- grain: What one row represents (e.g., ["id"], ["customer_id", "product_id"])
|
- grain: What one row represents (e.g., ["id"], ["customer_id", "product_id"])
|
||||||
- columns: All columns with type (string/number/time/boolean) and optional descriptions
|
- columns: All columns with type (string/number/time/boolean) and optional descriptions. On overlays, columns are computed-only and require expr + type.
|
||||||
|
- column_overrides: Overlay-only metadata patches for existing manifest columns (descriptions, role, visibility, constraints, enum_values, tests). Do not include type or expr.
|
||||||
- joins: Relationships to other sources (to, on, relationship: many_to_one/one_to_many/one_to_one)
|
- joins: Relationships to other sources (to, on, relationship: many_to_one/one_to_many/one_to_one)
|
||||||
- measures: Pre-defined aggregations (name, expr like "sum(amount)", optional filter, optional segments — bare names of segments defined on the same source, optional description)
|
- measures: Pre-defined aggregations (name, expr like "sum(amount)", optional filter, optional segments — bare names of segments defined on the same source, optional description)
|
||||||
- segments: Named, reusable boolean predicates scoped to this source (name, expr — a SQL boolean over this source's columns, optional description). A measure references one with \`segments: [name]\`; a query references one with the dotted form \`source.segment_name\`. Use when the same predicate appears on 3+ measures — e.g. extract \`is_paid = true and is_refunded = '0'\` as \`segments: [{name: paid_non_refunded, expr: "..."}]\` and have each measure use \`segments: [paid_non_refunded]\` instead of re-typing the predicate inside \`sum(case when ... then x end)\`. Segments are predicates only — they cannot be selected as dimensions or grouped by; if you need to group by the predicate, add a \`columns[]\` entry instead.
|
- segments: Named, reusable boolean predicates scoped to this source (name, expr — a SQL boolean over this source's columns, optional description). A measure references one with \`segments: [name]\`; a query references one with the dotted form \`source.segment_name\`. Use when the same predicate appears on 3+ measures — e.g. extract \`is_paid = true and is_refunded = '0'\` as \`segments: [{name: paid_non_refunded, expr: "..."}]\` and have each measure use \`segments: [paid_non_refunded]\` instead of re-typing the predicate inside \`sum(case when ... then x end)\`. Segments are predicates only — they cannot be selected as dimensions or grouped by; if you need to group by the predicate, add a \`columns[]\` entry instead.
|
||||||
|
|
@ -113,7 +116,7 @@ Do NOT join back to a table that the SQL already aggregates from if the grain co
|
||||||
try {
|
try {
|
||||||
await semanticLayerService.deleteSource(connectionId, sourceName, author, authorEmail);
|
await semanticLayerService.deleteSource(connectionId, sourceName, author, authorEmail);
|
||||||
if (!skipIndex) {
|
if (!skipIndex) {
|
||||||
const allSources = await semanticLayerService.loadAllSources(connectionId);
|
const { sources: allSources } = await semanticLayerService.loadAllSources(connectionId);
|
||||||
await this.slSearchService.indexSources(connectionId, allSources).catch(() => {});
|
await this.slSearchService.indexSources(connectionId, allSources).catch(() => {});
|
||||||
}
|
}
|
||||||
if (context.session) {
|
if (context.session) {
|
||||||
|
|
@ -210,7 +213,7 @@ Do NOT join back to a table that the SQL already aggregates from if the grain co
|
||||||
);
|
);
|
||||||
|
|
||||||
if (!skipIndex) {
|
if (!skipIndex) {
|
||||||
const allSources = await semanticLayerService.loadAllSources(connectionId);
|
const { sources: allSources } = await semanticLayerService.loadAllSources(connectionId);
|
||||||
await this.slSearchService.indexSources(connectionId, allSources).catch(() => {});
|
await this.slSearchService.indexSources(connectionId, allSources).catch(() => {});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -317,8 +320,9 @@ Do NOT join back to a table that the SQL already aggregates from if the grain co
|
||||||
`Error: cannot write "${sourceName}" as a standalone source — a manifest entry with that name already exists.`,
|
`Error: cannot write "${sourceName}" as a standalone source — a manifest entry with that name already exists.`,
|
||||||
` Writing standalone would drop the manifest's columns and joins, leaving only what you list here.`,
|
` Writing standalone would drop the manifest's columns and joins, leaving only what you list here.`,
|
||||||
`To add measures/segments on top of the manifest, rewrite this YAML as an overlay:`,
|
`To add measures/segments on top of the manifest, rewrite this YAML as an overlay:`,
|
||||||
` - Remove "sql:", "table:", "grain:", "columns:", and "joins:".`,
|
` - Remove "sql:", "table:", "grain:", and base-table "columns:".`,
|
||||||
` - Keep only "name:", plus "measures:", "segments:", and/or "descriptions:".`,
|
` - Keep "name:" plus "measures:", "segments:", "descriptions:", "joins:", "disable_joins:",`,
|
||||||
|
` "exclude_columns:", "column_overrides:", and/or computed-only "columns:" entries with expr + type.`,
|
||||||
` - The manifest's schema is inherited automatically.`,
|
` - The manifest's schema is inherited automatically.`,
|
||||||
`If you really need a different base table, use a different source name.`,
|
`If you really need a different base table, use a different source name.`,
|
||||||
].join('\n');
|
].join('\n');
|
||||||
|
|
|
||||||
|
|
@ -47,6 +47,32 @@ export interface SemanticLayerSource {
|
||||||
usage?: TableUsageOutput;
|
usage?: TableUsageOutput;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type SemanticLayerColumn = SemanticLayerSource['columns'][number];
|
||||||
|
type SemanticLayerJoin = SemanticLayerSource['joins'][number];
|
||||||
|
|
||||||
|
export interface SemanticLayerColumnOverride {
|
||||||
|
name: string;
|
||||||
|
role?: string;
|
||||||
|
visibility?: string;
|
||||||
|
descriptions?: Record<string, string>;
|
||||||
|
constraints?: { dbt?: { not_null?: boolean; unique?: boolean } };
|
||||||
|
enum_values?: { dbt?: string[] };
|
||||||
|
tests?: {
|
||||||
|
dbt?: Array<{ name: string; package: string; kwargs?: Record<string, unknown> }>;
|
||||||
|
dbt_by_package?: Record<string, string[]>;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ResolvedSemanticLayerSource = Omit<
|
||||||
|
SemanticLayerSource,
|
||||||
|
'inherits_columns_from' | 'usage' | 'joins'
|
||||||
|
> & {
|
||||||
|
table?: string;
|
||||||
|
sql?: string;
|
||||||
|
columns: Array<SemanticLayerColumn & { type: string }>;
|
||||||
|
joins: Array<Omit<SemanticLayerJoin, 'source'>>;
|
||||||
|
};
|
||||||
|
|
||||||
export interface SemanticLayerQueryInput {
|
export interface SemanticLayerQueryInput {
|
||||||
measures: Array<string | { expr: string; name: string }>;
|
measures: Array<string | { expr: string; name: string }>;
|
||||||
dimensions: Array<string | { field: string; granularity?: string }>;
|
dimensions: Array<string | { field: string; granularity?: string }>;
|
||||||
|
|
|
||||||
3
pnpm-lock.yaml
generated
3
pnpm-lock.yaml
generated
|
|
@ -376,6 +376,9 @@ importers:
|
||||||
'@vitest/coverage-v8':
|
'@vitest/coverage-v8':
|
||||||
specifier: ^4.1.6
|
specifier: ^4.1.6
|
||||||
version: 4.1.6(vitest@4.1.6)
|
version: 4.1.6(vitest@4.1.6)
|
||||||
|
ajv:
|
||||||
|
specifier: 8.20.0
|
||||||
|
version: 8.20.0
|
||||||
typescript:
|
typescript:
|
||||||
specifier: ^6.0.3
|
specifier: ^6.0.3
|
||||||
version: 6.0.3
|
version: 6.0.3
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,22 @@
|
||||||
from semantic_layer.cli import main
|
from __future__ import annotations
|
||||||
|
|
||||||
main()
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from semantic_layer.cli import main as cli_main
|
||||||
|
from semantic_layer.models import SourceDefinition
|
||||||
|
|
||||||
|
|
||||||
|
def dump_schema() -> None:
|
||||||
|
json.dump(
|
||||||
|
SourceDefinition.model_json_schema(), sys.stdout, indent=2, sort_keys=True
|
||||||
|
)
|
||||||
|
sys.stdout.write("\n")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
if len(sys.argv) > 1 and sys.argv[1] in {"dump-schema", "schema"}:
|
||||||
|
sys.argv.pop(1)
|
||||||
|
dump_schema()
|
||||||
|
else:
|
||||||
|
cli_main()
|
||||||
|
|
|
||||||
|
|
@ -87,18 +87,23 @@ class SourceLoader:
|
||||||
sources[name] = SourceDefinition(**data)
|
sources[name] = SourceDefinition(**data)
|
||||||
else:
|
else:
|
||||||
# Overlay — validate and compose with matching manifest entry
|
# Overlay — validate and compose with matching manifest entry
|
||||||
errors = validate_overlay(data)
|
|
||||||
if errors:
|
|
||||||
raise ValueError(
|
|
||||||
f"Invalid overlay '{name}' in {path}: {'; '.join(errors)}"
|
|
||||||
)
|
|
||||||
base = sources.get(name)
|
base = sources.get(name)
|
||||||
if base:
|
if base:
|
||||||
|
errors = validate_overlay(data, {c.name for c in base.columns})
|
||||||
|
if errors:
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid overlay '{name}' in {path}: {'; '.join(errors)}"
|
||||||
|
)
|
||||||
(
|
(
|
||||||
sources[name],
|
sources[name],
|
||||||
description_sources[name],
|
description_sources[name],
|
||||||
) = self._compose(base, data, description_sources.get(name))
|
) = self._compose(base, data, description_sources.get(name))
|
||||||
else:
|
else:
|
||||||
|
errors = validate_overlay(data)
|
||||||
|
if errors:
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid overlay '{name}' in {path}: {'; '.join(errors)}"
|
||||||
|
)
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Orphan overlay '%s' in %s: no matching manifest entry, skipping",
|
"Orphan overlay '%s' in %s: no matching manifest entry, skipping",
|
||||||
name,
|
name,
|
||||||
|
|
@ -149,12 +154,55 @@ class SourceLoader:
|
||||||
description_sources or None,
|
description_sources or None,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Filter columns
|
excluded = set(overlay.get("exclude_columns", []))
|
||||||
|
overrides = overlay.get("column_overrides", [])
|
||||||
|
override_names = {override.get("name") for override in overrides}
|
||||||
|
conflicts = sorted(name for name in override_names if name in excluded)
|
||||||
|
if conflicts:
|
||||||
|
raise ValueError(
|
||||||
|
"column_overrides conflict with exclude_columns: "
|
||||||
|
+ ", ".join(conflicts)
|
||||||
|
)
|
||||||
|
|
||||||
|
base_by_name = {column.name: column for column in base.columns}
|
||||||
|
|
||||||
|
for override in overrides:
|
||||||
|
name = override.get("name")
|
||||||
|
base_column = base_by_name.get(name)
|
||||||
|
if base_column is None:
|
||||||
|
raise ValueError(
|
||||||
|
f"column '{name}' in column_overrides does not exist on manifest source '{base.name}'"
|
||||||
|
)
|
||||||
|
|
||||||
excluded = set(overlay.get("exclude_columns", []))
|
excluded = set(overlay.get("exclude_columns", []))
|
||||||
source.columns = [c for c in source.columns if c.name not in excluded]
|
source.columns = [c for c in source.columns if c.name not in excluded]
|
||||||
|
|
||||||
# Append computed columns (overlay columns with expr)
|
columns_by_name = {column.name: column for column in source.columns}
|
||||||
|
|
||||||
|
for override in overrides:
|
||||||
|
name = override["name"]
|
||||||
|
base_column = base_by_name[name]
|
||||||
|
merged = base_column.model_dump(mode="python", exclude_none=True)
|
||||||
|
base_descriptions = merged.get("descriptions") or {}
|
||||||
|
override_data = dict(override)
|
||||||
|
override_descriptions = override_data.get("descriptions") or {}
|
||||||
|
merged.update(override_data)
|
||||||
|
if base_descriptions or override_descriptions:
|
||||||
|
merged["descriptions"] = {
|
||||||
|
**base_descriptions,
|
||||||
|
**override_descriptions,
|
||||||
|
}
|
||||||
|
columns_by_name[name] = SourceColumn(**merged)
|
||||||
|
source.columns = list(columns_by_name.values())
|
||||||
|
|
||||||
|
# Append computed columns. Manifest column names cannot be reused here;
|
||||||
|
# use column_overrides for metadata patches.
|
||||||
for col in overlay.get("columns", []):
|
for col in overlay.get("columns", []):
|
||||||
|
name = col.get("name")
|
||||||
|
if name in base_by_name:
|
||||||
|
raise ValueError(
|
||||||
|
f"column '{name}' in columns patches a manifest column on '{base.name}' — move it to 'column_overrides:'"
|
||||||
|
)
|
||||||
source.columns.append(SourceColumn(**col))
|
source.columns.append(SourceColumn(**col))
|
||||||
|
|
||||||
# Set measures
|
# Set measures
|
||||||
|
|
@ -181,6 +229,11 @@ class SourceLoader:
|
||||||
]
|
]
|
||||||
source.joins = manifest_joins + new_joins
|
source.joins = manifest_joins + new_joins
|
||||||
|
|
||||||
|
if not source.table and not source.sql:
|
||||||
|
raise ValueError("resolved source must have 'table' or 'sql'")
|
||||||
|
if source.table and source.sql:
|
||||||
|
raise ValueError("'table' and 'sql' are mutually exclusive")
|
||||||
|
|
||||||
return source, (description_sources or None)
|
return source, (description_sources or None)
|
||||||
|
|
||||||
def _validate_cross_references(self, sources: dict[str, SourceDefinition]) -> None:
|
def _validate_cross_references(self, sources: dict[str, SourceDefinition]) -> None:
|
||||||
|
|
|
||||||
|
|
@ -143,7 +143,9 @@ class Manifest(BaseModel):
|
||||||
# ── Projection ──────────────────────────────────────────────────────
|
# ── Projection ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
def validate_overlay(data: dict) -> list[str]:
|
def validate_overlay(
|
||||||
|
data: dict, manifest_column_names: set[str] | None = None
|
||||||
|
) -> list[str]:
|
||||||
"""Validate that overlay data doesn't contain structural fields.
|
"""Validate that overlay data doesn't contain structural fields.
|
||||||
|
|
||||||
Returns a list of error messages (empty if valid).
|
Returns a list of error messages (empty if valid).
|
||||||
|
|
@ -162,11 +164,26 @@ def validate_overlay(data: dict) -> list[str]:
|
||||||
errors.append(
|
errors.append(
|
||||||
f"Overlay column '{col.get('name', '?')}' must use 'descriptions'"
|
f"Overlay column '{col.get('name', '?')}' must use 'descriptions'"
|
||||||
)
|
)
|
||||||
if "type" in col and "expr" not in col:
|
if "expr" not in col:
|
||||||
errors.append(
|
errors.append(
|
||||||
f"Overlay column '{col.get('name', '?')}' specifies 'type' without 'expr' "
|
f"Overlay column '{col.get('name', '?')}' in 'columns' must define "
|
||||||
f"(structural types are inherited from manifest — only computed columns may specify a type)"
|
f"'expr' and 'type' (use 'column_overrides' to patch manifest columns)"
|
||||||
)
|
)
|
||||||
|
if "type" not in col:
|
||||||
|
errors.append(
|
||||||
|
f"Overlay column '{col.get('name', '?')}' in 'columns' must define "
|
||||||
|
f"'type' and 'expr' (use 'column_overrides' to patch manifest columns)"
|
||||||
|
)
|
||||||
|
for col in data.get("column_overrides", []):
|
||||||
|
name = col.get("name", "?")
|
||||||
|
if "description" in col:
|
||||||
|
errors.append(f"Column override '{name}' must use 'descriptions'")
|
||||||
|
if "type" in col:
|
||||||
|
errors.append(f"Column override '{name}' must not contain 'type'")
|
||||||
|
if "expr" in col:
|
||||||
|
errors.append(f"Column override '{name}' must not contain 'expr'")
|
||||||
|
if manifest_column_names is not None and name not in manifest_column_names:
|
||||||
|
errors.append(f"Column override '{name}' does not match a manifest column")
|
||||||
return errors
|
return errors
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ from __future__ import annotations
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Any, Literal
|
from typing import Any, Literal
|
||||||
|
|
||||||
from pydantic import BaseModel, Field, model_validator
|
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||||
|
|
||||||
|
|
||||||
# ── Source Definition Models ──────────────────────────────────────────
|
# ── Source Definition Models ──────────────────────────────────────────
|
||||||
|
|
@ -105,6 +105,8 @@ class DefaultTimeDimensionDbt(BaseModel):
|
||||||
|
|
||||||
|
|
||||||
class SourceDefinition(BaseModel):
|
class SourceDefinition(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
name: str
|
name: str
|
||||||
description: str | None = None
|
description: str | None = None
|
||||||
descriptions: dict[str, str] | None = None
|
descriptions: dict[str, str] | None = None
|
||||||
|
|
@ -123,6 +125,8 @@ class SourceDefinition(BaseModel):
|
||||||
def validate_source(self) -> SourceDefinition:
|
def validate_source(self) -> SourceDefinition:
|
||||||
if self.description is None:
|
if self.description is None:
|
||||||
self.description = _resolve_description_map(self.descriptions)
|
self.description = _resolve_description_map(self.descriptions)
|
||||||
|
if not self.table and not self.sql:
|
||||||
|
raise ValueError("resolved source must have 'table' or 'sql'")
|
||||||
if self.table and self.sql:
|
if self.table and self.sql:
|
||||||
raise ValueError("'table' and 'sql' are mutually exclusive")
|
raise ValueError("'table' and 'sql' are mutually exclusive")
|
||||||
if not self.grain:
|
if not self.grain:
|
||||||
|
|
|
||||||
|
|
@ -148,11 +148,21 @@ class TestLoaderEdgeCases:
|
||||||
with open(Path(tmpdir) / "test.yaml", "w") as f:
|
with open(Path(tmpdir) / "test.yaml", "w") as f:
|
||||||
yaml.dump(data, f)
|
yaml.dump(data, f)
|
||||||
loader = SourceLoader(tmpdir)
|
loader = SourceLoader(tmpdir)
|
||||||
try:
|
with pytest.raises(Exception, match="unknown_field"):
|
||||||
sources = loader.load_all()
|
loader.load_all()
|
||||||
assert "test" in sources
|
|
||||||
except Exception:
|
def test_source_requires_table_or_sql(self):
|
||||||
pass
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
data = {
|
||||||
|
"name": "test",
|
||||||
|
"grain": ["id"],
|
||||||
|
"columns": [{"name": "id", "type": "number"}],
|
||||||
|
}
|
||||||
|
with open(Path(tmpdir) / "test.yaml", "w") as f:
|
||||||
|
yaml.dump(data, f)
|
||||||
|
loader = SourceLoader(tmpdir)
|
||||||
|
with pytest.raises(Exception, match="table.*sql"):
|
||||||
|
loader.load_file(Path(tmpdir) / "test.yaml")
|
||||||
|
|
||||||
def test_subdirectory_sources(self):
|
def test_subdirectory_sources(self):
|
||||||
with tempfile.TemporaryDirectory() as tmpdir:
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
|
|
||||||
|
|
@ -205,12 +205,15 @@ class TestValidateOverlay:
|
||||||
"descriptions": {"user": "Revenue-bearing orders"},
|
"descriptions": {"user": "Revenue-bearing orders"},
|
||||||
"grain": ["id"],
|
"grain": ["id"],
|
||||||
"measures": [{"name": "revenue", "expr": "sum(total)"}],
|
"measures": [{"name": "revenue", "expr": "sum(total)"}],
|
||||||
|
"column_overrides": [
|
||||||
|
{"name": "status", "descriptions": {"user": "Order lifecycle status"}}
|
||||||
|
],
|
||||||
"columns": [
|
"columns": [
|
||||||
{"name": "is_high_value", "expr": "total > 1000", "type": "boolean"}
|
{"name": "is_high_value", "expr": "total > 1000", "type": "boolean"}
|
||||||
],
|
],
|
||||||
"exclude_columns": ["status"],
|
"exclude_columns": ["status"],
|
||||||
}
|
}
|
||||||
errors = validate_overlay(data)
|
errors = validate_overlay(data, {"status", "total"})
|
||||||
assert errors == []
|
assert errors == []
|
||||||
|
|
||||||
def test_validate_overlay_rejects_table(self):
|
def test_validate_overlay_rejects_table(self):
|
||||||
|
|
@ -225,14 +228,13 @@ class TestValidateOverlay:
|
||||||
assert len(errors) == 1
|
assert len(errors) == 1
|
||||||
assert "sql" in errors[0].lower()
|
assert "sql" in errors[0].lower()
|
||||||
|
|
||||||
def test_validate_overlay_rejects_type_without_expr(self):
|
def test_validate_overlay_rejects_column_without_expr(self):
|
||||||
data = {
|
data = {
|
||||||
"name": "orders",
|
"name": "orders",
|
||||||
"columns": [{"name": "status", "type": "string"}],
|
"columns": [{"name": "status", "type": "string"}],
|
||||||
}
|
}
|
||||||
errors = validate_overlay(data)
|
errors = validate_overlay(data)
|
||||||
assert len(errors) == 1
|
assert len(errors) == 1
|
||||||
assert "type" in errors[0].lower()
|
|
||||||
assert "expr" in errors[0].lower()
|
assert "expr" in errors[0].lower()
|
||||||
|
|
||||||
def test_validate_overlay_allows_type_with_expr(self):
|
def test_validate_overlay_allows_type_with_expr(self):
|
||||||
|
|
@ -243,6 +245,33 @@ class TestValidateOverlay:
|
||||||
errors = validate_overlay(data)
|
errors = validate_overlay(data)
|
||||||
assert errors == []
|
assert errors == []
|
||||||
|
|
||||||
|
def test_validate_overlay_rejects_column_override_structural_fields(self):
|
||||||
|
data = {
|
||||||
|
"name": "orders",
|
||||||
|
"column_overrides": [
|
||||||
|
{
|
||||||
|
"name": "status",
|
||||||
|
"description": "Status",
|
||||||
|
"type": "string",
|
||||||
|
"expr": "status",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
errors = validate_overlay(data, {"status"})
|
||||||
|
assert len(errors) == 3
|
||||||
|
assert "descriptions" in errors[0]
|
||||||
|
assert "type" in errors[1]
|
||||||
|
assert "expr" in errors[2]
|
||||||
|
|
||||||
|
def test_validate_overlay_rejects_unknown_column_override(self):
|
||||||
|
data = {
|
||||||
|
"name": "orders",
|
||||||
|
"column_overrides": [{"name": "missing", "descriptions": {"user": "Nope"}}],
|
||||||
|
}
|
||||||
|
errors = validate_overlay(data, {"status"})
|
||||||
|
assert len(errors) == 1
|
||||||
|
assert "does not match" in errors[0]
|
||||||
|
|
||||||
|
|
||||||
# ── Two-Tier Loading Tests ─────────────────────────────────────────
|
# ── Two-Tier Loading Tests ─────────────────────────────────────────
|
||||||
|
|
||||||
|
|
@ -502,6 +531,77 @@ class TestTwoTierLoading:
|
||||||
assert hv.expr == "total > 1000"
|
assert hv.expr == "total > 1000"
|
||||||
assert hv.type == "boolean"
|
assert hv.type == "boolean"
|
||||||
|
|
||||||
|
def test_overlay_column_overrides_patch_manifest_columns(self, tmp_path: Path):
|
||||||
|
schema_dir = tmp_path / "_schema"
|
||||||
|
_write_yaml(schema_dir / "public.yaml", _manifest_tables())
|
||||||
|
|
||||||
|
overlay = {
|
||||||
|
"name": "orders",
|
||||||
|
"column_overrides": [
|
||||||
|
{"name": "status", "descriptions": {"user": "Order lifecycle status"}}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
_write_yaml(tmp_path / "orders.yaml", overlay)
|
||||||
|
_write_yaml(tmp_path / "customers.yaml", {"name": "customers"})
|
||||||
|
|
||||||
|
loader = SourceLoader(tmp_path)
|
||||||
|
sources = loader.load_all()
|
||||||
|
|
||||||
|
status = next(c for c in sources["orders"].columns if c.name == "status")
|
||||||
|
assert status.type == "string"
|
||||||
|
assert status.description == "Order lifecycle status"
|
||||||
|
assert status.descriptions == {"user": "Order lifecycle status"}
|
||||||
|
|
||||||
|
def test_overlay_rejects_unknown_column_override(self, tmp_path: Path):
|
||||||
|
schema_dir = tmp_path / "_schema"
|
||||||
|
_write_yaml(schema_dir / "public.yaml", _manifest_tables())
|
||||||
|
|
||||||
|
overlay = {
|
||||||
|
"name": "orders",
|
||||||
|
"column_overrides": [
|
||||||
|
{"name": "missing", "descriptions": {"user": "No such column"}}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
_write_yaml(tmp_path / "orders.yaml", overlay)
|
||||||
|
_write_yaml(tmp_path / "customers.yaml", {"name": "customers"})
|
||||||
|
|
||||||
|
loader = SourceLoader(tmp_path)
|
||||||
|
with pytest.raises(ValueError, match="Column override 'missing'"):
|
||||||
|
loader.load_all()
|
||||||
|
|
||||||
|
def test_overlay_rejects_computed_column_name_collision(self, tmp_path: Path):
|
||||||
|
schema_dir = tmp_path / "_schema"
|
||||||
|
_write_yaml(schema_dir / "public.yaml", _manifest_tables())
|
||||||
|
|
||||||
|
overlay = {
|
||||||
|
"name": "orders",
|
||||||
|
"columns": [{"name": "status", "type": "string", "expr": "status"}],
|
||||||
|
}
|
||||||
|
_write_yaml(tmp_path / "orders.yaml", overlay)
|
||||||
|
_write_yaml(tmp_path / "customers.yaml", {"name": "customers"})
|
||||||
|
|
||||||
|
loader = SourceLoader(tmp_path)
|
||||||
|
with pytest.raises(ValueError, match="move it to 'column_overrides:'"):
|
||||||
|
loader.load_all()
|
||||||
|
|
||||||
|
def test_overlay_rejects_exclude_override_conflict(self, tmp_path: Path):
|
||||||
|
schema_dir = tmp_path / "_schema"
|
||||||
|
_write_yaml(schema_dir / "public.yaml", _manifest_tables())
|
||||||
|
|
||||||
|
overlay = {
|
||||||
|
"name": "orders",
|
||||||
|
"exclude_columns": ["status"],
|
||||||
|
"column_overrides": [
|
||||||
|
{"name": "status", "descriptions": {"user": "Hidden status"}}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
_write_yaml(tmp_path / "orders.yaml", overlay)
|
||||||
|
_write_yaml(tmp_path / "customers.yaml", {"name": "customers"})
|
||||||
|
|
||||||
|
loader = SourceLoader(tmp_path)
|
||||||
|
with pytest.raises(ValueError, match="conflict with exclude_columns"):
|
||||||
|
loader.load_all()
|
||||||
|
|
||||||
def test_overlay_measures_set(self, tmp_path: Path):
|
def test_overlay_measures_set(self, tmp_path: Path):
|
||||||
schema_dir = tmp_path / "_schema"
|
schema_dir = tmp_path / "_schema"
|
||||||
_write_yaml(schema_dir / "public.yaml", _manifest_tables())
|
_write_yaml(schema_dir / "public.yaml", _manifest_tables())
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue