mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-22 08:38:08 +02:00
feat(context): add driver-discriminated connection schemas (#96)
* refactor(context): export and describe mapping shape schemas * feat(context): add driver-schemas module with warehouse drivers * feat(context): add metabase, looker, lookml driver schemas with mappings * feat(context): add notion, dbt, metricflow driver schemas * refactor(context): make connectionSchema a driver-discriminated union * chore(context): re-export KtxConnectionConfig from project package * docs(context): add connection driver schema plan * chore(secrets): allowlist example credentials in driver-schemas fixtures * test(cli): align metabase fixtures with required api_url field The driver-discriminated union added in this branch now requires api_url for metabase connections and a known driver for warehouses. Update slow CLI test fixtures and assertions so they exercise the new schema: - ingest.test-utils.ts: add api_url to the prod-metabase fixture. - setup.test.ts: switch metabase fixture from 'url' to 'api_url'. - local-scan-connectors.test.ts: invalid-driver/missing-driver tests now expect the schema error from loadKtxProject (parse-time rejection).
This commit is contained in:
parent
d244261aa7
commit
f8db99811a
20 changed files with 1283 additions and 49 deletions
|
|
@ -36,7 +36,13 @@ describe('localConnectionToWarehouseDescriptor', () => {
|
|||
});
|
||||
|
||||
it('returns null for non-warehouse adapters', () => {
|
||||
expect(localConnectionToWarehouseDescriptor('looker', { driver: 'looker' })).toBeNull();
|
||||
expect(
|
||||
localConnectionToWarehouseDescriptor('looker', {
|
||||
driver: 'looker',
|
||||
base_url: 'https://looker.example.com',
|
||||
client_id: 'client',
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -48,7 +54,9 @@ describe('local connection info helpers', () => {
|
|||
});
|
||||
|
||||
it('keeps non-warehouse adapter labels for display-only local connection surfaces', () => {
|
||||
expect(localConnectionTypeForConfig('prod-metabase', { driver: 'metabase' })).toBe('metabase');
|
||||
expect(localConnectionTypeForConfig('prod-metabase', { driver: 'metabase', api_url: 'https://metabase.example.com' })).toBe(
|
||||
'metabase',
|
||||
);
|
||||
expect(localConnectionTypeForConfig('missing-driver', {} as never)).toBe('unknown');
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,20 @@ export const KTX_NOTION_ORG_KNOWLEDGE_WARNING =
|
|||
|
||||
type KtxNotionCrawlMode = 'all_accessible' | 'selected_roots';
|
||||
|
||||
export interface KtxNotionConnectionConfig extends KtxProjectConnectionConfig {
|
||||
type RawKtxNotionConnectionConfig = Extract<KtxProjectConnectionConfig, { driver: 'notion' }>;
|
||||
|
||||
export type KtxNotionConnectionConfig = Omit<
|
||||
RawKtxNotionConnectionConfig,
|
||||
| 'auth_token'
|
||||
| 'auth_token_ref'
|
||||
| 'crawl_mode'
|
||||
| 'root_page_ids'
|
||||
| 'root_database_ids'
|
||||
| 'root_data_source_ids'
|
||||
| 'max_pages_per_run'
|
||||
| 'max_knowledge_creates_per_run'
|
||||
| 'max_knowledge_updates_per_run'
|
||||
> & {
|
||||
driver: 'notion';
|
||||
auth_token: string | null;
|
||||
auth_token_ref: string | null;
|
||||
|
|
@ -24,7 +37,7 @@ export interface KtxNotionConnectionConfig extends KtxProjectConnectionConfig {
|
|||
max_pages_per_run: number;
|
||||
max_knowledge_creates_per_run: number;
|
||||
max_knowledge_updates_per_run: number;
|
||||
}
|
||||
};
|
||||
|
||||
export interface RedactedKtxNotionConnectionConfig {
|
||||
driver: 'notion';
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { tmpdir } from 'node:os';
|
|||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { buildDefaultKtxProjectConfig } from '../../../project/index.js';
|
||||
import { connectionConfigSchema } from '../../../project/driver-schemas.js';
|
||||
import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './local-source-state-store.js';
|
||||
|
||||
describe('Metabase YAML source state and discovery cache', () => {
|
||||
|
|
@ -23,10 +24,11 @@ describe('Metabase YAML source state and discovery cache', () => {
|
|||
config: {
|
||||
...buildDefaultKtxProjectConfig(),
|
||||
connections: {
|
||||
'prod-metabase': {
|
||||
'prod-metabase': connectionConfigSchema.parse({
|
||||
driver: 'metabase',
|
||||
api_url: 'https://metabase.example.com',
|
||||
mappings,
|
||||
},
|
||||
}),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -27,11 +27,12 @@ describe('local mapping yaml reconciliation bridge', () => {
|
|||
const project = projectWithConnections({
|
||||
'prod-metabase': {
|
||||
driver: 'metabase',
|
||||
api_url: 'https://metabase.example.com',
|
||||
mappings: {
|
||||
databaseMappings: { '1': 'prod-warehouse' },
|
||||
syncEnabled: { '1': true },
|
||||
syncMode: 'ONLY',
|
||||
selections: { collections: [12] },
|
||||
selections: { collections: [12], items: [] },
|
||||
defaultTagNames: ['ktx'],
|
||||
},
|
||||
},
|
||||
|
|
@ -46,6 +47,8 @@ describe('local mapping yaml reconciliation bridge', () => {
|
|||
const project = projectWithConnections({
|
||||
'prod-looker': {
|
||||
driver: 'looker',
|
||||
base_url: 'https://looker.example.com',
|
||||
client_id: 'client',
|
||||
mappings: { connectionMappings: { analytics: 'prod-warehouse' } },
|
||||
},
|
||||
'prod-warehouse': { driver: 'postgres', url: 'postgresql://readonly@db.test/analytics' },
|
||||
|
|
|
|||
|
|
@ -509,4 +509,11 @@ describe('generateKtxProjectConfigJsonSchema', () => {
|
|||
const relationships = scan?.properties?.relationships as { properties?: Record<string, { description?: string }> };
|
||||
expect(relationships?.properties?.acceptThreshold?.description).toMatch(/auto-accepted/);
|
||||
});
|
||||
|
||||
it('emits the mappings shapes under connections', () => {
|
||||
const serialized = JSON.stringify(schema);
|
||||
expect(serialized).toContain('databaseMappings');
|
||||
expect(serialized).toContain('connectionMappings');
|
||||
expect(serialized).toContain('expectedLookerConnectionName');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { KTX_MODEL_ROLES } from '@ktx/llm';
|
||||
import YAML from 'yaml';
|
||||
import * as z from 'zod';
|
||||
import { connectionConfigSchema } from './driver-schemas.js';
|
||||
|
||||
const KTX_LLM_BACKENDS = ['none', 'anthropic', 'vertex', 'gateway'] as const;
|
||||
const KTX_EMBEDDING_BACKENDS = ['none', 'deterministic', 'openai', 'sentence-transformers'] as const;
|
||||
|
|
@ -206,12 +207,7 @@ const storageSchema = z
|
|||
})
|
||||
.describe('Storage backends and commit policy for KTX state and search indexes.');
|
||||
|
||||
const connectionSchema = z
|
||||
.looseObject({
|
||||
driver: z.string().min(1).optional().describe('Connector driver identifier (e.g. "postgres", "bigquery", "snowflake").'),
|
||||
url: z.string().optional().describe('Connection URL or DSN. Format depends on the driver; may contain environment-variable references.'),
|
||||
})
|
||||
.describe('A single database/connector connection entry. Additional driver-specific fields are accepted and passed through.');
|
||||
const connectionSchema = connectionConfigSchema;
|
||||
|
||||
const agentSchema = z
|
||||
.strictObject({
|
||||
|
|
|
|||
140
packages/context/src/project/driver-schemas.test.ts
Normal file
140
packages/context/src/project/driver-schemas.test.ts
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { connectionConfigSchema } from './driver-schemas.js';
|
||||
|
||||
describe('connectionConfigSchema (driver discriminated union)', () => {
|
||||
it.each([
|
||||
['postgres', 'postgres://user:pass@host:5432/db'], // pragma: allowlist secret
|
||||
['postgresql', 'postgresql://user:pass@host:5432/db'], // pragma: allowlist secret
|
||||
['mysql', 'mysql://user:pass@host:3306/db'], // pragma: allowlist secret
|
||||
['snowflake', 'snowflake://account/db'],
|
||||
['bigquery', 'bigquery://project/dataset'],
|
||||
['sqlite', 'sqlite:///tmp/db.sqlite'],
|
||||
['clickhouse', 'clickhouse://host:8123/db'],
|
||||
['sqlserver', 'sqlserver://host:1433;database=db'],
|
||||
])('parses %s warehouse connection', (driver, url) => {
|
||||
expect(connectionConfigSchema.parse({ driver, url })).toMatchObject({ driver, url });
|
||||
});
|
||||
|
||||
it('preserves unknown warehouse fields via looseObject passthrough', () => {
|
||||
const parsed = connectionConfigSchema.parse({
|
||||
driver: 'postgres',
|
||||
url: 'postgres://x',
|
||||
historicSql: { enabled: true },
|
||||
context: { queryHistory: { enabled: false } },
|
||||
});
|
||||
expect(parsed).toMatchObject({
|
||||
driver: 'postgres',
|
||||
historicSql: { enabled: true },
|
||||
context: { queryHistory: { enabled: false } },
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects an unknown driver', () => {
|
||||
expect(() => connectionConfigSchema.parse({ driver: 'nope', url: 'x' })).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('connectionConfigSchema - context source drivers with mappings', () => {
|
||||
it('parses a metabase connection with mappings', () => {
|
||||
const parsed = connectionConfigSchema.parse({
|
||||
driver: 'metabase',
|
||||
api_url: 'https://metabase.example.com',
|
||||
api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret
|
||||
mappings: {
|
||||
databaseMappings: { '3': 'prod-warehouse' },
|
||||
syncEnabled: { '3': true },
|
||||
syncMode: 'ONLY',
|
||||
},
|
||||
});
|
||||
expect(parsed).toMatchObject({
|
||||
driver: 'metabase',
|
||||
api_url: 'https://metabase.example.com',
|
||||
mappings: {
|
||||
databaseMappings: { '3': 'prod-warehouse' },
|
||||
syncMode: 'ONLY',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('parses a looker connection with connectionMappings', () => {
|
||||
const parsed = connectionConfigSchema.parse({
|
||||
driver: 'looker',
|
||||
base_url: 'https://looker.example.com',
|
||||
client_id: 'abc',
|
||||
client_secret_ref: 'env:LOOKER_CLIENT_SECRET', // pragma: allowlist secret
|
||||
mappings: { connectionMappings: { bigquery_prod: 'wh' } },
|
||||
});
|
||||
expect(parsed.mappings).toEqual({ connectionMappings: { bigquery_prod: 'wh' } });
|
||||
});
|
||||
|
||||
it('parses a lookml connection with expectedLookerConnectionName', () => {
|
||||
const parsed = connectionConfigSchema.parse({
|
||||
driver: 'lookml',
|
||||
repoUrl: 'https://github.com/acme/looker.git',
|
||||
branch: 'main',
|
||||
mappings: { expectedLookerConnectionName: 'bigquery_prod' },
|
||||
});
|
||||
expect(parsed.mappings).toEqual({ expectedLookerConnectionName: 'bigquery_prod' });
|
||||
});
|
||||
|
||||
it('rejects metabase mapping with non-integer database key', () => {
|
||||
expect(() =>
|
||||
connectionConfigSchema.parse({
|
||||
driver: 'metabase',
|
||||
api_url: 'https://x',
|
||||
mappings: { databaseMappings: { abc: 'wh' } },
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('connectionConfigSchema - notion / dbt / metricflow', () => {
|
||||
it('parses a notion connection with selected_roots crawl', () => {
|
||||
const parsed = connectionConfigSchema.parse({
|
||||
driver: 'notion',
|
||||
auth_token_ref: 'env:NOTION_TOKEN',
|
||||
crawl_mode: 'selected_roots',
|
||||
root_page_ids: ['abc', 'def'],
|
||||
max_pages_per_run: 500,
|
||||
});
|
||||
expect(parsed).toMatchObject({
|
||||
driver: 'notion',
|
||||
crawl_mode: 'selected_roots',
|
||||
root_page_ids: ['abc', 'def'],
|
||||
max_pages_per_run: 500,
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects notion with unknown crawl_mode', () => {
|
||||
expect(() =>
|
||||
connectionConfigSchema.parse({
|
||||
driver: 'notion',
|
||||
auth_token_ref: 'env:NOTION_TOKEN',
|
||||
crawl_mode: 'everything',
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it('parses a dbt connection from a local source_dir', () => {
|
||||
const parsed = connectionConfigSchema.parse({
|
||||
driver: 'dbt',
|
||||
source_dir: '/tmp/dbt-project',
|
||||
target: 'dev',
|
||||
});
|
||||
expect(parsed).toMatchObject({ driver: 'dbt', source_dir: '/tmp/dbt-project', target: 'dev' });
|
||||
});
|
||||
|
||||
it('parses a metricflow connection with nested config', () => {
|
||||
const parsed = connectionConfigSchema.parse({
|
||||
driver: 'metricflow',
|
||||
metricflow: {
|
||||
repoUrl: 'https://github.com/acme/sl.git',
|
||||
branch: 'main',
|
||||
},
|
||||
});
|
||||
expect(parsed).toMatchObject({
|
||||
driver: 'metricflow',
|
||||
metricflow: { repoUrl: 'https://github.com/acme/sl.git' },
|
||||
});
|
||||
});
|
||||
});
|
||||
205
packages/context/src/project/driver-schemas.ts
Normal file
205
packages/context/src/project/driver-schemas.ts
Normal file
|
|
@ -0,0 +1,205 @@
|
|||
import * as z from 'zod';
|
||||
import {
|
||||
lookerMappingsSchema,
|
||||
lookmlMappingsSchema,
|
||||
metabaseMappingsSchema,
|
||||
} from './mappings-yaml-schema.js';
|
||||
|
||||
const warehouseDrivers = [
|
||||
'postgres',
|
||||
'postgresql',
|
||||
'mysql',
|
||||
'snowflake',
|
||||
'bigquery',
|
||||
'sqlite',
|
||||
'clickhouse',
|
||||
'sqlserver',
|
||||
] as const;
|
||||
|
||||
type WarehouseDriver = (typeof warehouseDrivers)[number];
|
||||
|
||||
function warehouseConnectionSchema<const Driver extends WarehouseDriver>(driver: Driver) {
|
||||
return z
|
||||
.looseObject({
|
||||
driver: z.literal(driver),
|
||||
url: z
|
||||
.string()
|
||||
.min(1)
|
||||
.optional()
|
||||
.describe('Warehouse connection URL or DSN; may contain environment-variable references like env:DATABASE_URL.'),
|
||||
})
|
||||
.describe(
|
||||
`${driver} warehouse connection. Additional driver-tunable fields (e.g. historicSql, context.queryHistory) are accepted and passed through.`,
|
||||
);
|
||||
}
|
||||
|
||||
const warehouseConnectionSchemas = [
|
||||
warehouseConnectionSchema('postgres'),
|
||||
warehouseConnectionSchema('postgresql'),
|
||||
warehouseConnectionSchema('mysql'),
|
||||
warehouseConnectionSchema('snowflake'),
|
||||
warehouseConnectionSchema('bigquery'),
|
||||
warehouseConnectionSchema('sqlite'),
|
||||
warehouseConnectionSchema('clickhouse'),
|
||||
warehouseConnectionSchema('sqlserver'),
|
||||
] as const;
|
||||
|
||||
const positiveIntKeyMessage = (field: string) => `${field} keys must be positive-integer strings (e.g. "1", "42")`;
|
||||
|
||||
const positiveIntKeyRegex = /^[1-9]\d*$/;
|
||||
|
||||
const metabaseMappingsStrictSchema = metabaseMappingsSchema.superRefine((value, ctx) => {
|
||||
for (const key of Object.keys(value.databaseMappings ?? {})) {
|
||||
if (!positiveIntKeyRegex.test(key)) {
|
||||
ctx.addIssue({
|
||||
code: 'custom',
|
||||
path: ['databaseMappings', key],
|
||||
message: positiveIntKeyMessage('databaseMappings'),
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const key of Object.keys(value.syncEnabled ?? {})) {
|
||||
if (!positiveIntKeyRegex.test(key)) {
|
||||
ctx.addIssue({
|
||||
code: 'custom',
|
||||
path: ['syncEnabled', key],
|
||||
message: positiveIntKeyMessage('syncEnabled'),
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const metabaseConnectionSchema = z
|
||||
.looseObject({
|
||||
driver: z.literal('metabase'),
|
||||
api_url: z.string().url().describe('Metabase instance API URL (e.g. https://metabase.example.com).'),
|
||||
api_key: z.string().min(1).optional().describe('Literal Metabase API key. Prefer api_key_ref for safety.'),
|
||||
api_key_ref: z
|
||||
.string()
|
||||
.min(1)
|
||||
.optional()
|
||||
.describe('Reference to Metabase API key (e.g. env:METABASE_API_KEY or file:/path).'),
|
||||
network_proxy: z.looseObject({}).optional().describe('Optional network proxy configuration (snake_case form).'),
|
||||
networkProxy: z.looseObject({}).optional().describe('Optional network proxy configuration (camelCase form).'),
|
||||
mappings: metabaseMappingsStrictSchema
|
||||
.optional()
|
||||
.describe('Metabase database-to-warehouse mappings and sync configuration.'),
|
||||
})
|
||||
.describe('Metabase context-source connection.');
|
||||
|
||||
const lookerConnectionSchema = z
|
||||
.looseObject({
|
||||
driver: z.literal('looker'),
|
||||
base_url: z.string().url().describe('Looker instance base URL (e.g. https://looker.example.com).'),
|
||||
client_id: z.string().min(1).describe('Looker OAuth client ID.'),
|
||||
client_secret: z.string().min(1).optional().describe('Literal Looker OAuth client secret. Prefer client_secret_ref.'),
|
||||
client_secret_ref: z
|
||||
.string()
|
||||
.min(1)
|
||||
.optional()
|
||||
.describe('Reference to Looker OAuth client secret (e.g. env:LOOKER_CLIENT_SECRET).'),
|
||||
mappings: lookerMappingsSchema.optional().describe('Looker connection-name to KTX warehouse mappings.'),
|
||||
})
|
||||
.describe('Looker context-source connection.');
|
||||
|
||||
const lookmlConnectionSchema = z
|
||||
.looseObject({
|
||||
driver: z.literal('lookml'),
|
||||
repoUrl: z
|
||||
.string()
|
||||
.min(1)
|
||||
.describe('Git URL of the LookML project (https, ssh, or file:). Field is camelCase by convention.'),
|
||||
branch: z.string().min(1).optional().describe('Git branch (default "main" downstream).'),
|
||||
path: z.string().optional().describe('Subdirectory within the repo when the LookML project lives in a monorepo.'),
|
||||
auth_token_ref: z.string().min(1).optional().describe('Reference to Git auth token for private repos (e.g. env:GITHUB_TOKEN).'),
|
||||
mappings: lookmlMappingsSchema.optional().describe('LookML expected-connection mapping for ingest gating.'),
|
||||
})
|
||||
.describe('LookML context-source connection.');
|
||||
|
||||
const notionConnectionSchema = z
|
||||
.looseObject({
|
||||
driver: z.literal('notion'),
|
||||
auth_token: z.string().min(1).optional().describe('Literal Notion integration token. Prefer auth_token_ref.'),
|
||||
auth_token_ref: z
|
||||
.string()
|
||||
.min(1)
|
||||
.optional()
|
||||
.describe('Reference to Notion integration token (e.g. env:NOTION_TOKEN).'),
|
||||
crawl_mode: z
|
||||
.enum(['selected_roots', 'all_accessible'])
|
||||
.optional()
|
||||
.describe(
|
||||
'Crawl scope. "selected_roots" requires at least one of root_page_ids, root_database_ids, root_data_source_ids.',
|
||||
),
|
||||
root_page_ids: z.array(z.string().min(1)).optional().describe('Notion page IDs to crawl when crawl_mode is selected_roots.'),
|
||||
root_database_ids: z
|
||||
.array(z.string().min(1))
|
||||
.optional()
|
||||
.describe('Notion database IDs to crawl when crawl_mode is selected_roots.'),
|
||||
root_data_source_ids: z
|
||||
.array(z.string().min(1))
|
||||
.optional()
|
||||
.describe('Notion data source IDs to crawl when crawl_mode is selected_roots.'),
|
||||
max_pages_per_run: z
|
||||
.number()
|
||||
.int()
|
||||
.min(1)
|
||||
.max(10000)
|
||||
.optional()
|
||||
.describe('Maximum Notion pages fetched in a single ingest run.'),
|
||||
max_knowledge_creates_per_run: z
|
||||
.number()
|
||||
.int()
|
||||
.min(0)
|
||||
.max(25)
|
||||
.optional()
|
||||
.describe('Maximum new wiki pages created per run.'),
|
||||
max_knowledge_updates_per_run: z
|
||||
.number()
|
||||
.int()
|
||||
.min(0)
|
||||
.max(100)
|
||||
.optional()
|
||||
.describe('Maximum existing wiki pages updated per run.'),
|
||||
})
|
||||
.describe('Notion context-source connection.');
|
||||
|
||||
const dbtConnectionSchema = z
|
||||
.looseObject({
|
||||
driver: z.literal('dbt'),
|
||||
source_dir: z.string().min(1).optional().describe('Absolute or project-relative path to a local dbt project.'),
|
||||
repo_url: z.string().min(1).optional().describe('Git URL of the dbt project (https, ssh, or file:).'),
|
||||
branch: z.string().min(1).optional().describe('Git branch when using repo_url.'),
|
||||
path: z.string().optional().describe('Subdirectory within the repo when the dbt project lives in a monorepo.'),
|
||||
auth_token_ref: z.string().min(1).optional().describe('Reference to Git auth token for private repos.'),
|
||||
profiles_path: z.string().optional().describe('Override path to dbt profiles.yml.'),
|
||||
target: z.string().min(1).optional().describe('dbt target name (e.g. dev, prod).'),
|
||||
project_name: z.string().min(1).optional().describe('Override auto-detected dbt project name.'),
|
||||
})
|
||||
.describe('dbt context-source connection.');
|
||||
|
||||
const metricflowConnectionSchema = z
|
||||
.looseObject({
|
||||
driver: z.literal('metricflow'),
|
||||
metricflow: z
|
||||
.looseObject({
|
||||
repoUrl: z.string().min(1).describe('Git URL of the MetricFlow / SL project.'),
|
||||
branch: z.string().min(1).optional().describe('Git branch (default "main").'),
|
||||
path: z.string().optional().describe('Subdirectory within the repo when the SL config lives in a monorepo.'),
|
||||
auth_token_ref: z.string().min(1).optional().describe('Reference to Git auth token for private repos.'),
|
||||
})
|
||||
.describe('Nested MetricFlow configuration block.'),
|
||||
})
|
||||
.describe('MetricFlow / SL context-source connection.');
|
||||
|
||||
export const connectionConfigSchema = z.discriminatedUnion('driver', [
|
||||
...warehouseConnectionSchemas,
|
||||
metabaseConnectionSchema,
|
||||
lookerConnectionSchema,
|
||||
lookmlConnectionSchema,
|
||||
notionConnectionSchema,
|
||||
dbtConnectionSchema,
|
||||
metricflowConnectionSchema,
|
||||
]);
|
||||
|
||||
export type KtxConnectionConfig = z.infer<typeof connectionConfigSchema>;
|
||||
|
|
@ -15,6 +15,7 @@ export {
|
|||
serializeKtxProjectConfig,
|
||||
validateKtxProjectConfig,
|
||||
} from './config.js';
|
||||
export type { KtxConnectionConfig } from './driver-schemas.js';
|
||||
export type { LocalGitFileStoreDeps } from './local-git-file-store.js';
|
||||
export { LocalGitFileStore } from './local-git-file-store.js';
|
||||
export { ktxLocalStateDbPath } from './local-state-db.js';
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
lookerMappingsSchema,
|
||||
lookmlMappingsSchema,
|
||||
metabaseMappingsSchema,
|
||||
parseConnectionMappingBootstrap,
|
||||
parseLookmlMappingBootstrap,
|
||||
parseLookerMappingBootstrap,
|
||||
|
|
@ -82,4 +85,17 @@ describe('ktx.yaml mapping bootstrap schema', () => {
|
|||
}),
|
||||
).toMatchObject({ adapter: 'looker', connectionId: 'prod-looker' });
|
||||
});
|
||||
|
||||
it('exports mapping shapes that parse documented examples', () => {
|
||||
expect(metabaseMappingsSchema.parse({ databaseMappings: { '1': 'wh' } })).toMatchObject({
|
||||
databaseMappings: { '1': 'wh' },
|
||||
syncMode: 'ALL',
|
||||
});
|
||||
expect(lookerMappingsSchema.parse({ connectionMappings: { x: 'wh' } })).toEqual({
|
||||
connectionMappings: { x: 'wh' },
|
||||
});
|
||||
expect(lookmlMappingsSchema.parse({ expectedLookerConnectionName: 'x' })).toEqual({
|
||||
expectedLookerConnectionName: 'x',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import * as z from 'zod';
|
||||
import type { KtxProjectConnectionConfig } from './config.js';
|
||||
|
||||
const metabaseSyncModeSchema = z.enum(['ALL', 'ONLY', 'EXCEPT']);
|
||||
const positiveIntegerValueSchema = z.number().int().positive();
|
||||
|
|
@ -11,24 +10,48 @@ const metabaseSelectionsSchema = z
|
|||
items: z.array(positiveIntegerValueSchema).default([]),
|
||||
});
|
||||
|
||||
const metabaseMappingsSchema = z
|
||||
export const metabaseMappingsSchema = z
|
||||
.object({
|
||||
databaseMappings: z.record(z.string(), stringTargetSchema).default({}),
|
||||
syncEnabled: z.record(z.string(), z.boolean()).default({}),
|
||||
syncMode: metabaseSyncModeSchema.default('ALL'),
|
||||
selections: metabaseSelectionsSchema.default({ collections: [], items: [] }),
|
||||
defaultTagNames: z.array(z.string().min(1)).default([]),
|
||||
});
|
||||
databaseMappings: z
|
||||
.record(z.string(), stringTargetSchema)
|
||||
.default({})
|
||||
.describe('Map of Metabase database ID (positive integer string) to KTX connection ID. Use null to explicitly unmap.'),
|
||||
syncEnabled: z
|
||||
.record(z.string(), z.boolean())
|
||||
.default({})
|
||||
.describe('Per-Metabase-database sync toggle, keyed by Metabase database ID string.'),
|
||||
syncMode: metabaseSyncModeSchema
|
||||
.default('ALL')
|
||||
.describe('Sync scope: ALL ingests every mapped DB; ONLY restricts to syncEnabled=true; EXCEPT excludes syncEnabled=true.'),
|
||||
selections: metabaseSelectionsSchema
|
||||
.default({ collections: [], items: [] })
|
||||
.describe('Optional Metabase collection and item IDs to scope ingest.'),
|
||||
defaultTagNames: z
|
||||
.array(z.string().min(1))
|
||||
.default([])
|
||||
.describe('Default tag names applied to ingested Metabase artifacts.'),
|
||||
})
|
||||
.describe('Metabase database-to-warehouse mapping and sync configuration.');
|
||||
|
||||
const lookerMappingsSchema = z
|
||||
export const lookerMappingsSchema = z
|
||||
.object({
|
||||
connectionMappings: z.record(z.string().min(1), stringTargetSchema).default({}),
|
||||
});
|
||||
connectionMappings: z
|
||||
.record(z.string().min(1), stringTargetSchema)
|
||||
.default({})
|
||||
.describe('Map of Looker connection name to KTX connection ID. Use null to explicitly unmap.'),
|
||||
})
|
||||
.describe('Looker connection-to-warehouse mapping configuration.');
|
||||
|
||||
const lookmlMappingsSchema = z
|
||||
export const lookmlMappingsSchema = z
|
||||
.object({
|
||||
expectedLookerConnectionName: z.string().min(1).nullable().default(null),
|
||||
});
|
||||
expectedLookerConnectionName: z
|
||||
.string()
|
||||
.min(1)
|
||||
.nullable()
|
||||
.default(null)
|
||||
.describe('Looker connection name that LookML models must declare; mismatches block sl_write_source at ingest time.'),
|
||||
})
|
||||
.describe('LookML connection-name expectation for ingest gating.');
|
||||
|
||||
export type MetabaseMappingBootstrap = {
|
||||
adapter: 'metabase';
|
||||
|
|
@ -54,6 +77,11 @@ export type LookmlMappingBootstrap = {
|
|||
|
||||
export type ConnectionMappingBootstrap = MetabaseMappingBootstrap | LookerMappingBootstrap | LookmlMappingBootstrap;
|
||||
|
||||
type MappingConnectionInput = Record<string, unknown> & {
|
||||
driver?: unknown;
|
||||
mappings?: unknown;
|
||||
};
|
||||
|
||||
function recordValue(value: unknown): Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
|
||||
}
|
||||
|
|
@ -66,13 +94,13 @@ function assertPositiveIntegerKeys(field: string, record: Record<string, unknown
|
|||
}
|
||||
}
|
||||
|
||||
function driverOf(connection: KtxProjectConnectionConfig): string {
|
||||
function driverOf(connection: MappingConnectionInput): string {
|
||||
return String(connection.driver ?? '').toLowerCase();
|
||||
}
|
||||
|
||||
export function parseMetabaseMappingBootstrap(
|
||||
connectionId: string,
|
||||
connection: KtxProjectConnectionConfig,
|
||||
connection: MappingConnectionInput,
|
||||
): MetabaseMappingBootstrap {
|
||||
const rawMappings = recordValue(connection.mappings);
|
||||
assertPositiveIntegerKeys('databaseMappings', recordValue(rawMappings.databaseMappings));
|
||||
|
|
@ -91,7 +119,7 @@ export function parseMetabaseMappingBootstrap(
|
|||
|
||||
export function parseLookerMappingBootstrap(
|
||||
connectionId: string,
|
||||
connection: KtxProjectConnectionConfig,
|
||||
connection: MappingConnectionInput,
|
||||
): LookerMappingBootstrap {
|
||||
const parsed = lookerMappingsSchema.parse(recordValue(connection.mappings));
|
||||
return {
|
||||
|
|
@ -103,7 +131,7 @@ export function parseLookerMappingBootstrap(
|
|||
|
||||
export function parseLookmlMappingBootstrap(
|
||||
connectionId: string,
|
||||
connection: KtxProjectConnectionConfig,
|
||||
connection: MappingConnectionInput,
|
||||
): LookmlMappingBootstrap {
|
||||
const parsed = lookmlMappingsSchema.parse(recordValue(connection.mappings));
|
||||
return {
|
||||
|
|
@ -115,7 +143,7 @@ export function parseLookmlMappingBootstrap(
|
|||
|
||||
export function parseConnectionMappingBootstrap(
|
||||
connectionId: string,
|
||||
connection: KtxProjectConnectionConfig,
|
||||
connection: MappingConnectionInput,
|
||||
): ConnectionMappingBootstrap | null {
|
||||
if (!connection.mappings || typeof connection.mappings !== 'object' || Array.isArray(connection.mappings)) {
|
||||
return null;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue