mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-10 08:05:14 +02:00
fix(context): expose warehouse targets for LookML and MetricFlow
This commit is contained in:
parent
3ebd5e0a62
commit
af9fd77780
6 changed files with 76 additions and 2 deletions
|
|
@ -15,6 +15,18 @@ describe('LookmlSourceAdapter validation sidecars', () => {
|
|||
|
||||
afterEach(async () => rm(tmpRoot, { recursive: true, force: true }));
|
||||
|
||||
it('returns configured target warehouse connection ids', async () => {
|
||||
const adapter = new LookmlSourceAdapter({
|
||||
homeDir: join(tmpRoot, 'home'),
|
||||
targetConnectionIds: ['warehouse', 'analytics', 'warehouse'],
|
||||
});
|
||||
|
||||
await expect(adapter.listTargetConnectionIds?.(join(tmpRoot, 'staged'))).resolves.toEqual([
|
||||
'analytics',
|
||||
'warehouse',
|
||||
]);
|
||||
});
|
||||
|
||||
it('writes a partial fetch report and marks mismatched chunks as SL-disallowed', async () => {
|
||||
const originRoot = join(tmpRoot, 'origin-src');
|
||||
await mkdir(join(originRoot, 'views'), { recursive: true });
|
||||
|
|
|
|||
|
|
@ -14,6 +14,11 @@ import { parseLookmlPullConfig } from './pull-config.js';
|
|||
|
||||
export interface LookmlSourceAdapterDeps {
|
||||
homeDir: string;
|
||||
targetConnectionIds?: string[];
|
||||
}
|
||||
|
||||
function uniqueSorted(values: readonly string[] | undefined): string[] {
|
||||
return [...new Set(values ?? [])].sort((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
export class LookmlSourceAdapter implements SourceAdapter {
|
||||
|
|
@ -43,6 +48,10 @@ export class LookmlSourceAdapter implements SourceAdapter {
|
|||
return readLookmlFetchReport(stagedDir);
|
||||
}
|
||||
|
||||
async listTargetConnectionIds(_stagedDir: string): Promise<string[]> {
|
||||
return uniqueSorted(this.deps.targetConnectionIds);
|
||||
}
|
||||
|
||||
async chunk(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult> {
|
||||
const project = await parseLookmlStagedDir(stagedDir);
|
||||
const mismatchedModelNames = await readLookmlMismatchedModelNames(stagedDir);
|
||||
|
|
|
|||
|
|
@ -42,6 +42,15 @@ describe('MetricflowSourceAdapter', () => {
|
|||
expect(adapter.skillNames).toEqual(['metricflow_ingest']);
|
||||
});
|
||||
|
||||
it('returns configured target warehouse connection ids', async () => {
|
||||
const metricflow = new MetricflowSourceAdapter({
|
||||
homeDir: join(tmpRoot, 'cache-home'),
|
||||
targetConnectionIds: ['warehouse', 'analytics', 'warehouse'],
|
||||
});
|
||||
|
||||
await expect(metricflow.listTargetConnectionIds?.(stagedDir)).resolves.toEqual(['analytics', 'warehouse']);
|
||||
});
|
||||
|
||||
it('detects a staged dir with a semantic_models YAML', async () => {
|
||||
await mkdir(join(stagedDir, 'models'), { recursive: true });
|
||||
await writeFile(
|
||||
|
|
|
|||
|
|
@ -9,6 +9,11 @@ import { parseMetricflowPullConfig } from './pull-config.js';
|
|||
|
||||
export interface MetricflowSourceAdapterDeps {
|
||||
homeDir: string;
|
||||
targetConnectionIds?: string[];
|
||||
}
|
||||
|
||||
function uniqueSorted(values: readonly string[] | undefined): string[] {
|
||||
return [...new Set(values ?? [])].sort((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
export class MetricflowSourceAdapter implements SourceAdapter {
|
||||
|
|
@ -30,6 +35,10 @@ export class MetricflowSourceAdapter implements SourceAdapter {
|
|||
});
|
||||
}
|
||||
|
||||
async listTargetConnectionIds(_stagedDir: string): Promise<string[]> {
|
||||
return uniqueSorted(this.deps.targetConnectionIds);
|
||||
}
|
||||
|
||||
async chunk(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult> {
|
||||
const project = await parseMetricFlowStagedDir(stagedDir);
|
||||
const chunk = await chunkMetricFlowProject(project, { diffSet });
|
||||
|
|
|
|||
|
|
@ -523,6 +523,35 @@ describe('local ingest adapters', () => {
|
|||
await expect(notion?.listTargetConnectionIds?.('/tmp/staged-notion')).resolves.toEqual(['warehouse']);
|
||||
});
|
||||
|
||||
it('passes primary warehouse connection ids to local LookML and MetricFlow adapters', async () => {
|
||||
const adapters = createDefaultLocalIngestAdapters(
|
||||
projectWithConnections({
|
||||
warehouse: {
|
||||
driver: 'postgres',
|
||||
url: 'postgresql://readonly@db.example.test/analytics',
|
||||
},
|
||||
lookml_docs: {
|
||||
driver: 'lookml',
|
||||
lookml: {
|
||||
repoUrl: 'https://github.com/acme/lookml.git',
|
||||
},
|
||||
},
|
||||
metrics_repo: {
|
||||
driver: 'metricflow',
|
||||
metricflow: {
|
||||
repoUrl: 'https://github.com/acme/metrics.git',
|
||||
},
|
||||
},
|
||||
} as never),
|
||||
);
|
||||
|
||||
const lookml = adapters.find((adapter) => adapter.source === 'lookml');
|
||||
const metricflow = adapters.find((adapter) => adapter.source === 'metricflow');
|
||||
|
||||
await expect(lookml?.listTargetConnectionIds?.('/tmp/staged-lookml')).resolves.toEqual(['warehouse']);
|
||||
await expect(metricflow?.listTargetConnectionIds?.('/tmp/staged-metricflow')).resolves.toEqual(['warehouse']);
|
||||
});
|
||||
|
||||
it('resolves MetricFlow auth_token_ref without writing literal tokens to config', async () => {
|
||||
const project = projectWithConnections({
|
||||
metricflow_main: {
|
||||
|
|
|
|||
|
|
@ -88,7 +88,10 @@ export function createDefaultLocalIngestAdapters(
|
|||
...(options.databaseIntrospectionUrl ? { baseUrl: options.databaseIntrospectionUrl } : {}),
|
||||
}),
|
||||
}),
|
||||
new LookmlSourceAdapter({ homeDir: join(project.projectDir, '.ktx/cache') }),
|
||||
new LookmlSourceAdapter({
|
||||
homeDir: join(project.projectDir, '.ktx/cache'),
|
||||
targetConnectionIds: primaryWarehouseConnectionIds(project),
|
||||
}),
|
||||
new DbtSourceAdapter({
|
||||
homeDir: join(project.projectDir, '.ktx/cache'),
|
||||
targetConnectionIds: primaryWarehouseConnectionIds(project),
|
||||
|
|
@ -106,7 +109,10 @@ export function createDefaultLocalIngestAdapters(
|
|||
},
|
||||
},
|
||||
}),
|
||||
new MetricflowSourceAdapter({ homeDir: join(project.projectDir, '.ktx/cache') }),
|
||||
new MetricflowSourceAdapter({
|
||||
homeDir: join(project.projectDir, '.ktx/cache'),
|
||||
targetConnectionIds: primaryWarehouseConnectionIds(project),
|
||||
}),
|
||||
new NotionSourceAdapter({
|
||||
targetConnectionIds: primaryWarehouseConnectionIds(project),
|
||||
...(options.logger ? { logger: options.logger } : {}),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue