mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-25 12:01:03 +02:00
fix(cli): support local dbt source_dir in ktx connection test
Extract readGitRepoConnectionFields as the single reader of where the git-repo context-source drivers store their fields (dbt's snake_case repo_url/source_dir, lookml's camelCase repoUrl, metricflow's nested metricflow.*). `ktx connection test` now routes through it and, when a dbt connection points at a local source_dir, verifies the directory exists instead of running a remote repo reachability check that can never apply.
This commit is contained in:
parent
7e5551906e
commit
834e60625c
3 changed files with 123 additions and 33 deletions
|
|
@ -1,3 +1,5 @@
|
|||
import { stat } from 'node:fs/promises';
|
||||
import { resolve } from 'node:path';
|
||||
import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory } from './context/ingest/adapters/metabase/client.js';
|
||||
import { DefaultLookerConnectionClientFactory } from './context/ingest/adapters/looker/factory.js';
|
||||
import type { LookerClient } from './context/ingest/adapters/looker/client.js';
|
||||
|
|
@ -9,6 +11,7 @@ import { testRepoConnection } from './context/ingest/repo-fetch.js';
|
|||
import { getDriverRegistration } from './context/connections/drivers.js';
|
||||
import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from './context/connections/notion-config.js';
|
||||
import { resolveKtxConfigReference } from './context/core/config-reference.js';
|
||||
import { readGitRepoConnectionFields } from './context/project/git-connection-fields.js';
|
||||
import { type KtxLocalProject, loadKtxProject } from './context/project/project.js';
|
||||
import type { KtxScanConnector } from './context/scan/types.js';
|
||||
import type { KtxCliIo } from './index.js';
|
||||
|
|
@ -182,35 +185,14 @@ async function testNotionConnection(
|
|||
return { bot: describeNotionBot(bot) };
|
||||
}
|
||||
|
||||
interface GitConnectionFields {
|
||||
repoUrl: string;
|
||||
authToken: string | null;
|
||||
}
|
||||
|
||||
function extractGitConnectionFields(
|
||||
project: KtxLocalProject,
|
||||
connectionId: string,
|
||||
driver: string,
|
||||
): GitConnectionFields {
|
||||
const connection = project.config.connections[connectionId];
|
||||
if (!connection) {
|
||||
throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
|
||||
async function checkLocalDbtSourceDir(projectDir: string, sourceDir: string): Promise<void> {
|
||||
const stats = await stat(resolve(projectDir, sourceDir)).catch(() => null);
|
||||
if (!stats) {
|
||||
throw new Error(`dbt source_dir "${sourceDir}" does not exist`);
|
||||
}
|
||||
const stringField = (value: unknown): string | null =>
|
||||
typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
|
||||
const record =
|
||||
driver === 'metricflow' && typeof connection.metricflow === 'object' && connection.metricflow !== null
|
||||
? (connection.metricflow as Record<string, unknown>)
|
||||
: (connection as Record<string, unknown>);
|
||||
const repoUrl = driver === 'dbt' ? stringField(record.repo_url) : stringField(record.repoUrl);
|
||||
if (!repoUrl) {
|
||||
const field = driver === 'dbt' ? 'repo_url' : 'repoUrl';
|
||||
throw new Error(`Connection "${connectionId}" (driver: ${driver}) is missing ${field}`);
|
||||
if (!stats.isDirectory()) {
|
||||
throw new Error(`dbt source_dir "${sourceDir}" is not a directory`);
|
||||
}
|
||||
const literalToken = stringField(record.auth_token);
|
||||
const ref = stringField(record.auth_token_ref);
|
||||
const resolvedRef = ref ? resolveKtxConfigReference(ref, process.env) : null;
|
||||
return { repoUrl, authToken: literalToken ?? resolvedRef ?? null };
|
||||
}
|
||||
|
||||
async function testGitRepoConnection(
|
||||
|
|
@ -218,13 +200,31 @@ async function testGitRepoConnection(
|
|||
connectionId: string,
|
||||
driver: string,
|
||||
runTest: TestRepoConnection,
|
||||
): Promise<{ repoUrl: string }> {
|
||||
const { repoUrl, authToken } = extractGitConnectionFields(project, connectionId, driver);
|
||||
const result = await runTest({ repoUrl, authToken });
|
||||
): Promise<{ location: string }> {
|
||||
const connection = project.config.connections[connectionId];
|
||||
if (!connection) {
|
||||
throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
|
||||
}
|
||||
const fields = readGitRepoConnectionFields(connection, driver);
|
||||
|
||||
// dbt setup writes source_dir for a local project and ingest consumes it, so a
|
||||
// local source check must stand in for the remote repo reachability check.
|
||||
if (fields.sourceDir) {
|
||||
await checkLocalDbtSourceDir(project.projectDir, fields.sourceDir);
|
||||
return { location: fields.sourceDir };
|
||||
}
|
||||
|
||||
if (!fields.repoUrl) {
|
||||
const field = driver === 'dbt' ? 'repo_url or source_dir' : 'repoUrl';
|
||||
throw new Error(`Connection "${connectionId}" (driver: ${driver}) is missing ${field}`);
|
||||
}
|
||||
const resolvedRef = fields.authTokenRef ? resolveKtxConfigReference(fields.authTokenRef, process.env) : null;
|
||||
const authToken = fields.authTokenLiteral ?? resolvedRef ?? null;
|
||||
const result = await runTest({ repoUrl: fields.repoUrl, authToken });
|
||||
if (!result.ok) {
|
||||
throw new Error(`${driver} repository check failed: ${result.error}`);
|
||||
}
|
||||
return { repoUrl };
|
||||
return { location: fields.repoUrl };
|
||||
}
|
||||
|
||||
interface DriverTestOutcome {
|
||||
|
|
@ -277,7 +277,7 @@ async function testConnectionByDriver(
|
|||
driver,
|
||||
deps.testRepoConnection ?? testRepoConnection,
|
||||
);
|
||||
return { driver, detailKey: 'Repo', detailValue: result.repoUrl };
|
||||
return { driver, detailKey: 'Repo', detailValue: result.location };
|
||||
}
|
||||
|
||||
if (getDriverRegistration(driver)) {
|
||||
|
|
|
|||
47
packages/cli/src/context/project/git-connection-fields.ts
Normal file
47
packages/cli/src/context/project/git-connection-fields.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
import type { KtxProjectConnectionConfig } from './config.js';
|
||||
|
||||
export interface GitRepoConnectionFields {
|
||||
repoUrl: string | null;
|
||||
sourceDir: string | null;
|
||||
branch: string | null;
|
||||
path: string | null;
|
||||
authTokenLiteral: string | null;
|
||||
authTokenRef: string | null;
|
||||
}
|
||||
|
||||
function trimmedString(value: unknown): string | null {
|
||||
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
// metricflow nests its repo fields under `metricflow.*`; dbt/lookml keep them top-level.
|
||||
function fieldRecord(connection: KtxProjectConnectionConfig, driver: string): Record<string, unknown> {
|
||||
if (driver === 'metricflow') {
|
||||
const nested = (connection as Record<string, unknown>).metricflow;
|
||||
if (typeof nested === 'object' && nested !== null && !Array.isArray(nested)) {
|
||||
return nested as Record<string, unknown>;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
return connection as Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Single source of truth for where the git-repo context-source drivers
|
||||
* (dbt, metricflow, lookml) store their repository fields: dbt uses snake_case
|
||||
* `repo_url`/`source_dir`, lookml uses camelCase `repoUrl`, metricflow nests
|
||||
* camelCase fields under `metricflow.*`.
|
||||
*/
|
||||
export function readGitRepoConnectionFields(
|
||||
connection: KtxProjectConnectionConfig,
|
||||
driver: string,
|
||||
): GitRepoConnectionFields {
|
||||
const record = fieldRecord(connection, driver);
|
||||
return {
|
||||
repoUrl: trimmedString(driver === 'dbt' ? record.repo_url : record.repoUrl),
|
||||
sourceDir: driver === 'dbt' ? trimmedString(record.source_dir) : null,
|
||||
branch: trimmedString(record.branch),
|
||||
path: trimmedString(record.path),
|
||||
authTokenLiteral: trimmedString(record.auth_token),
|
||||
authTokenRef: trimmedString(record.auth_token_ref),
|
||||
};
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { LookerClient } from '../src/context/ingest/adapters/looker/client.js';
|
||||
|
|
@ -470,6 +470,49 @@ describe('runKtxConnection', () => {
|
|||
expect(io.stderr()).toContain('dbt repository check failed: fatal: auth failed');
|
||||
});
|
||||
|
||||
it('tests a local dbt source_dir without a remote repo check', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKtxProject({ projectDir });
|
||||
const sourceDir = join(tempDir, 'local-dbt');
|
||||
await mkdir(sourceDir, { recursive: true });
|
||||
await writeConnections(projectDir, {
|
||||
'dbt-local': {
|
||||
driver: 'dbt',
|
||||
source_dir: sourceDir,
|
||||
},
|
||||
});
|
||||
const testRepoConnection = vi.fn(async () => ({ ok: true as const }));
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKtxConnection({ command: 'test', projectDir, connectionId: 'dbt-local' }, io.io, { testRepoConnection }),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(testRepoConnection).not.toHaveBeenCalled();
|
||||
expect(io.stdout()).toContain('Connection test passed: dbt-local');
|
||||
expect(io.stdout()).toContain(`Repo: ${sourceDir}`);
|
||||
});
|
||||
|
||||
it('fails a dbt source_dir that does not exist', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKtxProject({ projectDir });
|
||||
await writeConnections(projectDir, {
|
||||
'dbt-local': {
|
||||
driver: 'dbt',
|
||||
source_dir: join(tempDir, 'missing-dbt'),
|
||||
},
|
||||
});
|
||||
const testRepoConnection = vi.fn(async () => ({ ok: true as const }));
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKtxConnection({ command: 'test', projectDir, connectionId: 'dbt-local' }, io.io, { testRepoConnection }),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(testRepoConnection).not.toHaveBeenCalled();
|
||||
expect(io.stderr()).toContain('does not exist');
|
||||
});
|
||||
|
||||
it('tests a LookML connection via testRepoConnection with camelCase repoUrl', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKtxProject({ projectDir });
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue