fix(cli): support local dbt source_dir in ktx connection test

Extract readGitRepoConnectionFields as the single reader of where the git-repo
context-source drivers store their fields (dbt's snake_case repo_url/source_dir,
lookml's camelCase repoUrl, metricflow's nested metricflow.*). `ktx connection
test` now routes through it and, when a dbt connection points at a local
source_dir, verifies the directory exists instead of running a remote repo
reachability check that can never apply.
This commit is contained in:
Andrey Avtomonov 2026-06-18 13:34:52 +02:00
parent 7e5551906e
commit 834e60625c
3 changed files with 123 additions and 33 deletions

View file

@ -1,3 +1,5 @@
import { stat } from 'node:fs/promises';
import { resolve } from 'node:path';
import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory } from './context/ingest/adapters/metabase/client.js';
import { DefaultLookerConnectionClientFactory } from './context/ingest/adapters/looker/factory.js';
import type { LookerClient } from './context/ingest/adapters/looker/client.js';
@ -9,6 +11,7 @@ import { testRepoConnection } from './context/ingest/repo-fetch.js';
import { getDriverRegistration } from './context/connections/drivers.js';
import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from './context/connections/notion-config.js';
import { resolveKtxConfigReference } from './context/core/config-reference.js';
import { readGitRepoConnectionFields } from './context/project/git-connection-fields.js';
import { type KtxLocalProject, loadKtxProject } from './context/project/project.js';
import type { KtxScanConnector } from './context/scan/types.js';
import type { KtxCliIo } from './index.js';
@ -182,35 +185,14 @@ async function testNotionConnection(
return { bot: describeNotionBot(bot) };
}
interface GitConnectionFields {
repoUrl: string;
authToken: string | null;
}
function extractGitConnectionFields(
project: KtxLocalProject,
connectionId: string,
driver: string,
): GitConnectionFields {
const connection = project.config.connections[connectionId];
if (!connection) {
throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
async function checkLocalDbtSourceDir(projectDir: string, sourceDir: string): Promise<void> {
const stats = await stat(resolve(projectDir, sourceDir)).catch(() => null);
if (!stats) {
throw new Error(`dbt source_dir "${sourceDir}" does not exist`);
}
const stringField = (value: unknown): string | null =>
typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
const record =
driver === 'metricflow' && typeof connection.metricflow === 'object' && connection.metricflow !== null
? (connection.metricflow as Record<string, unknown>)
: (connection as Record<string, unknown>);
const repoUrl = driver === 'dbt' ? stringField(record.repo_url) : stringField(record.repoUrl);
if (!repoUrl) {
const field = driver === 'dbt' ? 'repo_url' : 'repoUrl';
throw new Error(`Connection "${connectionId}" (driver: ${driver}) is missing ${field}`);
if (!stats.isDirectory()) {
throw new Error(`dbt source_dir "${sourceDir}" is not a directory`);
}
const literalToken = stringField(record.auth_token);
const ref = stringField(record.auth_token_ref);
const resolvedRef = ref ? resolveKtxConfigReference(ref, process.env) : null;
return { repoUrl, authToken: literalToken ?? resolvedRef ?? null };
}
async function testGitRepoConnection(
@ -218,13 +200,31 @@ async function testGitRepoConnection(
connectionId: string,
driver: string,
runTest: TestRepoConnection,
): Promise<{ repoUrl: string }> {
const { repoUrl, authToken } = extractGitConnectionFields(project, connectionId, driver);
const result = await runTest({ repoUrl, authToken });
): Promise<{ location: string }> {
const connection = project.config.connections[connectionId];
if (!connection) {
throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
}
const fields = readGitRepoConnectionFields(connection, driver);
// dbt setup writes source_dir for a local project and ingest consumes it, so a
// local source check must stand in for the remote repo reachability check.
if (fields.sourceDir) {
await checkLocalDbtSourceDir(project.projectDir, fields.sourceDir);
return { location: fields.sourceDir };
}
if (!fields.repoUrl) {
const field = driver === 'dbt' ? 'repo_url or source_dir' : 'repoUrl';
throw new Error(`Connection "${connectionId}" (driver: ${driver}) is missing ${field}`);
}
const resolvedRef = fields.authTokenRef ? resolveKtxConfigReference(fields.authTokenRef, process.env) : null;
const authToken = fields.authTokenLiteral ?? resolvedRef ?? null;
const result = await runTest({ repoUrl: fields.repoUrl, authToken });
if (!result.ok) {
throw new Error(`${driver} repository check failed: ${result.error}`);
}
return { repoUrl };
return { location: fields.repoUrl };
}
interface DriverTestOutcome {
@ -277,7 +277,7 @@ async function testConnectionByDriver(
driver,
deps.testRepoConnection ?? testRepoConnection,
);
return { driver, detailKey: 'Repo', detailValue: result.repoUrl };
return { driver, detailKey: 'Repo', detailValue: result.location };
}
if (getDriverRegistration(driver)) {

View file

@ -0,0 +1,47 @@
import type { KtxProjectConnectionConfig } from './config.js';
export interface GitRepoConnectionFields {
repoUrl: string | null;
sourceDir: string | null;
branch: string | null;
path: string | null;
authTokenLiteral: string | null;
authTokenRef: string | null;
}
function trimmedString(value: unknown): string | null {
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
}
// metricflow nests its repo fields under `metricflow.*`; dbt/lookml keep them top-level.
function fieldRecord(connection: KtxProjectConnectionConfig, driver: string): Record<string, unknown> {
if (driver === 'metricflow') {
const nested = (connection as Record<string, unknown>).metricflow;
if (typeof nested === 'object' && nested !== null && !Array.isArray(nested)) {
return nested as Record<string, unknown>;
}
return {};
}
return connection as Record<string, unknown>;
}
/**
* Single source of truth for where the git-repo context-source drivers
* (dbt, metricflow, lookml) store their repository fields: dbt uses snake_case
* `repo_url`/`source_dir`, lookml uses camelCase `repoUrl`, metricflow nests
* camelCase fields under `metricflow.*`.
*/
export function readGitRepoConnectionFields(
connection: KtxProjectConnectionConfig,
driver: string,
): GitRepoConnectionFields {
const record = fieldRecord(connection, driver);
return {
repoUrl: trimmedString(driver === 'dbt' ? record.repo_url : record.repoUrl),
sourceDir: driver === 'dbt' ? trimmedString(record.source_dir) : null,
branch: trimmedString(record.branch),
path: trimmedString(record.path),
authTokenLiteral: trimmedString(record.auth_token),
authTokenRef: trimmedString(record.auth_token_ref),
};
}

View file

@ -1,4 +1,4 @@
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { LookerClient } from '../src/context/ingest/adapters/looker/client.js';
@ -470,6 +470,49 @@ describe('runKtxConnection', () => {
expect(io.stderr()).toContain('dbt repository check failed: fatal: auth failed');
});
it('tests a local dbt source_dir without a remote repo check', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir });
const sourceDir = join(tempDir, 'local-dbt');
await mkdir(sourceDir, { recursive: true });
await writeConnections(projectDir, {
'dbt-local': {
driver: 'dbt',
source_dir: sourceDir,
},
});
const testRepoConnection = vi.fn(async () => ({ ok: true as const }));
const io = makeIo();
await expect(
runKtxConnection({ command: 'test', projectDir, connectionId: 'dbt-local' }, io.io, { testRepoConnection }),
).resolves.toBe(0);
expect(testRepoConnection).not.toHaveBeenCalled();
expect(io.stdout()).toContain('Connection test passed: dbt-local');
expect(io.stdout()).toContain(`Repo: ${sourceDir}`);
});
it('fails a dbt source_dir that does not exist', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir });
await writeConnections(projectDir, {
'dbt-local': {
driver: 'dbt',
source_dir: join(tempDir, 'missing-dbt'),
},
});
const testRepoConnection = vi.fn(async () => ({ ok: true as const }));
const io = makeIo();
await expect(
runKtxConnection({ command: 'test', projectDir, connectionId: 'dbt-local' }, io.io, { testRepoConnection }),
).resolves.toBe(1);
expect(testRepoConnection).not.toHaveBeenCalled();
expect(io.stderr()).toContain('does not exist');
});
it('tests a LookML connection via testRepoConnection with camelCase repoUrl', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir });