diff --git a/packages/cli/src/connection.ts b/packages/cli/src/connection.ts index 1e267833..facadd01 100644 --- a/packages/cli/src/connection.ts +++ b/packages/cli/src/connection.ts @@ -1,3 +1,5 @@ +import { stat } from 'node:fs/promises'; +import { resolve } from 'node:path'; import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory } from './context/ingest/adapters/metabase/client.js'; import { DefaultLookerConnectionClientFactory } from './context/ingest/adapters/looker/factory.js'; import type { LookerClient } from './context/ingest/adapters/looker/client.js'; @@ -9,6 +11,7 @@ import { testRepoConnection } from './context/ingest/repo-fetch.js'; import { getDriverRegistration } from './context/connections/drivers.js'; import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from './context/connections/notion-config.js'; import { resolveKtxConfigReference } from './context/core/config-reference.js'; +import { readGitRepoConnectionFields } from './context/project/git-connection-fields.js'; import { type KtxLocalProject, loadKtxProject } from './context/project/project.js'; import type { KtxScanConnector } from './context/scan/types.js'; import type { KtxCliIo } from './index.js'; @@ -182,35 +185,14 @@ async function testNotionConnection( return { bot: describeNotionBot(bot) }; } -interface GitConnectionFields { - repoUrl: string; - authToken: string | null; -} - -function extractGitConnectionFields( - project: KtxLocalProject, - connectionId: string, - driver: string, -): GitConnectionFields { - const connection = project.config.connections[connectionId]; - if (!connection) { - throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`); +async function checkLocalDbtSourceDir(projectDir: string, sourceDir: string): Promise { + const stats = await stat(resolve(projectDir, sourceDir)).catch(() => null); + if (!stats) { + throw new Error(`dbt source_dir "${sourceDir}" does not exist`); } - const stringField = (value: unknown): string | null => - typeof value === 'string' && value.trim().length > 0 ? value.trim() : null; - const record = - driver === 'metricflow' && typeof connection.metricflow === 'object' && connection.metricflow !== null - ? (connection.metricflow as Record) - : (connection as Record); - const repoUrl = driver === 'dbt' ? stringField(record.repo_url) : stringField(record.repoUrl); - if (!repoUrl) { - const field = driver === 'dbt' ? 'repo_url' : 'repoUrl'; - throw new Error(`Connection "${connectionId}" (driver: ${driver}) is missing ${field}`); + if (!stats.isDirectory()) { + throw new Error(`dbt source_dir "${sourceDir}" is not a directory`); } - const literalToken = stringField(record.auth_token); - const ref = stringField(record.auth_token_ref); - const resolvedRef = ref ? resolveKtxConfigReference(ref, process.env) : null; - return { repoUrl, authToken: literalToken ?? resolvedRef ?? null }; } async function testGitRepoConnection( @@ -218,13 +200,31 @@ async function testGitRepoConnection( connectionId: string, driver: string, runTest: TestRepoConnection, -): Promise<{ repoUrl: string }> { - const { repoUrl, authToken } = extractGitConnectionFields(project, connectionId, driver); - const result = await runTest({ repoUrl, authToken }); +): Promise<{ location: string }> { + const connection = project.config.connections[connectionId]; + if (!connection) { + throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`); + } + const fields = readGitRepoConnectionFields(connection, driver); + + // dbt setup writes source_dir for a local project and ingest consumes it, so a + // local source check must stand in for the remote repo reachability check. + if (fields.sourceDir) { + await checkLocalDbtSourceDir(project.projectDir, fields.sourceDir); + return { location: fields.sourceDir }; + } + + if (!fields.repoUrl) { + const field = driver === 'dbt' ? 'repo_url or source_dir' : 'repoUrl'; + throw new Error(`Connection "${connectionId}" (driver: ${driver}) is missing ${field}`); + } + const resolvedRef = fields.authTokenRef ? resolveKtxConfigReference(fields.authTokenRef, process.env) : null; + const authToken = fields.authTokenLiteral ?? resolvedRef ?? null; + const result = await runTest({ repoUrl: fields.repoUrl, authToken }); if (!result.ok) { throw new Error(`${driver} repository check failed: ${result.error}`); } - return { repoUrl }; + return { location: fields.repoUrl }; } interface DriverTestOutcome { @@ -277,7 +277,7 @@ async function testConnectionByDriver( driver, deps.testRepoConnection ?? testRepoConnection, ); - return { driver, detailKey: 'Repo', detailValue: result.repoUrl }; + return { driver, detailKey: 'Repo', detailValue: result.location }; } if (getDriverRegistration(driver)) { diff --git a/packages/cli/src/context/project/git-connection-fields.ts b/packages/cli/src/context/project/git-connection-fields.ts new file mode 100644 index 00000000..4cec6edc --- /dev/null +++ b/packages/cli/src/context/project/git-connection-fields.ts @@ -0,0 +1,47 @@ +import type { KtxProjectConnectionConfig } from './config.js'; + +export interface GitRepoConnectionFields { + repoUrl: string | null; + sourceDir: string | null; + branch: string | null; + path: string | null; + authTokenLiteral: string | null; + authTokenRef: string | null; +} + +function trimmedString(value: unknown): string | null { + return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null; +} + +// metricflow nests its repo fields under `metricflow.*`; dbt/lookml keep them top-level. +function fieldRecord(connection: KtxProjectConnectionConfig, driver: string): Record { + if (driver === 'metricflow') { + const nested = (connection as Record).metricflow; + if (typeof nested === 'object' && nested !== null && !Array.isArray(nested)) { + return nested as Record; + } + return {}; + } + return connection as Record; +} + +/** + * Single source of truth for where the git-repo context-source drivers + * (dbt, metricflow, lookml) store their repository fields: dbt uses snake_case + * `repo_url`/`source_dir`, lookml uses camelCase `repoUrl`, metricflow nests + * camelCase fields under `metricflow.*`. + */ +export function readGitRepoConnectionFields( + connection: KtxProjectConnectionConfig, + driver: string, +): GitRepoConnectionFields { + const record = fieldRecord(connection, driver); + return { + repoUrl: trimmedString(driver === 'dbt' ? record.repo_url : record.repoUrl), + sourceDir: driver === 'dbt' ? trimmedString(record.source_dir) : null, + branch: trimmedString(record.branch), + path: trimmedString(record.path), + authTokenLiteral: trimmedString(record.auth_token), + authTokenRef: trimmedString(record.auth_token_ref), + }; +} diff --git a/packages/cli/test/connection.test.ts b/packages/cli/test/connection.test.ts index 22c8bbe9..4069ba98 100644 --- a/packages/cli/test/connection.test.ts +++ b/packages/cli/test/connection.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { LookerClient } from '../src/context/ingest/adapters/looker/client.js'; @@ -470,6 +470,49 @@ describe('runKtxConnection', () => { expect(io.stderr()).toContain('dbt repository check failed: fatal: auth failed'); }); + it('tests a local dbt source_dir without a remote repo check', async () => { + const projectDir = join(tempDir, 'project'); + await initKtxProject({ projectDir }); + const sourceDir = join(tempDir, 'local-dbt'); + await mkdir(sourceDir, { recursive: true }); + await writeConnections(projectDir, { + 'dbt-local': { + driver: 'dbt', + source_dir: sourceDir, + }, + }); + const testRepoConnection = vi.fn(async () => ({ ok: true as const })); + const io = makeIo(); + + await expect( + runKtxConnection({ command: 'test', projectDir, connectionId: 'dbt-local' }, io.io, { testRepoConnection }), + ).resolves.toBe(0); + + expect(testRepoConnection).not.toHaveBeenCalled(); + expect(io.stdout()).toContain('Connection test passed: dbt-local'); + expect(io.stdout()).toContain(`Repo: ${sourceDir}`); + }); + + it('fails a dbt source_dir that does not exist', async () => { + const projectDir = join(tempDir, 'project'); + await initKtxProject({ projectDir }); + await writeConnections(projectDir, { + 'dbt-local': { + driver: 'dbt', + source_dir: join(tempDir, 'missing-dbt'), + }, + }); + const testRepoConnection = vi.fn(async () => ({ ok: true as const })); + const io = makeIo(); + + await expect( + runKtxConnection({ command: 'test', projectDir, connectionId: 'dbt-local' }, io.io, { testRepoConnection }), + ).resolves.toBe(1); + + expect(testRepoConnection).not.toHaveBeenCalled(); + expect(io.stderr()).toContain('does not exist'); + }); + it('tests a LookML connection via testRepoConnection with camelCase repoUrl', async () => { const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir });