Initial open-source release

packages/context/src/ingest/dbt-shared/project-vars.test.ts

@@ -0,0 +1,118 @@
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import {
+  loadProjectInfo,
+  parseProjectName,
+  parseProjectVars,
+  resolveJinjaVariables,
+} from './project-vars.js';
+
+function entries(map: Map<string, string>): Record<string, string> {
+  return Object.fromEntries([...map.entries()].sort(([a], [b]) => a.localeCompare(b)));
+}
+
+describe('dbt-shared project vars', () => {
+  let tmpRoot: string;
+
+  beforeEach(async () => {
+    tmpRoot = await mkdtemp(join(tmpdir(), 'dbt-project-vars-'));
+  });
+
+  afterEach(async () => {
+    await rm(tmpRoot, { recursive: true, force: true });
+  });
+
+  it('extracts top-level vars, nested dotted vars, and scalar values only', () => {
+    const vars = parseProjectVars(`
+name: revenue_project
+vars:
+  database: analytics
+  enabled: true
+  threads: 4
+  ignored_list:
+    - a
+  ignored_null:
+  pkg:
+    region: us
+    fiscal_year: 2026
+`);
+
+    expect(entries(vars)).toEqual({
+      database: 'analytics',
+      enabled: 'true',
+      'pkg.fiscal_year': '2026',
+      'pkg.region': 'us',
+      threads: '4',
+    });
+  });
+
+  it('returns an empty variable map for missing vars, malformed YAML, arrays, and scalar documents', () => {
+    expect(entries(parseProjectVars('name: no_vars\n'))).toEqual({});
+    expect(entries(parseProjectVars('{{{{ invalid yaml'))).toEqual({});
+    expect(entries(parseProjectVars('- just\n- a\n- list\n'))).toEqual({});
+  });
+
+  it('extracts a string project name and returns null for invalid or missing names', () => {
+    expect(parseProjectName('name: revenue_project\n')).toBe('revenue_project');
+    expect(parseProjectName('version: 1\n')).toBeNull();
+    expect(parseProjectName('{{{{ invalid yaml')).toBeNull();
+    expect(parseProjectName('name: 42\n')).toBeNull();
+  });
+
+  it('resolves exact var names, honors defaults, and reports unresolved names without throwing', () => {
+    const variables = new Map<string, string>([
+      ['database', 'analytics'],
+      ['pkg.region', 'us'],
+    ]);
+
+    const result = resolveJinjaVariables(
+      [
+        'database: "{{ var(\'database\') }}"',
+        'region: "{{ var("pkg.region") }}"',
+        'schema: "{{ var(\'schema\', \'public\') }}"',
+        'missing: "{{ var(\'missing\') }}"',
+      ].join('\n'),
+      variables,
+    );
+
+    expect(result.content).toContain('database: "analytics"');
+    expect(result.content).toContain('region: "us"');
+    expect(result.content).toContain('schema: "public"');
+    expect(result.content).toContain('missing: "{{ var(\'missing\') }}"');
+    expect(result.unresolvedVars).toEqual(['missing']);
+  });
+
+  it('keeps package-scoped variables exact and does not resolve by suffix', () => {
+    const variables = parseProjectVars(`
+vars:
+  pkg:
+    database: package_db
+`);
+
+    const result = resolveJinjaVariables(
+      'database: "{{ var(\'database\', \'fallback_db\') }}"\npackage_database: "{{ var(\'pkg.database\') }}"\n',
+      variables,
+    );
+
+    expect(result.content).toContain('database: "fallback_db"');
+    expect(result.content).toContain('package_database: "package_db"');
+    expect(result.unresolvedVars).toEqual([]);
+  });
+
+  it('loads dbt_project.yml before dbt_project.yaml and falls back to an empty project info object', async () => {
+    const projectDir = join(tmpRoot, 'project');
+    await mkdir(projectDir, { recursive: true });
+    await writeFile(join(projectDir, 'dbt_project.yaml'), 'name: yaml_project\nvars:\n  database: yaml_db\n');
+    await writeFile(join(projectDir, 'dbt_project.yml'), 'name: yml_project\nvars:\n  database: yml_db\n');
+
+    const loaded = await loadProjectInfo(projectDir);
+    expect(loaded.projectName).toBe('yml_project');
+    expect(entries(loaded.variables)).toEqual({ database: 'yml_db' });
+
+    const missing = await loadProjectInfo(join(tmpRoot, 'missing'));
+    expect(missing.projectName).toBeNull();
+    expect(entries(missing.variables)).toEqual({});
+  });
+});

packages/context/src/ingest/dbt-shared/project-vars.ts

@@ -0,0 +1,121 @@
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { parse as parseYaml } from 'yaml';
+
+interface DbtProjectYaml {
+  name?: unknown;
+  vars?: unknown;
+  [key: string]: unknown;
+}
+
+export interface DbtProjectInfo {
+  variables: Map<string, string>;
+  projectName: string | null;
+}
+
+export interface ResolveJinjaVariablesResult {
+  content: string;
+  unresolvedVars: string[];
+}
+
+export function parseProjectVars(yamlContent: string): Map<string, string> {
+  const variables = new Map<string, string>();
+  const project = parseProjectYaml(yamlContent);
+
+  if (!isRecord(project) || !isRecord(project.vars)) {
+    return variables;
+  }
+
+  extractVariables(project.vars, '', variables);
+  return variables;
+}
+
+export function parseProjectName(yamlContent: string): string | null {
+  const project = parseProjectYaml(yamlContent);
+
+  if (!isRecord(project) || typeof project.name !== 'string') {
+    return null;
+  }
+
+  return project.name;
+}
+
+export async function loadProjectInfo(projectDir: string): Promise<DbtProjectInfo> {
+  for (const fileName of ['dbt_project.yml', 'dbt_project.yaml']) {
+    const filePath = join(projectDir, fileName);
+    try {
+      const content = await readFile(filePath, 'utf-8');
+      return {
+        variables: parseProjectVars(content),
+        projectName: parseProjectName(content),
+      };
+    } catch {
+      // Try the next dbt project filename.
+    }
+  }
+
+  return { variables: new Map(), projectName: null };
+}
+
+export function resolveJinjaVariables(
+  content: string,
+  variables: Map<string, string>,
+): ResolveJinjaVariablesResult {
+  const varPattern = /\{\{\s*var\s*\(\s*['"]([^'"]+)['"]\s*(?:,\s*['"]([^'"]*)['"]\s*)?\)\s*\}\}/g;
+  const unresolvedVars = new Set<string>();
+
+  const resolvedContent = content.replace(
+    varPattern,
+    (fullMatch, varName: string, defaultValue: string | undefined) => {
+      const value = variables.get(varName);
+      if (value !== undefined) {
+        return value;
+      }
+
+      if (defaultValue !== undefined) {
+        return defaultValue;
+      }
+
+      unresolvedVars.add(varName);
+      return fullMatch;
+    },
+  );
+
+  return {
+    content: resolvedContent,
+    unresolvedVars: [...unresolvedVars].sort(),
+  };
+}
+
+function parseProjectYaml(yamlContent: string): DbtProjectYaml | null {
+  try {
+    const parsed = parseYaml(yamlContent) as unknown;
+    return isRecord(parsed) ? parsed : null;
+  } catch {
+    return null;
+  }
+}
+
+function extractVariables(obj: Record<string, unknown>, prefix: string, variables: Map<string, string>): void {
+  for (const [key, value] of Object.entries(obj)) {
+    const fullKey = prefix ? `${prefix}.${key}` : key;
+
+    if (value === null || value === undefined) {
+      continue;
+    }
+
+    if (typeof value === 'string') {
+      variables.set(fullKey, value);
+    } else if (typeof value === 'number' || typeof value === 'boolean') {
+      variables.set(fullKey, String(value));
+    } else if (Array.isArray(value)) {
+      continue;
+    } else if (isRecord(value)) {
+      extractVariables(value, fullKey, variables);
+    }
+  }
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}

packages/context/src/ingest/dbt-shared/schema-files.test.ts

@@ -0,0 +1,41 @@
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { findDbtSchemaFiles, loadDbtSchemaFiles } from './schema-files.js';
+
+describe('dbt shared schema files', () => {
+  let tmpRoot: string;
+
+  beforeEach(async () => {
+    tmpRoot = await mkdtemp(join(tmpdir(), 'dbt-schema-files-'));
+  });
+
+  afterEach(async () => {
+    await rm(tmpRoot, { recursive: true, force: true });
+  });
+
+  it('loads schema yaml files from dbt search directories and skips project config files', async () => {
+    await mkdir(join(tmpRoot, 'models', 'nested'), { recursive: true });
+    await mkdir(join(tmpRoot, 'seeds'), { recursive: true });
+    await writeFile(join(tmpRoot, 'dbt_project.yml'), 'name: ignored\n');
+    await writeFile(join(tmpRoot, 'packages.yml'), 'packages: []\n');
+    await writeFile(join(tmpRoot, 'models', 'schema.yml'), 'version: 2\nmodels: []\n');
+    await writeFile(join(tmpRoot, 'models', 'nested', 'customers.yaml'), 'version: 2\nmodels: []\n');
+    await writeFile(join(tmpRoot, 'seeds', 'seed.yml'), 'version: 2\nseeds: []\n');
+
+    const paths = await findDbtSchemaFiles(tmpRoot);
+    expect(paths.map((path) => path.replace(`${tmpRoot}/`, '')).sort()).toEqual([
+      'models/nested/customers.yaml',
+      'models/schema.yml',
+      'seeds/seed.yml',
+    ]);
+
+    const files = await loadDbtSchemaFiles(tmpRoot);
+    expect(files.map((file) => file.path).sort()).toEqual([
+      'models/nested/customers.yaml',
+      'models/schema.yml',
+      'seeds/seed.yml',
+    ]);
+  });
+});

packages/context/src/ingest/dbt-shared/schema-files.ts

@@ -0,0 +1,76 @@
+import { promises as fs } from 'node:fs';
+import { join, relative } from 'node:path';
+import type { DbtSchemaFile } from '../adapters/dbt-descriptions/parse-schema.js';
+
+const DBT_SCHEMA_SEARCH_DIRS = ['models', 'seeds', 'snapshots', 'analyses', '.'] as const;
+const DBT_CONFIG_YAML_FILES = new Set([
+  'dbt_project.yml',
+  'dbt_project.yaml',
+  'packages.yml',
+  'packages.yaml',
+  'selectors.yml',
+  'selectors.yaml',
+]);
+
+export async function loadDbtSchemaFiles(projectDir: string): Promise<DbtSchemaFile[]> {
+  const schemaFiles = await findDbtSchemaFiles(projectDir);
+  return Promise.all(
+    schemaFiles.map(async (filePath) => ({
+      content: await fs.readFile(filePath, 'utf-8'),
+      path: relative(projectDir, filePath),
+    })),
+  );
+}
+
+export async function findDbtSchemaFiles(projectDir: string): Promise<string[]> {
+  const schemaFiles: string[] = [];
+
+  for (const dir of DBT_SCHEMA_SEARCH_DIRS) {
+    const searchPath = join(projectDir, dir);
+    try {
+      await fs.access(searchPath);
+      schemaFiles.push(...(await findYamlFilesRecursive(searchPath)));
+    } catch {
+      // Missing dbt search directories are normal.
+    }
+  }
+
+  return [...new Set(schemaFiles)].sort();
+}
+
+async function findYamlFilesRecursive(dir: string): Promise<string[]> {
+  const files: string[] = [];
+
+  let entries;
+  try {
+    entries = await fs.readdir(dir, { withFileTypes: true });
+  } catch {
+    return files;
+  }
+
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name);
+
+    if (entry.isDirectory()) {
+      if (!entry.name.startsWith('.') && entry.name !== 'node_modules') {
+        files.push(...(await findYamlFilesRecursive(fullPath)));
+      }
+      continue;
+    }
+
+    if (!entry.isFile()) {
+      continue;
+    }
+
+    const name = entry.name.toLowerCase();
+    if (DBT_CONFIG_YAML_FILES.has(name)) {
+      continue;
+    }
+
+    if (name.endsWith('.yml') || name.endsWith('.yaml')) {
+      files.push(fullPath);
+    }
+  }
+
+  return files;
+}