mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-16 08:25:14 +02:00
refactor(cli): delete internal barrel index.ts files
The 34 `index.ts` re-export barrels inside `packages/cli/src/` were holdovers from the pre-fold multi-workspace structure. Post-fold-in they served no production purpose: external consumers go through the single package main entry, and in-repo callers mostly imported through them only because the path was short. Internally, knip flagged most barrel re-exports as production-dead (only reached via tests). This change: - Deletes every internal barrel except `packages/cli/src/index.ts` (the published package entry). - Rewrites ~270 source/test files to import each name directly from the file that defines it. - Moves `tools/warehouse-verification/index.ts` to `create-warehouse-verification-tools.ts` (the function it defined locally) and updates its single consumer. - Renames `search/backend-conformance.ts` → `.test-utils.ts` to match the existing test-helper file convention. - Deletes 13 dead test-only chains (dbt-descriptions/*, live-database/extracted-schema, live-database/structural-sync, relationship-* feedback/review chain) plus their tests and a cascading orphan integration test. - Updates test mocks that pointed at deleted barrel paths (notion-client, connector barrels in scan/local-scan-connectors tests) to mock the source files instead. - Points the maintainer benchmark script (`scripts/relationship-benchmark-report.mjs`) at source files instead of `dist/context/scan/index.js`. - Drops the barrel `!` entries from `knip.json`; adds explicit production entries only for the benchmark code reached via dist by the maintainer script. Net: 413 files changed, ~1.2k insertions, ~9.4k deletions. `pnpm run dead-code` (Biome + knip default + knip production) and `pnpm run type-check` are clean; 2277 tests pass.
This commit is contained in:
Andrey Avtomonov
parent
b690e6988b
commit
34d4a1e9e1
413 changed files with 1260 additions and 8739 deletions
|
|
@ -1,9 +0,0 @@
|
|||
export type {
|
||||
AgentRunnerPort,
|
||||
RunLoopParams,
|
||||
RunLoopResult,
|
||||
RunLoopStepInfo,
|
||||
RunLoopStopReason,
|
||||
} from '../llm/runtime-port.js';
|
||||
export { RuntimeAgentRunner } from '../llm/runtime-port.js';
|
||||
export type { AgentTelemetryPort } from '../llm/ai-sdk-runtime.js';
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import type { KtxSchemaDimensionType, KtxTableRef } from '../scan/types.js';
|
||||
|
||||
export type SupportedDriver =
|
||||
type SupportedDriver =
|
||||
| 'postgres'
|
||||
| 'postgresql'
|
||||
| 'mysql'
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
export type {
|
||||
KtxSqlQueryExecutionInput,
|
||||
KtxSqlQueryExecutionResult,
|
||||
KtxSqlQueryExecutorPort,
|
||||
} from './query-executor.js';
|
||||
export type { KtxDialect, SupportedDriver } from './dialects.js';
|
||||
export { createDefaultLocalQueryExecutor, type DefaultLocalQueryExecutorOptions } from './local-query-executor.js';
|
||||
export { getDialectForDriver } from './dialects.js';
|
||||
export { normalizeQueryRows } from './query-executor.js';
|
||||
export { createPostgresQueryExecutor } from './postgres-query-executor.js';
|
||||
export { assertReadOnlySql, limitSqlForExecution } from './read-only-sql.js';
|
||||
export { createSqliteQueryExecutor, sqliteDatabasePathFromConnection } from './sqlite-query-executor.js';
|
||||
export { connectionTypeSchema, type ConnectionType } from './connection-type.js';
|
||||
export {
|
||||
localConnectionInfoFromConfig,
|
||||
localConnectionToWarehouseDescriptor,
|
||||
localConnectionTypeForConfig,
|
||||
type LocalConnectionInfo,
|
||||
type LocalWarehouseDescriptor,
|
||||
} from './local-warehouse-descriptor.js';
|
||||
export {
|
||||
KTX_NOTION_ORG_KNOWLEDGE_WARNING,
|
||||
notionConnectionToPullConfig,
|
||||
parseNotionConnectionConfig,
|
||||
redactNotionConnectionConfig,
|
||||
resolveNotionConnectionAuthToken,
|
||||
resolveNotionAuthToken,
|
||||
type KtxNotionConnectionConfig,
|
||||
type RedactedKtxNotionConnectionConfig,
|
||||
} from './notion-config.js';
|
||||
|
|
@ -8,7 +8,7 @@ import {
|
|||
} from '../ingest/adapters/notion/types.js';
|
||||
import type { KtxProjectConnectionConfig } from '../project/config.js';
|
||||
|
||||
export const KTX_NOTION_ORG_KNOWLEDGE_WARNING =
|
||||
const KTX_NOTION_ORG_KNOWLEDGE_WARNING =
|
||||
'Anything accessible to this Notion integration can become organization knowledge.';
|
||||
|
||||
type KtxNotionCrawlMode = 'all_accessible' | 'selected_roots';
|
||||
|
|
@ -39,6 +39,7 @@ export type KtxNotionConnectionConfig = Omit<
|
|||
max_knowledge_updates_per_run: number;
|
||||
};
|
||||
|
||||
/** @internal */
|
||||
export interface RedactedKtxNotionConnectionConfig {
|
||||
driver: 'notion';
|
||||
hasAuthToken: boolean;
|
||||
|
|
@ -152,6 +153,7 @@ export function parseNotionConnectionConfig(raw: unknown): KtxNotionConnectionCo
|
|||
};
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function redactNotionConnectionConfig(config: KtxNotionConnectionConfig): RedactedKtxNotionConnectionConfig {
|
||||
return {
|
||||
driver: 'notion',
|
||||
|
|
@ -171,6 +173,7 @@ function expandHome(path: string): string {
|
|||
return path === '~' || path.startsWith('~/') ? resolve(homedir(), path.slice(2)) : path;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export async function resolveNotionAuthToken(
|
||||
authTokenRef: string,
|
||||
options: ResolveNotionTokenOptions = {},
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KtxProjectConnectionConfig } from '../project/index.js';
|
||||
import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
|
||||
|
||||
export interface KtxSqlQueryExecutionInput {
|
||||
connectionId: string;
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ function sqlitePathFromUrl(url: string): string {
|
|||
return url;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function sqliteDatabasePathFromConnection(input: KtxSqlQueryExecutionInput): string {
|
||||
const driver = connectionDriver(input);
|
||||
if (driver !== 'sqlite' && driver !== 'sqlite3') {
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { readFileSync } from 'node:fs';
|
|||
import { homedir } from 'node:os';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
/** @internal */
|
||||
export function resolveKtxHomePath(path: string): string {
|
||||
if (path === '~') {
|
||||
return homedir();
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
export interface KtxStorageConfig {
|
||||
interface KtxStorageConfig {
|
||||
configDir?: string;
|
||||
homeDir?: string;
|
||||
worktreesDir?: string;
|
||||
}
|
||||
|
||||
export interface KtxGitConfig {
|
||||
interface KtxGitConfig {
|
||||
userName: string;
|
||||
userEmail: string;
|
||||
bootstrapMessage?: string;
|
||||
|
|
|
|||
|
|
@ -1,27 +0,0 @@
|
|||
export type { KtxCoreConfig, KtxGitConfig, KtxLogger, KtxStorageConfig } from './config.js';
|
||||
export { noopLogger, resolveConfigDir, resolveWorktreesDir } from './config.js';
|
||||
export { resolveKtxConfigReference, resolveKtxHomePath } from './config-reference.js';
|
||||
export type { KtxEmbeddingPort } from './embedding.js';
|
||||
export {
|
||||
REDACTED_KTX_CREDENTIAL_VALUE,
|
||||
redactKtxSensitiveMetadata,
|
||||
redactKtxSensitiveText,
|
||||
redactKtxSensitiveValue,
|
||||
} from './redaction.js';
|
||||
export type {
|
||||
KtxFileHistoryEntry,
|
||||
KtxFileListResult,
|
||||
KtxFileReadResult,
|
||||
KtxFileStorePort,
|
||||
KtxFileWriteResult,
|
||||
} from './file-store.js';
|
||||
export type { GitCommitInfo, SquashMergeResult, WorktreeEntry } from './git.service.js';
|
||||
export { GitService } from './git.service.js';
|
||||
export type {
|
||||
SentinelPayload,
|
||||
SessionOutcome,
|
||||
SessionWorktree,
|
||||
SessionWorktreeServiceDeps,
|
||||
WorktreeConfigPort,
|
||||
} from './session-worktree.service.js';
|
||||
export { SessionWorktreeService } from './session-worktree.service.js';
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
/** @internal */
|
||||
export const REDACTED_KTX_CREDENTIAL_VALUE = '<redacted>';
|
||||
|
||||
const SENSITIVE_FIELD_NAME = /(password|secret|token|api[_-]?key|private[_-]?key|passphrase|credential|authorization|url)/i;
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import { GitService } from './git.service.js';
|
|||
|
||||
export type SessionOutcome = 'success' | 'empty' | 'conflict' | 'crash';
|
||||
|
||||
export interface SentinelPayload {
|
||||
interface SentinelPayload {
|
||||
outcome: SessionOutcome;
|
||||
at: string;
|
||||
chatId: string;
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
export * from './semantic-layer-compute.js';
|
||||
|
|
@ -4,21 +4,21 @@ import { URL } from 'node:url';
|
|||
import { spawn } from 'node:child_process';
|
||||
import type { ResolvedSemanticLayerSource, SemanticLayerQueryInput } from '../sl/types.js';
|
||||
|
||||
export interface KtxSemanticLayerComputeQueryResult {
|
||||
interface KtxSemanticLayerComputeQueryResult {
|
||||
sql: string;
|
||||
dialect: string;
|
||||
columns: Array<Record<string, unknown>>;
|
||||
plan: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface KtxSemanticLayerComputeValidationResult {
|
||||
interface KtxSemanticLayerComputeValidationResult {
|
||||
valid: boolean;
|
||||
errors: string[];
|
||||
warnings: string[];
|
||||
perSourceWarnings: Record<string, string[]>;
|
||||
}
|
||||
|
||||
export interface KtxSemanticLayerSourceGenerationColumnInput {
|
||||
interface KtxSemanticLayerSourceGenerationColumnInput {
|
||||
name: string;
|
||||
type: string;
|
||||
primaryKey?: boolean;
|
||||
|
|
@ -26,7 +26,7 @@ export interface KtxSemanticLayerSourceGenerationColumnInput {
|
|||
comment?: string | null;
|
||||
}
|
||||
|
||||
export interface KtxSemanticLayerSourceGenerationTableInput {
|
||||
interface KtxSemanticLayerSourceGenerationTableInput {
|
||||
name: string;
|
||||
catalog?: string | null;
|
||||
db?: string | null;
|
||||
|
|
@ -34,7 +34,7 @@ export interface KtxSemanticLayerSourceGenerationTableInput {
|
|||
columns: KtxSemanticLayerSourceGenerationColumnInput[];
|
||||
}
|
||||
|
||||
export interface KtxSemanticLayerSourceGenerationLinkInput {
|
||||
interface KtxSemanticLayerSourceGenerationLinkInput {
|
||||
fromTable: string;
|
||||
fromColumn: string;
|
||||
toTable: string;
|
||||
|
|
@ -42,13 +42,13 @@ export interface KtxSemanticLayerSourceGenerationLinkInput {
|
|||
relationshipType: string;
|
||||
}
|
||||
|
||||
export interface KtxSemanticLayerSourceGenerationInput {
|
||||
interface KtxSemanticLayerSourceGenerationInput {
|
||||
tables: KtxSemanticLayerSourceGenerationTableInput[];
|
||||
links: KtxSemanticLayerSourceGenerationLinkInput[];
|
||||
dialect?: string;
|
||||
}
|
||||
|
||||
export interface KtxSemanticLayerSourceGenerationResult {
|
||||
interface KtxSemanticLayerSourceGenerationResult {
|
||||
sources: Array<Record<string, unknown>>;
|
||||
sourceCount: number;
|
||||
}
|
||||
|
|
@ -75,14 +75,14 @@ export interface KtxSemanticLayerComputePort {
|
|||
generateSources(input: KtxSemanticLayerSourceGenerationInput): Promise<KtxSemanticLayerSourceGenerationResult>;
|
||||
}
|
||||
|
||||
export type KtxDaemonCommand = 'semantic-query' | 'semantic-validate' | 'semantic-generate-sources';
|
||||
type KtxDaemonCommand = 'semantic-query' | 'semantic-validate' | 'semantic-generate-sources';
|
||||
|
||||
export type KtxDaemonJsonRunner = (
|
||||
type KtxDaemonJsonRunner = (
|
||||
subcommand: KtxDaemonCommand,
|
||||
payload: Record<string, unknown>,
|
||||
) => Promise<Record<string, unknown>>;
|
||||
|
||||
export type KtxDaemonHttpJsonRunner = (path: string, payload: Record<string, unknown>) => Promise<Record<string, unknown>>;
|
||||
type KtxDaemonHttpJsonRunner = (path: string, payload: Record<string, unknown>) => Promise<Record<string, unknown>>;
|
||||
|
||||
export interface PythonSemanticLayerComputeOptions {
|
||||
command?: string;
|
||||
|
|
@ -92,6 +92,7 @@ export interface PythonSemanticLayerComputeOptions {
|
|||
runJson?: KtxDaemonJsonRunner;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface HttpSemanticLayerComputeOptions {
|
||||
baseUrl: string;
|
||||
requestJson?: KtxDaemonHttpJsonRunner;
|
||||
|
|
@ -272,6 +273,7 @@ export function createPythonSemanticLayerComputePort(
|
|||
};
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function createHttpSemanticLayerComputePort(
|
||||
options: HttpSemanticLayerComputeOptions,
|
||||
): KtxSemanticLayerComputePort {
|
||||
|
|
|
|||
|
|
@ -1,2 +0,0 @@
|
|||
export type { ReindexOptions, ReindexScopeResult, ReindexSummary, ReindexWorkResult } from './types.js';
|
||||
export { discoverReindexScopes, reindexLocalIndexes } from './reindex.js';
|
||||
|
|
@ -2,8 +2,8 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import type { KtxEmbeddingPort } from '../core/index.js';
|
||||
import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../project/index.js';
|
||||
import type { KtxEmbeddingPort } from '../../context/core/embedding.js';
|
||||
import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../context/project/project.js';
|
||||
import { SqliteKnowledgeIndex } from '../wiki/sqlite-knowledge-index.js';
|
||||
import { reindexLocalIndexes } from './reindex.js';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,12 @@
|
|||
import { readdir, stat } from 'node:fs/promises';
|
||||
import { join, relative } from 'node:path';
|
||||
import { ktxLocalStateDbPath, type KtxLocalProject } from '../project/index.js';
|
||||
import { loadLocalSlSourceRecords, SlSearchService, SqliteSlSourcesIndex } from '../sl/index.js';
|
||||
import { KnowledgeWikiService, SqliteKnowledgeIndex } from '../wiki/index.js';
|
||||
import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js';
|
||||
import type { KtxLocalProject } from '../../context/project/project.js';
|
||||
import { loadLocalSlSourceRecords } from '../../context/sl/local-sl.js';
|
||||
import { SlSearchService } from '../../context/sl/sl-search.service.js';
|
||||
import { SqliteSlSourcesIndex } from '../../context/sl/sqlite-sl-sources-index.js';
|
||||
import { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js';
|
||||
import { SqliteKnowledgeIndex } from '../../context/wiki/sqlite-knowledge-index.js';
|
||||
import type { ReindexOptions, ReindexScopeResult, ReindexSummary, ReindexWorkResult } from './types.js';
|
||||
|
||||
type DiscoveredScope =
|
||||
|
|
@ -41,7 +45,7 @@ async function childDirectories(path: string): Promise<string[]> {
|
|||
}
|
||||
}
|
||||
|
||||
export async function discoverReindexScopes(project: KtxLocalProject): Promise<DiscoveredScope[]> {
|
||||
async function discoverReindexScopes(project: KtxLocalProject): Promise<DiscoveredScope[]> {
|
||||
const scopes: DiscoveredScope[] = [];
|
||||
if (await directoryExists(join(project.projectDir, 'wiki/global'))) {
|
||||
scopes.push({ kind: 'wiki', scope: 'GLOBAL', scopeId: null, label: 'global' });
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KtxEmbeddingPort } from '../core/index.js';
|
||||
import type { KtxEmbeddingPort } from '../../context/core/embedding.js';
|
||||
|
||||
export interface ReindexOptions {
|
||||
force: boolean;
|
||||
|
|
|
|||
|
|
@ -1,128 +0,0 @@
|
|||
export * from './agent/index.js';
|
||||
export * from './core/index.js';
|
||||
export * from './daemon/index.js';
|
||||
export * from './ingest/index.js';
|
||||
export * from './index-sync/index.js';
|
||||
export * from './llm/index.js';
|
||||
export type {
|
||||
CaptureSession,
|
||||
CaptureSignals,
|
||||
MemoryAgentInput,
|
||||
MemoryAgentResult,
|
||||
MemoryAgentServiceDeps,
|
||||
MemoryAgentSettings,
|
||||
MemoryAgentSourceType,
|
||||
MemoryCommitMessagePort,
|
||||
MemoryConnectionPort,
|
||||
MemoryFileStorePort,
|
||||
MemoryKnowledgeSlRefsPort,
|
||||
MemoryLockPort,
|
||||
MemorySlSourceReconcilerPort,
|
||||
MemoryTelemetryPort,
|
||||
MemoryToolSetLike,
|
||||
MemoryToolsetFactoryPort,
|
||||
} from './memory/index.js';
|
||||
export * from './project/index.js';
|
||||
export * from './prompts/index.js';
|
||||
export * from './search/index.js';
|
||||
export * from './sql-analysis/index.js';
|
||||
export type {
|
||||
KtxColumnAnalysisResult,
|
||||
KtxColumnDescriptionPromptInput,
|
||||
KtxColumnEmbeddingForeignKeys,
|
||||
KtxColumnEmbeddingTextInput,
|
||||
KtxColumnSampleInput,
|
||||
KtxColumnSampleResult,
|
||||
KtxColumnSampleUpdate,
|
||||
KtxColumnStatsInput,
|
||||
KtxColumnStatsResult,
|
||||
KtxConnectionDriver,
|
||||
KtxConnectorCapabilities,
|
||||
KtxCredentialEnvelope,
|
||||
KtxCredentialEnvReference,
|
||||
KtxCredentialFileReference,
|
||||
KtxDataDictionaryColumnState,
|
||||
KtxDataDictionarySampleDecision,
|
||||
KtxDataDictionarySettings,
|
||||
KtxDataDictionarySkipReason,
|
||||
KtxDataSourceDescriptionPromptInput,
|
||||
KtxDescriptionCachePort,
|
||||
KtxDescriptionColumn,
|
||||
KtxDescriptionColumnTable,
|
||||
KtxDescriptionGenerationSettings,
|
||||
KtxDescriptionGeneratorOptions,
|
||||
KtxDescriptionSource,
|
||||
KtxDescriptionTableInput,
|
||||
KtxDescriptionUpdate,
|
||||
KtxEmbeddingPort as KtxScanEmbeddingPort,
|
||||
KtxEmbeddingUpdate,
|
||||
KtxEnrichedColumn,
|
||||
KtxEnrichedRelationship,
|
||||
KtxEnrichedSchema,
|
||||
KtxEnrichedTable,
|
||||
KtxGenerateColumnDescriptionsInput,
|
||||
KtxGenerateDataSourceDescriptionInput,
|
||||
KtxGenerateTableDescriptionInput,
|
||||
KtxOptionalConnectorCapabilities,
|
||||
KtxProgressPort,
|
||||
KtxQueryResult as KtxScanQueryResult,
|
||||
KtxReadOnlyQueryInput,
|
||||
KtxRelationshipEndpoint,
|
||||
KtxRelationshipSource,
|
||||
KtxRelationshipType,
|
||||
KtxRelationshipUpdate,
|
||||
KtxResolvedCredentialEnvelope,
|
||||
KtxScanArtifactPaths,
|
||||
KtxScanConnector,
|
||||
KtxScanContext,
|
||||
KtxScanDiffSummary,
|
||||
KtxScanEnrichmentSummary,
|
||||
KtxScanInput,
|
||||
KtxScanLoggerPort,
|
||||
KtxScanMetadataStore,
|
||||
KtxScanMode,
|
||||
KtxScanRelationshipSummary,
|
||||
KtxScanReport,
|
||||
KtxScanTrigger,
|
||||
KtxScanWarning,
|
||||
KtxScanWarningCode,
|
||||
KtxSchemaColumn,
|
||||
KtxSchemaDimensionType,
|
||||
KtxSchemaForeignKey,
|
||||
KtxSchemaScope,
|
||||
KtxSchemaSnapshot,
|
||||
KtxSchemaTable,
|
||||
KtxSchemaTableKind,
|
||||
KtxSkippedRelationship,
|
||||
KtxStructuralSyncPlan,
|
||||
KtxStructuralSyncStats,
|
||||
KtxTableDescriptionPromptInput,
|
||||
KtxTableRef,
|
||||
KtxTableSampleInput,
|
||||
KtxTableSampleResult,
|
||||
KtxColumnTypeMapping,
|
||||
} from './scan/index.js';
|
||||
export {
|
||||
buildKtxColumnDescriptionPrompt,
|
||||
buildKtxColumnEmbeddingText,
|
||||
buildKtxDataSourceDescriptionPrompt,
|
||||
buildKtxTableDescriptionPrompt,
|
||||
createKtxConnectorCapabilities,
|
||||
defaultKtxDataDictionarySettings,
|
||||
inferKtxDimensionType,
|
||||
isKtxDataDictionaryCandidate,
|
||||
ktxColumnTypeMappingFromNative,
|
||||
KtxDescriptionGenerator,
|
||||
normalizeKtxNativeType,
|
||||
REDACTED_KTX_CREDENTIAL_VALUE,
|
||||
redactKtxCredentialEnvelope,
|
||||
redactKtxCredentialValue,
|
||||
redactKtxScanMetadata,
|
||||
redactKtxScanReport,
|
||||
redactKtxScanWarning,
|
||||
shouldKtxSampleColumnForDictionary,
|
||||
} from './scan/index.js';
|
||||
export * from './skills/index.js';
|
||||
export * from './sl/index.js';
|
||||
export * from './tools/index.js';
|
||||
export * from './wiki/index.js';
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import type { MemoryAction } from '../memory/index.js';
|
||||
import type { MemoryAction } from '../../context/memory/types.js';
|
||||
|
||||
export function actionTargetConnectionId(action: MemoryAction, runConnectionId: string): string {
|
||||
return action.target === 'sl' ? (action.targetConnectionId ?? runConnectionId) : runConnectionId;
|
||||
|
|
|
|||
|
|
@ -1,75 +0,0 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type { DbtParsedTable } from './parse-schema.js';
|
||||
import { findMatchingKtxTable, matchDbtTables, type DbtHostTableLite } from './match-tables.js';
|
||||
|
||||
const hostTables: DbtHostTableLite[] = [
|
||||
{ id: '1', name: 'orders', catalog: 'warehouse', db: 'analytics', columns: [{ id: 'c1', name: 'id' }] },
|
||||
{ id: '2', name: 'orders', catalog: 'warehouse', db: 'staging', columns: [{ id: 'c2', name: 'id' }] },
|
||||
{ id: '3', name: 'customers', catalog: null, db: null, columns: [{ id: 'c3', name: 'id' }] },
|
||||
];
|
||||
|
||||
function table(input: Partial<DbtParsedTable>): DbtParsedTable {
|
||||
return {
|
||||
name: 'orders',
|
||||
description: null,
|
||||
database: null,
|
||||
schema: null,
|
||||
columns: [],
|
||||
resourceType: 'model',
|
||||
...input,
|
||||
};
|
||||
}
|
||||
|
||||
describe('dbt descriptions table matching', () => {
|
||||
it('uses schema plus name first and checks catalog when dbt database is present', () => {
|
||||
expect(
|
||||
findMatchingKtxTable(table({ database: 'warehouse', schema: 'analytics' }), hostTables, null)?.id,
|
||||
).toBe('1');
|
||||
});
|
||||
|
||||
it('does not fall back to name-only for source tables', () => {
|
||||
expect(findMatchingKtxTable(table({ resourceType: 'source' }), hostTables, null)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('uses targetSchema for models and name-only only when unique', () => {
|
||||
expect(findMatchingKtxTable(table({ resourceType: 'model' }), hostTables, 'staging')?.id).toBe('2');
|
||||
expect(findMatchingKtxTable(table({ name: 'customers', resourceType: 'model' }), hostTables, null)?.id).toBe(
|
||||
'3',
|
||||
);
|
||||
expect(findMatchingKtxTable(table({ resourceType: 'model' }), hostTables, null)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('summarizes matched columns and descriptions', () => {
|
||||
const matches = matchDbtTables(
|
||||
[
|
||||
table({
|
||||
name: 'customers',
|
||||
description: 'Customers',
|
||||
columns: [
|
||||
{ name: 'id', description: 'Primary key', dataType: null },
|
||||
{ name: 'missing', description: 'Missing', dataType: null },
|
||||
],
|
||||
}),
|
||||
],
|
||||
hostTables,
|
||||
null,
|
||||
);
|
||||
|
||||
expect(matches).toEqual([
|
||||
{
|
||||
dbtTable: 'customers',
|
||||
dbtSchema: null,
|
||||
dbtDatabase: null,
|
||||
hostTableId: '3',
|
||||
hostTableName: 'customers',
|
||||
matched: true,
|
||||
tableDescriptionAction: 'import',
|
||||
tableDescriptionFound: true,
|
||||
columnsToImport: 1,
|
||||
columnsMatched: 1,
|
||||
columnsTotal: 2,
|
||||
columnDescriptionsFound: 1,
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,127 +0,0 @@
|
|||
import type { DbtParsedTable } from './parse-schema.js';
|
||||
|
||||
export interface DbtHostTableLite {
|
||||
id: string;
|
||||
name: string;
|
||||
catalog: string | null;
|
||||
db: string | null;
|
||||
columns: Array<{ id: string; name: string }>;
|
||||
}
|
||||
|
||||
export interface DbtTableMatch {
|
||||
dbtTable: string;
|
||||
dbtSchema: string | null;
|
||||
dbtDatabase: string | null;
|
||||
hostTableId: string | null;
|
||||
hostTableName: string | null;
|
||||
matched: boolean;
|
||||
tableDescriptionAction: 'skip' | 'import';
|
||||
tableDescriptionFound: boolean;
|
||||
columnsToImport: number;
|
||||
columnsMatched: number;
|
||||
columnsTotal: number;
|
||||
columnDescriptionsFound: number;
|
||||
}
|
||||
|
||||
export function matchDbtTables(
|
||||
dbtTables: DbtParsedTable[],
|
||||
hostTables: DbtHostTableLite[],
|
||||
targetSchema?: string | null,
|
||||
): DbtTableMatch[] {
|
||||
return dbtTables.map((dbtTable) => {
|
||||
const hostTable = findMatchingKtxTable(dbtTable, hostTables, targetSchema);
|
||||
|
||||
if (!hostTable) {
|
||||
return {
|
||||
dbtTable: dbtTable.name,
|
||||
dbtSchema: dbtTable.schema,
|
||||
dbtDatabase: dbtTable.database,
|
||||
hostTableId: null,
|
||||
hostTableName: null,
|
||||
matched: false,
|
||||
tableDescriptionAction: 'skip',
|
||||
tableDescriptionFound: Boolean(dbtTable.description),
|
||||
columnsToImport: 0,
|
||||
columnsMatched: 0,
|
||||
columnsTotal: dbtTable.columns.length,
|
||||
columnDescriptionsFound: dbtTable.columns.filter((column) => Boolean(column.description)).length,
|
||||
};
|
||||
}
|
||||
|
||||
const analysis = analyzeColumns(dbtTable, hostTable);
|
||||
return {
|
||||
dbtTable: dbtTable.name,
|
||||
dbtSchema: dbtTable.schema,
|
||||
dbtDatabase: dbtTable.database,
|
||||
hostTableId: hostTable.id,
|
||||
hostTableName: hostTable.name,
|
||||
matched: true,
|
||||
tableDescriptionAction: dbtTable.description ? 'import' : 'skip',
|
||||
tableDescriptionFound: Boolean(dbtTable.description),
|
||||
...analysis,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function findMatchingKtxTable(
|
||||
dbtTable: DbtParsedTable,
|
||||
hostTables: DbtHostTableLite[],
|
||||
targetSchema?: string | null,
|
||||
): DbtHostTableLite | undefined {
|
||||
const dbtName = dbtTable.name.toLowerCase();
|
||||
const effectiveSchema = dbtTable.schema ?? targetSchema ?? null;
|
||||
|
||||
if (effectiveSchema) {
|
||||
const strictMatch = hostTables.find((table) => {
|
||||
const nameMatches = table.name.toLowerCase() === dbtName;
|
||||
const schemaMatches = table.db?.toLowerCase() === effectiveSchema.toLowerCase();
|
||||
if (!nameMatches || !schemaMatches) {
|
||||
return false;
|
||||
}
|
||||
if (dbtTable.database && table.catalog) {
|
||||
return table.catalog.toLowerCase() === dbtTable.database.toLowerCase();
|
||||
}
|
||||
return true;
|
||||
});
|
||||
if (strictMatch) {
|
||||
return strictMatch;
|
||||
}
|
||||
}
|
||||
|
||||
if (dbtTable.resourceType === 'source') {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const nameMatches = hostTables.filter((table) => table.name.toLowerCase() === dbtName);
|
||||
return nameMatches.length === 1 ? nameMatches[0] : undefined;
|
||||
}
|
||||
|
||||
function analyzeColumns(
|
||||
dbtTable: DbtParsedTable,
|
||||
hostTable: DbtHostTableLite,
|
||||
): Pick<DbtTableMatch, 'columnsToImport' | 'columnsMatched' | 'columnsTotal' | 'columnDescriptionsFound'> {
|
||||
let columnsToImport = 0;
|
||||
let columnsMatched = 0;
|
||||
let columnDescriptionsFound = 0;
|
||||
|
||||
for (const dbtColumn of dbtTable.columns) {
|
||||
const hostColumn = hostTable.columns.find(
|
||||
(column) => column.name.toLowerCase() === dbtColumn.name.toLowerCase(),
|
||||
);
|
||||
if (!hostColumn) {
|
||||
continue;
|
||||
}
|
||||
columnsMatched++;
|
||||
if (dbtColumn.description) {
|
||||
columnDescriptionsFound++;
|
||||
columnsToImport++;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
columnsToImport,
|
||||
columnsMatched,
|
||||
columnsTotal: dbtTable.columns.length,
|
||||
columnDescriptionsFound,
|
||||
};
|
||||
}
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type { ParsedSemanticModel } from '../metricflow/deep-parse.js';
|
||||
import { mergeSemanticModelTables } from './merge-semantic-model-tables.js';
|
||||
import type { DbtSchemaParseResult } from './parse-schema.js';
|
||||
|
||||
const semanticModel: ParsedSemanticModel = {
|
||||
name: 'orders_semantic',
|
||||
description: 'Order facts',
|
||||
modelRef: 'fct_orders',
|
||||
dimensions: [
|
||||
{ name: 'status', column: 'status', type: 'categorical', description: 'Order status' },
|
||||
{ name: 'ordered_at', column: 'ordered_at', type: 'time' },
|
||||
],
|
||||
measures: [],
|
||||
entities: [],
|
||||
defaultTimeDimension: null,
|
||||
};
|
||||
|
||||
describe('mergeSemanticModelTables', () => {
|
||||
it('adds missing MetricFlow model refs as dbt model tables', () => {
|
||||
const input: DbtSchemaParseResult = { projectName: null, dbtVersion: null, tables: [], relationships: [] };
|
||||
|
||||
expect(mergeSemanticModelTables(input, [semanticModel])).toEqual({
|
||||
projectName: null,
|
||||
dbtVersion: null,
|
||||
relationships: [],
|
||||
tables: [
|
||||
{
|
||||
name: 'fct_orders',
|
||||
description: 'Order facts',
|
||||
database: null,
|
||||
schema: null,
|
||||
resourceType: 'model',
|
||||
columns: [
|
||||
{ name: 'status', description: 'Order status', dataType: null },
|
||||
{ name: 'ordered_at', description: null, dataType: 'TIMESTAMP' },
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('does not add a duplicate table when schema parsing already found the model ref', () => {
|
||||
const input: DbtSchemaParseResult = {
|
||||
projectName: null,
|
||||
dbtVersion: null,
|
||||
relationships: [],
|
||||
tables: [
|
||||
{
|
||||
name: 'FCT_ORDERS',
|
||||
description: 'Existing',
|
||||
database: null,
|
||||
schema: null,
|
||||
resourceType: 'model',
|
||||
columns: [],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
expect(mergeSemanticModelTables(input, [semanticModel]).tables).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
import type { ParsedSemanticModel } from '../metricflow/deep-parse.js';
|
||||
import type { DbtSchemaParseResult } from './parse-schema.js';
|
||||
|
||||
export function mergeSemanticModelTables(
|
||||
parseResult: DbtSchemaParseResult,
|
||||
semanticModels: ParsedSemanticModel[],
|
||||
): DbtSchemaParseResult {
|
||||
const merged: DbtSchemaParseResult = {
|
||||
...parseResult,
|
||||
tables: [...parseResult.tables],
|
||||
relationships: [...parseResult.relationships],
|
||||
};
|
||||
const existingTableNames = new Set(merged.tables.map((table) => table.name.toLowerCase()));
|
||||
|
||||
for (const model of semanticModels) {
|
||||
const tableName = model.modelRef;
|
||||
if (existingTableNames.has(tableName.toLowerCase())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
merged.tables.push({
|
||||
name: tableName,
|
||||
description: model.description,
|
||||
database: null,
|
||||
schema: null,
|
||||
columns: model.dimensions.map((dimension) => ({
|
||||
name: dimension.column,
|
||||
description: dimension.description ?? null,
|
||||
dataType: dimension.type === 'time' ? 'TIMESTAMP' : null,
|
||||
})),
|
||||
resourceType: 'model',
|
||||
});
|
||||
existingTableNames.add(tableName.toLowerCase());
|
||||
}
|
||||
|
||||
return merged;
|
||||
}
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { parse as parseYaml } from 'yaml';
|
||||
import { type KtxLogger, noopLogger } from '../../../core/index.js';
|
||||
import { type KtxLogger, noopLogger } from '../../../../context/core/config.js';
|
||||
import { resolveJinjaVariables } from '../../dbt-shared/project-vars.js';
|
||||
|
||||
export interface DbtParsedColumn {
|
||||
interface DbtParsedColumn {
|
||||
name: string;
|
||||
description: string | null;
|
||||
dataType: string | null;
|
||||
|
|
@ -12,20 +12,20 @@ export interface DbtParsedColumn {
|
|||
enumValuesDbt?: string[];
|
||||
}
|
||||
|
||||
export interface DbtDataTestRef {
|
||||
interface DbtDataTestRef {
|
||||
name: string;
|
||||
package: string;
|
||||
kwargs?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface DbtColumnConstraints {
|
||||
interface DbtColumnConstraints {
|
||||
dbt: {
|
||||
not_null?: boolean;
|
||||
unique?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
export interface DbtParsedRelationship {
|
||||
interface DbtParsedRelationship {
|
||||
fromTable: string;
|
||||
fromColumn: string;
|
||||
toTable: string;
|
||||
|
|
@ -35,7 +35,7 @@ export interface DbtParsedRelationship {
|
|||
description?: string;
|
||||
}
|
||||
|
||||
export interface DbtParsedTable {
|
||||
interface DbtParsedTable {
|
||||
name: string;
|
||||
description: string | null;
|
||||
database: string | null;
|
||||
|
|
@ -126,6 +126,7 @@ type DbtSchemaDataTest =
|
|||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
/** @internal */
|
||||
export function parseDbtSchemaFile(content: string, options: ParseDbtSchemaOptions = {}): DbtSchemaParseResult {
|
||||
return new DbtSchemaParser(options.logger ?? noopLogger).parseFile(content, options);
|
||||
}
|
||||
|
|
@ -138,13 +139,6 @@ export function parseDbtSchemaFiles(
|
|||
return new DbtSchemaParser(options.logger ?? noopLogger).parseFiles(files, variables, options.projectName ?? null);
|
||||
}
|
||||
|
||||
export function computeDbtSchemaHash(files: DbtSchemaFile[]): string {
|
||||
const combined = [...files]
|
||||
.sort((a, b) => a.path.localeCompare(b.path))
|
||||
.map((file) => `${file.path}:${file.content}`)
|
||||
.join('\n');
|
||||
return createHash('sha256').update(combined).digest('hex').substring(0, 16);
|
||||
}
|
||||
|
||||
class DbtSchemaParser {
|
||||
constructor(private readonly logger: KtxLogger) {}
|
||||
|
|
|
|||
|
|
@ -1,102 +0,0 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type { DbtSchemaParseResult } from './parse-schema.js';
|
||||
import { toDescriptionUpdates } from './to-description-updates.js';
|
||||
import type { DbtHostTableLite } from './match-tables.js';
|
||||
|
||||
const hostTables: DbtHostTableLite[] = [
|
||||
{
|
||||
id: '1',
|
||||
name: 'orders',
|
||||
catalog: 'warehouse',
|
||||
db: 'analytics',
|
||||
columns: [
|
||||
{ id: 'c1', name: 'id' },
|
||||
{ id: 'c2', name: 'amount' },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
function parseResult(description: string | null, columnDescription: string | null): DbtSchemaParseResult {
|
||||
return {
|
||||
projectName: null,
|
||||
dbtVersion: null,
|
||||
relationships: [],
|
||||
tables: [
|
||||
{
|
||||
name: 'orders',
|
||||
description,
|
||||
database: 'warehouse',
|
||||
schema: 'analytics',
|
||||
resourceType: 'model',
|
||||
columns: [
|
||||
{ name: 'id', description: columnDescription, dataType: null },
|
||||
{ name: 'missing', description: 'not imported', dataType: null },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe('dbt descriptions update payloads', () => {
|
||||
it('emits dbt writes and matching ai invalidations when descriptions exist', () => {
|
||||
expect(
|
||||
toDescriptionUpdates({
|
||||
connectionId: 'conn-1',
|
||||
parseResult: parseResult('Orders table', 'Primary key'),
|
||||
hostTables,
|
||||
targetSchema: null,
|
||||
}),
|
||||
).toEqual({
|
||||
dbt: [
|
||||
{
|
||||
connectionId: 'conn-1',
|
||||
table: { catalog: 'warehouse', db: 'analytics', name: 'orders' },
|
||||
source: 'dbt',
|
||||
tableDescription: 'Orders table',
|
||||
columnDescriptions: { id: 'Primary key' },
|
||||
},
|
||||
],
|
||||
aiInvalidations: [
|
||||
{
|
||||
connectionId: 'conn-1',
|
||||
table: { catalog: 'warehouse', db: 'analytics', name: 'orders' },
|
||||
source: 'ai',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('does not emit spurious dbt writes or ai invalidations when no descriptions exist', () => {
|
||||
expect(
|
||||
toDescriptionUpdates({
|
||||
connectionId: 'conn-1',
|
||||
parseResult: parseResult(null, null),
|
||||
hostTables,
|
||||
targetSchema: null,
|
||||
}),
|
||||
).toEqual({ dbt: [], aiInvalidations: [] });
|
||||
});
|
||||
|
||||
it('emits ai invalidation without a dbt description write when only structural metadata exists', () => {
|
||||
const result = parseResult(null, null);
|
||||
result.tables[0]!.tagsDbt = ['finance'];
|
||||
|
||||
expect(
|
||||
toDescriptionUpdates({
|
||||
connectionId: 'conn-1',
|
||||
parseResult: result,
|
||||
hostTables,
|
||||
targetSchema: null,
|
||||
}),
|
||||
).toEqual({
|
||||
dbt: [],
|
||||
aiInvalidations: [
|
||||
{
|
||||
connectionId: 'conn-1',
|
||||
table: { catalog: 'warehouse', db: 'analytics', name: 'orders' },
|
||||
source: 'ai',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
import type { KtxDescriptionUpdate } from '../../../scan/enrichment-types.js';
|
||||
import { findMatchingKtxTable, type DbtHostTableLite } from './match-tables.js';
|
||||
import type { DbtSchemaParseResult } from './parse-schema.js';
|
||||
|
||||
export interface DbtDescriptionUpdates {
|
||||
dbt: KtxDescriptionUpdate[];
|
||||
aiInvalidations: KtxDescriptionUpdate[];
|
||||
}
|
||||
|
||||
export function toDescriptionUpdates(input: {
|
||||
connectionId: string;
|
||||
parseResult: DbtSchemaParseResult;
|
||||
hostTables: DbtHostTableLite[];
|
||||
targetSchema: string | null;
|
||||
}): DbtDescriptionUpdates {
|
||||
const dbt: KtxDescriptionUpdate[] = [];
|
||||
const aiInvalidations: KtxDescriptionUpdate[] = [];
|
||||
|
||||
for (const dbtTable of input.parseResult.tables) {
|
||||
const hostTable = findMatchingKtxTable(dbtTable, input.hostTables, input.targetSchema);
|
||||
if (!hostTable) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const tableDescription = dbtTable.description ?? undefined;
|
||||
const columnDescriptions: Record<string, string | null> = {};
|
||||
|
||||
for (const dbtColumn of dbtTable.columns) {
|
||||
if (!dbtColumn.description) {
|
||||
continue;
|
||||
}
|
||||
const hostColumn = hostTable.columns.find(
|
||||
(column) => column.name.toLowerCase() === dbtColumn.name.toLowerCase(),
|
||||
);
|
||||
if (hostColumn) {
|
||||
columnDescriptions[hostColumn.name] = dbtColumn.description;
|
||||
}
|
||||
}
|
||||
|
||||
const hasColumnDescriptions = Object.keys(columnDescriptions).length > 0;
|
||||
const hasDescriptionChange = tableDescription !== undefined || hasColumnDescriptions;
|
||||
const hasMetadataChange =
|
||||
!!dbtTable.tagsDbt?.length ||
|
||||
dbtTable.freshnessDbt !== undefined ||
|
||||
dbtTable.columns.some(
|
||||
(column) => column.constraints !== undefined || !!column.enumValuesDbt?.length || !!column.dataTests?.length,
|
||||
);
|
||||
if (!hasDescriptionChange && !hasMetadataChange) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const tableRef = { catalog: hostTable.catalog, db: hostTable.db, name: hostTable.name };
|
||||
if (hasDescriptionChange) {
|
||||
dbt.push({
|
||||
connectionId: input.connectionId,
|
||||
table: tableRef,
|
||||
source: 'dbt',
|
||||
...(tableDescription !== undefined ? { tableDescription } : {}),
|
||||
...(hasColumnDescriptions ? { columnDescriptions } : {}),
|
||||
});
|
||||
}
|
||||
aiInvalidations.push({
|
||||
connectionId: input.connectionId,
|
||||
table: tableRef,
|
||||
source: 'ai',
|
||||
});
|
||||
}
|
||||
|
||||
return { dbt, aiInvalidations };
|
||||
}
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { toMetadataUpdates } from './to-metadata-updates.js';
|
||||
|
||||
describe('toMetadataUpdates', () => {
|
||||
it('emits source-keyed dbt metadata updates for matched tables and columns', () => {
|
||||
const updates = toMetadataUpdates({
|
||||
connectionId: 'conn_1',
|
||||
targetSchema: 'analytics',
|
||||
hostTables: [
|
||||
{
|
||||
id: 'orders-id',
|
||||
name: 'orders',
|
||||
catalog: 'warehouse',
|
||||
db: 'analytics',
|
||||
columns: [
|
||||
{ id: 'status-id', name: 'status' },
|
||||
{ id: 'created-id', name: 'created_at' },
|
||||
],
|
||||
},
|
||||
],
|
||||
parseResult: {
|
||||
projectName: null,
|
||||
dbtVersion: null,
|
||||
relationships: [],
|
||||
tables: [
|
||||
{
|
||||
name: 'orders',
|
||||
description: null,
|
||||
database: 'warehouse',
|
||||
schema: 'analytics',
|
||||
resourceType: 'model',
|
||||
tagsDbt: ['finance'],
|
||||
freshnessDbt: { loadedAtField: 'created_at' },
|
||||
columns: [
|
||||
{
|
||||
name: 'status',
|
||||
description: null,
|
||||
dataType: null,
|
||||
enumValuesDbt: ['placed', 'shipped'],
|
||||
constraints: { dbt: { not_null: true } },
|
||||
dataTests: [{ name: 'accepted_values', package: 'dbt', kwargs: { values: ['placed', 'shipped'] } }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(updates).toEqual([
|
||||
{
|
||||
connectionId: 'conn_1',
|
||||
table: { catalog: 'warehouse', db: 'analytics', name: 'orders' },
|
||||
source: 'dbt',
|
||||
tableFields: {
|
||||
tags: ['finance'],
|
||||
freshness: { loaded_at_field: 'created_at' },
|
||||
},
|
||||
columnFields: {
|
||||
status: {
|
||||
constraints: { not_null: true },
|
||||
enum_values: ['placed', 'shipped'],
|
||||
tests: [
|
||||
{ name: 'accepted_values', package: 'dbt', kwargs: { values: ['placed', 'shipped'] } },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
import type { KtxMetadataUpdate } from '../../../scan/enrichment-types.js';
|
||||
import { findMatchingKtxTable, type DbtHostTableLite } from './match-tables.js';
|
||||
import type { DbtSchemaParseResult } from './parse-schema.js';
|
||||
|
||||
export function toMetadataUpdates(input: {
|
||||
connectionId: string;
|
||||
parseResult: DbtSchemaParseResult;
|
||||
hostTables: DbtHostTableLite[];
|
||||
targetSchema: string | null;
|
||||
}): KtxMetadataUpdate[] {
|
||||
const updates: KtxMetadataUpdate[] = [];
|
||||
|
||||
for (const dbtTable of input.parseResult.tables) {
|
||||
const hostTable = findMatchingKtxTable(dbtTable, input.hostTables, input.targetSchema);
|
||||
if (!hostTable) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const tableFields: Record<string, unknown> = {};
|
||||
if (dbtTable.tagsDbt?.length) {
|
||||
tableFields.tags = dbtTable.tagsDbt;
|
||||
}
|
||||
if (dbtTable.freshnessDbt) {
|
||||
tableFields.freshness = {
|
||||
...(dbtTable.freshnessDbt.raw !== undefined ? { raw: dbtTable.freshnessDbt.raw } : {}),
|
||||
...(dbtTable.freshnessDbt.loadedAtField !== undefined
|
||||
? { loaded_at_field: dbtTable.freshnessDbt.loadedAtField }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
const columnFields: Record<string, Record<string, unknown>> = {};
|
||||
for (const dbtColumn of dbtTable.columns) {
|
||||
const hostColumn = hostTable.columns.find(
|
||||
(column) => column.name.toLowerCase() === dbtColumn.name.toLowerCase(),
|
||||
);
|
||||
if (!hostColumn) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fields: Record<string, unknown> = {};
|
||||
if (dbtColumn.constraints) {
|
||||
fields.constraints = dbtColumn.constraints.dbt;
|
||||
}
|
||||
if (dbtColumn.enumValuesDbt?.length) {
|
||||
fields.enum_values = dbtColumn.enumValuesDbt;
|
||||
}
|
||||
if (dbtColumn.dataTests?.length) {
|
||||
fields.tests = dbtColumn.dataTests.map((test) => ({
|
||||
name: test.name,
|
||||
package: test.package,
|
||||
...(test.kwargs ? { kwargs: test.kwargs } : {}),
|
||||
}));
|
||||
}
|
||||
if (Object.keys(fields).length > 0) {
|
||||
columnFields[hostColumn.name] = fields;
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(tableFields).length === 0 && Object.keys(columnFields).length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
updates.push({
|
||||
connectionId: input.connectionId,
|
||||
table: { catalog: hostTable.catalog, db: hostTable.db, name: hostTable.name },
|
||||
source: 'dbt',
|
||||
...(Object.keys(tableFields).length > 0 ? { tableFields } : {}),
|
||||
...(Object.keys(columnFields).length > 0 ? { columnFields } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
return updates;
|
||||
}
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type { DbtHostTableLite } from './match-tables.js';
|
||||
import type { DbtSchemaParseResult } from './parse-schema.js';
|
||||
import { toRelationshipUpdates } from './to-relationship-updates.js';
|
||||
|
||||
const DBT_SYSTEM_EMAIL = ['system@kae', 'lio.dev'].join('');
|
||||
|
||||
const hostTables: DbtHostTableLite[] = [
|
||||
{
|
||||
id: '1',
|
||||
name: 'orders',
|
||||
catalog: 'warehouse',
|
||||
db: 'analytics',
|
||||
columns: [{ id: 'c1', name: 'customer_id' }],
|
||||
},
|
||||
{
|
||||
id: '2',
|
||||
name: 'customers',
|
||||
catalog: 'warehouse',
|
||||
db: 'staging',
|
||||
columns: [{ id: 'c2', name: 'id' }],
|
||||
},
|
||||
];
|
||||
|
||||
const parseResult: DbtSchemaParseResult = {
|
||||
projectName: null,
|
||||
dbtVersion: null,
|
||||
tables: [],
|
||||
relationships: [
|
||||
{
|
||||
fromTable: 'orders',
|
||||
fromColumn: 'customer_id',
|
||||
toTable: 'customers',
|
||||
toColumn: 'id',
|
||||
fromSchema: 'analytics',
|
||||
toSchema: 'analytics',
|
||||
description: 'schema intentionally differs from the host customers table',
|
||||
},
|
||||
{ fromTable: 'orders', fromColumn: 'missing', toTable: 'customers', toColumn: 'id' },
|
||||
{ fromTable: 'orders', fromColumn: 'customer_id', toTable: 'missing_table', toColumn: 'id' },
|
||||
],
|
||||
};
|
||||
|
||||
describe('dbt relationship update payloads', () => {
|
||||
it('validates relationships using the current name-only matching behavior and dbt provenance', () => {
|
||||
expect(toRelationshipUpdates({ connectionId: 'conn-1', parseResult, hostTables })).toEqual({
|
||||
joins: [
|
||||
{
|
||||
connectionId: 'conn-1',
|
||||
fromTable: 'orders',
|
||||
fromColumns: ['customer_id'],
|
||||
toTable: 'customers',
|
||||
toColumns: ['id'],
|
||||
relationship: 'many_to_one',
|
||||
author: 'dbt',
|
||||
authorEmail: DBT_SYSTEM_EMAIL,
|
||||
},
|
||||
],
|
||||
skippedNoMatch: 2,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
import type { KtxJoinUpdate } from '../../../scan/enrichment-types.js';
|
||||
import type { DbtHostTableLite } from './match-tables.js';
|
||||
import type { DbtSchemaParseResult } from './parse-schema.js';
|
||||
|
||||
export interface DbtRelationshipUpdates {
|
||||
joins: KtxJoinUpdate[];
|
||||
skippedNoMatch: number;
|
||||
}
|
||||
|
||||
const DBT_SYSTEM_EMAIL = ['system@kae', 'lio.dev'].join('');
|
||||
|
||||
export function toRelationshipUpdates(input: {
|
||||
connectionId: string;
|
||||
parseResult: DbtSchemaParseResult;
|
||||
hostTables: DbtHostTableLite[];
|
||||
}): DbtRelationshipUpdates {
|
||||
const tablesByName = new Map<string, DbtHostTableLite>();
|
||||
for (const table of input.hostTables) {
|
||||
tablesByName.set(table.name.toLowerCase(), table);
|
||||
}
|
||||
|
||||
const joins: KtxJoinUpdate[] = [];
|
||||
let skippedNoMatch = 0;
|
||||
|
||||
for (const relationship of input.parseResult.relationships) {
|
||||
const fromTable = tablesByName.get(relationship.fromTable.toLowerCase());
|
||||
const toTable = tablesByName.get(relationship.toTable.toLowerCase());
|
||||
if (!fromTable || !toTable) {
|
||||
skippedNoMatch++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const fromColumn = fromTable.columns.find(
|
||||
(column) => column.name.toLowerCase() === relationship.fromColumn.toLowerCase(),
|
||||
);
|
||||
const toColumn = toTable.columns.find(
|
||||
(column) => column.name.toLowerCase() === relationship.toColumn.toLowerCase(),
|
||||
);
|
||||
if (!fromColumn || !toColumn) {
|
||||
skippedNoMatch++;
|
||||
continue;
|
||||
}
|
||||
|
||||
joins.push({
|
||||
connectionId: input.connectionId,
|
||||
fromTable: fromTable.name,
|
||||
fromColumns: [fromColumn.name],
|
||||
toTable: toTable.name,
|
||||
toColumns: [toColumn.name],
|
||||
relationship: 'many_to_one',
|
||||
author: 'dbt',
|
||||
authorEmail: DBT_SYSTEM_EMAIL,
|
||||
});
|
||||
}
|
||||
|
||||
return { joins, skippedNoMatch };
|
||||
}
|
||||
|
|
@ -1,410 +0,0 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { type DbtHostTableLite, matchDbtTables } from './dbt-descriptions/match-tables.js';
|
||||
import { mergeSemanticModelTables } from './dbt-descriptions/merge-semantic-model-tables.js';
|
||||
import { parseDbtSchemaFiles } from './dbt-descriptions/parse-schema.js';
|
||||
import { toDescriptionUpdates } from './dbt-descriptions/to-description-updates.js';
|
||||
import { toRelationshipUpdates } from './dbt-descriptions/to-relationship-updates.js';
|
||||
import { parseMetricflowFiles } from './metricflow/deep-parse.js';
|
||||
import { mapCrossModelMetricToSource, mapSemanticModelToSource } from './metricflow/semantic-models.js';
|
||||
|
||||
const DBT_SYSTEM_EMAIL = ['system@kae', 'lio.dev'].join('');
|
||||
|
||||
const metricflowYaml = `
|
||||
semantic_models:
|
||||
- name: orders_semantic
|
||||
description: MetricFlow order facts
|
||||
model: ref('fct_orders')
|
||||
defaults:
|
||||
agg_time_dimension: ordered_at
|
||||
entities:
|
||||
- name: customer
|
||||
type: foreign
|
||||
expr: customer_id
|
||||
description: Customer relationship
|
||||
dimensions:
|
||||
- name: status
|
||||
type: categorical
|
||||
expr: status
|
||||
description: Order status
|
||||
- name: ordered_at
|
||||
type: time
|
||||
expr: ordered_at
|
||||
measures:
|
||||
- name: total_revenue
|
||||
agg: sum
|
||||
expr: amount
|
||||
description: Revenue
|
||||
- name: customers_semantic
|
||||
description: Customer dimension
|
||||
model: ref('dim_customers')
|
||||
entities:
|
||||
- name: customer
|
||||
type: primary
|
||||
expr: id
|
||||
dimensions:
|
||||
- name: country
|
||||
type: categorical
|
||||
expr: country
|
||||
description: Customer country
|
||||
measures:
|
||||
- name: customer_count
|
||||
agg: count
|
||||
expr: id
|
||||
description: Customer count
|
||||
metrics:
|
||||
- name: total_revenue
|
||||
type: simple
|
||||
type_params:
|
||||
measure: total_revenue
|
||||
- name: customer_count
|
||||
type: simple
|
||||
type_params:
|
||||
measure: customer_count
|
||||
- name: revenue_per_customer
|
||||
description: Revenue per customer
|
||||
type: derived
|
||||
type_params:
|
||||
expr: total_revenue / NULLIF(customer_count, 0)
|
||||
metrics:
|
||||
- name: total_revenue
|
||||
alias: total_revenue
|
||||
- name: customer_count
|
||||
alias: customer_count
|
||||
`;
|
||||
|
||||
const schemaYaml = `
|
||||
version: 2
|
||||
sources:
|
||||
- name: raw
|
||||
database: warehouse
|
||||
schema: landing
|
||||
tables:
|
||||
- name: customers
|
||||
identifier: dim_customers
|
||||
description: Raw customer dimension
|
||||
columns:
|
||||
- name: id
|
||||
description: Customer primary key
|
||||
- name: country
|
||||
description: Country name
|
||||
models:
|
||||
- name: "{{ var('orders_model', 'fct_orders') }}"
|
||||
schema: "{{ var('mart_schema', 'analytics') }}"
|
||||
description: Modeled orders
|
||||
columns:
|
||||
- name: customer_id
|
||||
description: Linked customer id
|
||||
tests:
|
||||
- relationships:
|
||||
to: ref('dim_customers')
|
||||
field: id
|
||||
- name: status
|
||||
description: Order status
|
||||
- name: amount
|
||||
description: Gross amount
|
||||
`;
|
||||
|
||||
const hostTables: DbtHostTableLite[] = [
|
||||
{
|
||||
id: 'orders-table',
|
||||
name: 'fct_orders',
|
||||
catalog: 'warehouse',
|
||||
db: 'analytics',
|
||||
columns: [
|
||||
{ id: 'orders-customer-id', name: 'customer_id' },
|
||||
{ id: 'orders-status', name: 'status' },
|
||||
{ id: 'orders-amount', name: 'amount' },
|
||||
{ id: 'orders-ordered-at', name: 'ordered_at' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'customers-table',
|
||||
name: 'dim_customers',
|
||||
catalog: 'warehouse',
|
||||
db: 'landing',
|
||||
columns: [
|
||||
{ id: 'customers-id', name: 'id' },
|
||||
{ id: 'customers-country', name: 'country' },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
describe('dbt extraction golden parity fixture', () => {
|
||||
it('freezes the relocated MetricFlow and dbt-description contract together', () => {
|
||||
const metricflow = parseMetricflowFiles([{ path: 'semantic_models/orders.yml', content: metricflowYaml }]);
|
||||
|
||||
expect(metricflow).toEqual({
|
||||
semanticModels: [
|
||||
{
|
||||
name: 'orders_semantic',
|
||||
description: 'MetricFlow order facts',
|
||||
modelRef: 'fct_orders',
|
||||
dimensions: [
|
||||
{
|
||||
name: 'status',
|
||||
column: 'status',
|
||||
type: 'string',
|
||||
label: 'Status',
|
||||
description: 'Order status',
|
||||
},
|
||||
{
|
||||
name: 'ordered_at',
|
||||
column: 'ordered_at',
|
||||
type: 'time',
|
||||
label: 'Ordered At',
|
||||
description: undefined,
|
||||
},
|
||||
],
|
||||
measures: [
|
||||
{
|
||||
type: 'simple',
|
||||
name: 'total_revenue',
|
||||
column: 'amount',
|
||||
aggregation: 'sum',
|
||||
label: 'Total Revenue',
|
||||
description: 'Revenue',
|
||||
},
|
||||
],
|
||||
entities: [{ name: 'customer', type: 'foreign', expr: 'customer_id', description: 'Customer relationship' }],
|
||||
defaultTimeDimension: 'ordered_at',
|
||||
},
|
||||
{
|
||||
name: 'customers_semantic',
|
||||
description: 'Customer dimension',
|
||||
modelRef: 'dim_customers',
|
||||
dimensions: [
|
||||
{
|
||||
name: 'country',
|
||||
column: 'country',
|
||||
type: 'string',
|
||||
label: 'Country',
|
||||
description: 'Customer country',
|
||||
},
|
||||
],
|
||||
measures: [
|
||||
{
|
||||
type: 'simple',
|
||||
name: 'customer_count',
|
||||
column: 'id',
|
||||
aggregation: 'count',
|
||||
label: 'Customer Count',
|
||||
description: 'Customer count',
|
||||
},
|
||||
],
|
||||
entities: [{ name: 'customer', type: 'primary', expr: 'id' }],
|
||||
defaultTimeDimension: null,
|
||||
},
|
||||
],
|
||||
crossModelMetrics: [
|
||||
{
|
||||
name: 'revenue_per_customer',
|
||||
label: null,
|
||||
description: 'Revenue per customer',
|
||||
type: 'derived',
|
||||
expr: 'total_revenue / NULLIF(customer_count, 0)',
|
||||
dependsOn: [
|
||||
{ metricName: 'orders_semantic', alias: 'total_revenue' },
|
||||
{ metricName: 'customers_semantic', alias: 'customer_count' },
|
||||
],
|
||||
filter: null,
|
||||
},
|
||||
],
|
||||
relationships: [
|
||||
{
|
||||
fromTable: 'fct_orders',
|
||||
fromColumn: 'customer_id',
|
||||
toTable: 'dim_customers',
|
||||
toColumn: 'id',
|
||||
description: 'Customer relationship',
|
||||
},
|
||||
],
|
||||
warnings: [],
|
||||
});
|
||||
|
||||
expect(mapSemanticModelToSource(metricflow.semanticModels[0], 'analytics.fct_orders')).toEqual({
|
||||
name: 'fct-orders',
|
||||
table: 'analytics.fct_orders',
|
||||
grain: ['status', 'ordered_at'],
|
||||
columns: [
|
||||
{ name: 'status', type: 'string', description: 'Order status' },
|
||||
{ name: 'ordered_at', type: 'time' },
|
||||
],
|
||||
measures: [
|
||||
{
|
||||
name: 'total_revenue',
|
||||
expr: 'sum(amount)',
|
||||
description: 'Revenue',
|
||||
},
|
||||
],
|
||||
joins: [],
|
||||
descriptions: { dbt: 'MetricFlow order facts' },
|
||||
});
|
||||
|
||||
expect(mapCrossModelMetricToSource(metricflow.crossModelMetrics[0])).toEqual({
|
||||
name: 'revenue-per-customer',
|
||||
sql: 'total_revenue / NULLIF(customer_count, 0)',
|
||||
descriptions: { dbt: 'Revenue per customer' },
|
||||
grain: [],
|
||||
columns: [],
|
||||
measures: [
|
||||
{
|
||||
name: 'revenue_per_customer',
|
||||
expr: 'total_revenue / NULLIF(customer_count, 0)',
|
||||
description: 'Revenue per customer',
|
||||
},
|
||||
],
|
||||
joins: [],
|
||||
});
|
||||
|
||||
const schema = parseDbtSchemaFiles(
|
||||
[{ path: 'models/schema.yml', content: schemaYaml }],
|
||||
new Map([
|
||||
['orders_model', 'fct_orders'],
|
||||
['mart_schema', 'analytics'],
|
||||
]),
|
||||
);
|
||||
const merged = mergeSemanticModelTables(schema, metricflow.semanticModels);
|
||||
|
||||
expect(merged).toEqual({
|
||||
projectName: null,
|
||||
dbtVersion: null,
|
||||
tables: [
|
||||
{
|
||||
name: 'dim_customers',
|
||||
description: 'Raw customer dimension',
|
||||
database: 'warehouse',
|
||||
schema: 'landing',
|
||||
columns: [
|
||||
{ name: 'id', description: 'Customer primary key', dataType: null },
|
||||
{ name: 'country', description: 'Country name', dataType: null },
|
||||
],
|
||||
resourceType: 'source',
|
||||
},
|
||||
{
|
||||
name: 'fct_orders',
|
||||
description: 'Modeled orders',
|
||||
database: null,
|
||||
schema: 'analytics',
|
||||
columns: [
|
||||
{
|
||||
name: 'customer_id',
|
||||
description: 'Linked customer id',
|
||||
dataType: null,
|
||||
dataTests: [
|
||||
{
|
||||
name: 'relationships',
|
||||
package: 'dbt',
|
||||
kwargs: { to: "ref('dim_customers')", field: 'id' },
|
||||
},
|
||||
],
|
||||
},
|
||||
{ name: 'status', description: 'Order status', dataType: null },
|
||||
{ name: 'amount', description: 'Gross amount', dataType: null },
|
||||
],
|
||||
resourceType: 'model',
|
||||
},
|
||||
],
|
||||
relationships: [
|
||||
{
|
||||
fromTable: 'fct_orders',
|
||||
fromColumn: 'customer_id',
|
||||
toTable: 'dim_customers',
|
||||
toColumn: 'id',
|
||||
fromSchema: 'analytics',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(matchDbtTables(merged.tables, hostTables, 'analytics')).toEqual([
|
||||
{
|
||||
dbtTable: 'dim_customers',
|
||||
dbtSchema: 'landing',
|
||||
dbtDatabase: 'warehouse',
|
||||
hostTableId: 'customers-table',
|
||||
hostTableName: 'dim_customers',
|
||||
matched: true,
|
||||
tableDescriptionAction: 'import',
|
||||
tableDescriptionFound: true,
|
||||
columnsToImport: 2,
|
||||
columnsMatched: 2,
|
||||
columnsTotal: 2,
|
||||
columnDescriptionsFound: 2,
|
||||
},
|
||||
{
|
||||
dbtTable: 'fct_orders',
|
||||
dbtSchema: 'analytics',
|
||||
dbtDatabase: null,
|
||||
hostTableId: 'orders-table',
|
||||
hostTableName: 'fct_orders',
|
||||
matched: true,
|
||||
tableDescriptionAction: 'import',
|
||||
tableDescriptionFound: true,
|
||||
columnsToImport: 3,
|
||||
columnsMatched: 3,
|
||||
columnsTotal: 3,
|
||||
columnDescriptionsFound: 3,
|
||||
},
|
||||
]);
|
||||
|
||||
expect(
|
||||
toDescriptionUpdates({
|
||||
connectionId: 'warehouse-1',
|
||||
parseResult: merged,
|
||||
hostTables,
|
||||
targetSchema: 'analytics',
|
||||
}),
|
||||
).toEqual({
|
||||
dbt: [
|
||||
{
|
||||
connectionId: 'warehouse-1',
|
||||
table: { catalog: 'warehouse', db: 'landing', name: 'dim_customers' },
|
||||
source: 'dbt',
|
||||
tableDescription: 'Raw customer dimension',
|
||||
columnDescriptions: {
|
||||
id: 'Customer primary key',
|
||||
country: 'Country name',
|
||||
},
|
||||
},
|
||||
{
|
||||
connectionId: 'warehouse-1',
|
||||
table: { catalog: 'warehouse', db: 'analytics', name: 'fct_orders' },
|
||||
source: 'dbt',
|
||||
tableDescription: 'Modeled orders',
|
||||
columnDescriptions: {
|
||||
customer_id: 'Linked customer id',
|
||||
status: 'Order status',
|
||||
amount: 'Gross amount',
|
||||
},
|
||||
},
|
||||
],
|
||||
aiInvalidations: [
|
||||
{
|
||||
connectionId: 'warehouse-1',
|
||||
table: { catalog: 'warehouse', db: 'landing', name: 'dim_customers' },
|
||||
source: 'ai',
|
||||
},
|
||||
{
|
||||
connectionId: 'warehouse-1',
|
||||
table: { catalog: 'warehouse', db: 'analytics', name: 'fct_orders' },
|
||||
source: 'ai',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(toRelationshipUpdates({ connectionId: 'warehouse-1', parseResult: merged, hostTables })).toEqual({
|
||||
joins: [
|
||||
{
|
||||
connectionId: 'warehouse-1',
|
||||
fromTable: 'fct_orders',
|
||||
fromColumns: ['customer_id'],
|
||||
toTable: 'dim_customers',
|
||||
toColumns: ['id'],
|
||||
relationship: 'many_to_one',
|
||||
author: 'dbt',
|
||||
authorEmail: DBT_SYSTEM_EMAIL,
|
||||
},
|
||||
],
|
||||
skippedNoMatch: 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -9,6 +9,7 @@ function safeEvidenceSegment(value: string): string {
|
|||
return segment;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export const historicSqlTableUsageEvidenceSchema = z.object({
|
||||
kind: z.literal('table_usage'),
|
||||
connectionId: z.string().min(1),
|
||||
|
|
@ -16,15 +17,14 @@ export const historicSqlTableUsageEvidenceSchema = z.object({
|
|||
rawPath: z.string().min(1),
|
||||
usage: tableUsageOutputSchema,
|
||||
});
|
||||
export type HistoricSqlTableUsageEvidence = z.infer<typeof historicSqlTableUsageEvidenceSchema>;
|
||||
|
||||
/** @internal */
|
||||
export const historicSqlPatternEvidenceSchema = z.object({
|
||||
kind: z.literal('pattern'),
|
||||
connectionId: z.string().min(1),
|
||||
rawPath: z.string().min(1),
|
||||
pattern: patternOutputSchema,
|
||||
});
|
||||
export type HistoricSqlPatternEvidence = z.infer<typeof historicSqlPatternEvidenceSchema>;
|
||||
|
||||
export const historicSqlEvidenceEnvelopeSchema = z.discriminatedUnion('kind', [
|
||||
historicSqlTableUsageEvidenceSchema,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { mkdtemp } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import type { SqlAnalysisPort } from '../../../sql-analysis/index.js';
|
||||
import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js';
|
||||
import type { SourceAdapter } from '../../types.js';
|
||||
import { HistoricSqlSourceAdapter } from './historic-sql.adapter.js';
|
||||
import type { HistoricSqlReader } from './types.js';
|
||||
|
|
|
|||
|
|
@ -2,14 +2,9 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import YAML from 'yaml';
|
||||
import type { AgentRunnerPort, RunLoopParams } from '../../../llm/index.js';
|
||||
import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../../project/index.js';
|
||||
import {
|
||||
type SqlAnalysisBatchItem,
|
||||
type SqlAnalysisBatchResult,
|
||||
type SqlAnalysisDialect,
|
||||
type SqlAnalysisPort,
|
||||
} from '../../../sql-analysis/index.js';
|
||||
import type { AgentRunnerPort, RunLoopParams } from '../../../../context/llm/runtime-port.js';
|
||||
import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../../../context/project/project.js';
|
||||
import type { SqlAnalysisBatchItem, SqlAnalysisBatchResult, SqlAnalysisDialect, SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js';
|
||||
import { searchLocalSlSources } from '../../../sl/local-sl.js';
|
||||
import { searchLocalKnowledgePages } from '../../../wiki/local-knowledge.js';
|
||||
import { runLocalIngest } from '../../local-ingest.js';
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { access, mkdir, readdir, readFile, rename, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join, relative } from 'node:path';
|
||||
import YAML from 'yaml';
|
||||
import type { MemoryAction } from '../../../memory/index.js';
|
||||
import type { MemoryAction } from '../../../../context/memory/types.js';
|
||||
import { rawSourcesDirForSync } from '../../raw-sources-paths.js';
|
||||
import type { FinalizationOverrideReplay } from '../../types.js';
|
||||
import { mergeUsagePreservingExternal } from '../live-database/manifest.js';
|
||||
|
|
|
|||
|
|
@ -26,6 +26,6 @@ export const patternOutputSchema = z.object({
|
|||
slRefs: z.array(z.string()),
|
||||
constituentTemplateIds: z.array(z.string()),
|
||||
});
|
||||
export type PatternOutput = z.infer<typeof patternOutputSchema>;
|
||||
|
||||
/** @internal */
|
||||
export const patternsArraySchema = z.array(patternOutputSchema);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { mkdtemp, readFile, readdir } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { SqlAnalysisPort } from '../../../sql-analysis/index.js';
|
||||
import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js';
|
||||
import { stageHistoricSqlAggregatedSnapshot } from './stage-unified.js';
|
||||
import type { AggregatedTemplate, HistoricSqlReader } from './types.js';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join } from 'node:path';
|
||||
import type { SqlAnalysisPort } from '../../../sql-analysis/index.js';
|
||||
import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js';
|
||||
import {
|
||||
bucketDistinctUsers,
|
||||
bucketErrorRate,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { z } from 'zod';
|
||||
import type { SqlAnalysisPort } from '../../../sql-analysis/index.js';
|
||||
import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js';
|
||||
|
||||
export const HISTORIC_SQL_SOURCE_KEY = 'historic-sql' as const;
|
||||
|
||||
|
|
@ -115,9 +115,8 @@ export const stagedManifestSchema = z.object({
|
|||
probeWarnings: z.array(z.string()),
|
||||
staleArchiveAfterDays: z.number().int().positive().default(90),
|
||||
});
|
||||
export type StagedManifest = z.infer<typeof stagedManifestSchema>;
|
||||
|
||||
export interface HistoricSqlProbeResult {
|
||||
interface HistoricSqlProbeResult {
|
||||
warnings: string[];
|
||||
info?: string[];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,9 +8,9 @@ import type { KtxSchemaColumn, KtxSchemaForeignKey, KtxSchemaSnapshot, KtxSchema
|
|||
import { inferKtxDimensionType, normalizeKtxNativeType } from '../../../scan/type-normalization.js';
|
||||
import type { LiveDatabaseIntrospectionPort } from './types.js';
|
||||
|
||||
export type KtxDaemonDatabaseIntrospectionCommand = 'database-introspect';
|
||||
type KtxDaemonDatabaseIntrospectionCommand = 'database-introspect';
|
||||
|
||||
export type KtxDaemonDatabaseJsonRunner = (
|
||||
type KtxDaemonDatabaseJsonRunner = (
|
||||
subcommand: KtxDaemonDatabaseIntrospectionCommand,
|
||||
payload: Record<string, unknown>,
|
||||
) => Promise<Record<string, unknown>>;
|
||||
|
|
|
|||
|
|
@ -1,136 +0,0 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type { KtxSchemaSnapshot } from '../../../scan/types.js';
|
||||
import { buildLiveDatabaseTableNaturalKey, ktxSchemaSnapshotToExtractedSchema } from './extracted-schema.js';
|
||||
|
||||
function snapshot(): KtxSchemaSnapshot {
|
||||
return {
|
||||
connectionId: 'conn-1',
|
||||
driver: 'postgres',
|
||||
extractedAt: '2026-04-27T00:00:00.000Z',
|
||||
scope: { schemas: ['public'] },
|
||||
metadata: { driver: 'postgres' },
|
||||
tables: [
|
||||
{
|
||||
name: 'orders',
|
||||
catalog: null,
|
||||
db: 'public',
|
||||
kind: 'table',
|
||||
comment: 'Orders placed by customers',
|
||||
estimatedRows: null,
|
||||
columns: [
|
||||
{
|
||||
name: 'id',
|
||||
nativeType: 'integer',
|
||||
normalizedType: 'integer',
|
||||
dimensionType: 'number',
|
||||
nullable: false,
|
||||
primaryKey: true,
|
||||
comment: 'Primary key',
|
||||
},
|
||||
{
|
||||
name: 'customer_id',
|
||||
nativeType: 'integer',
|
||||
normalizedType: 'integer',
|
||||
dimensionType: 'number',
|
||||
nullable: false,
|
||||
primaryKey: false,
|
||||
comment: null,
|
||||
},
|
||||
],
|
||||
foreignKeys: [
|
||||
{
|
||||
fromColumn: 'customer_id',
|
||||
toCatalog: null,
|
||||
toDb: 'public',
|
||||
toTable: 'customers',
|
||||
toColumn: 'id',
|
||||
constraintName: 'orders_customer_id_fkey',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'customers',
|
||||
catalog: null,
|
||||
db: 'public',
|
||||
kind: 'table',
|
||||
comment: null,
|
||||
estimatedRows: null,
|
||||
columns: [
|
||||
{
|
||||
name: 'id',
|
||||
nativeType: 'integer',
|
||||
normalizedType: 'integer',
|
||||
dimensionType: 'number',
|
||||
nullable: false,
|
||||
primaryKey: true,
|
||||
comment: null,
|
||||
},
|
||||
],
|
||||
foreignKeys: [],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe('ktxSchemaSnapshotToExtractedSchema', () => {
|
||||
it('preserves structural table, column, comment, and key metadata', () => {
|
||||
const extracted = ktxSchemaSnapshotToExtractedSchema(snapshot());
|
||||
|
||||
expect(extracted.tables).toEqual([
|
||||
{
|
||||
name: 'orders',
|
||||
catalog: null,
|
||||
db: 'public',
|
||||
dbComment: 'Orders placed by customers',
|
||||
columns: [
|
||||
{
|
||||
name: 'id',
|
||||
type: 'integer',
|
||||
nullable: false,
|
||||
primaryKey: true,
|
||||
dbComment: 'Primary key',
|
||||
},
|
||||
{
|
||||
name: 'customer_id',
|
||||
type: 'integer',
|
||||
nullable: false,
|
||||
primaryKey: false,
|
||||
dbComment: null,
|
||||
},
|
||||
],
|
||||
foreignKeys: [
|
||||
{
|
||||
fromTable: 'orders',
|
||||
fromColumn: 'customer_id',
|
||||
toTable: 'customers',
|
||||
toColumn: 'id',
|
||||
constraintName: 'orders_customer_id_fkey',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'customers',
|
||||
catalog: null,
|
||||
db: 'public',
|
||||
dbComment: null,
|
||||
columns: [
|
||||
{
|
||||
name: 'id',
|
||||
type: 'integer',
|
||||
nullable: false,
|
||||
primaryKey: true,
|
||||
dbComment: null,
|
||||
},
|
||||
],
|
||||
foreignKeys: [],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('builds the same natural key shape used by schema sync', () => {
|
||||
expect(buildLiveDatabaseTableNaturalKey({ catalog: null, db: 'public', name: 'orders' })).toBe('|public|orders');
|
||||
expect(buildLiveDatabaseTableNaturalKey({ catalog: 'warehouse', db: 'analytics', name: 'events' })).toBe(
|
||||
'warehouse|analytics|events',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
import type { KtxSchemaSnapshot, KtxSchemaTable } from '../../../scan/types.js';
|
||||
|
||||
export interface LiveDatabaseExtractedForeignKey {
|
||||
fromTable: string;
|
||||
fromColumn: string;
|
||||
toTable: string;
|
||||
toColumn: string;
|
||||
constraintName?: string;
|
||||
}
|
||||
|
||||
export interface LiveDatabaseExtractedColumn {
|
||||
name: string;
|
||||
type: string;
|
||||
nullable: boolean;
|
||||
primaryKey: boolean;
|
||||
dbComment: string | null;
|
||||
}
|
||||
|
||||
export interface LiveDatabaseExtractedTable {
|
||||
name: string;
|
||||
catalog: string | null;
|
||||
db: string | null;
|
||||
dbComment: string | null;
|
||||
columns: LiveDatabaseExtractedColumn[];
|
||||
foreignKeys: LiveDatabaseExtractedForeignKey[];
|
||||
}
|
||||
|
||||
export interface LiveDatabaseExtractedSchema {
|
||||
connectionId?: string;
|
||||
tables: LiveDatabaseExtractedTable[];
|
||||
}
|
||||
|
||||
export function buildLiveDatabaseTableNaturalKey(table: Pick<KtxSchemaTable, 'catalog' | 'db' | 'name'>): string {
|
||||
return `${table.catalog ?? ''}|${table.db ?? ''}|${table.name}`;
|
||||
}
|
||||
|
||||
export function ktxSchemaSnapshotToExtractedSchema(snapshot: KtxSchemaSnapshot): LiveDatabaseExtractedSchema {
|
||||
return {
|
||||
connectionId: snapshot.connectionId,
|
||||
tables: snapshot.tables.map((table) => ({
|
||||
name: table.name,
|
||||
catalog: table.catalog ?? null,
|
||||
db: table.db ?? null,
|
||||
dbComment: table.comment ?? null,
|
||||
columns: table.columns.map((column) => ({
|
||||
name: column.name,
|
||||
type: column.nativeType,
|
||||
nullable: column.nullable,
|
||||
primaryKey: column.primaryKey,
|
||||
dbComment: column.comment ?? null,
|
||||
})),
|
||||
foreignKeys: table.foreignKeys.map((foreignKey) => ({
|
||||
fromTable: table.name,
|
||||
fromColumn: foreignKey.fromColumn,
|
||||
toTable: foreignKey.toTable,
|
||||
toColumn: foreignKey.toColumn,
|
||||
...(foreignKey.constraintName ? { constraintName: foreignKey.constraintName } : {}),
|
||||
})),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
|
@ -22,7 +22,7 @@ const HISTORIC_SQL_MANAGED_USAGE_KEYS = new Set([
|
|||
'staleSince',
|
||||
]);
|
||||
|
||||
export interface LiveDatabaseManifestColumn {
|
||||
interface LiveDatabaseManifestColumn {
|
||||
name: string;
|
||||
type: string;
|
||||
pk?: boolean;
|
||||
|
|
@ -37,7 +37,7 @@ export interface LiveDatabaseManifestJoinEntry {
|
|||
source: string;
|
||||
}
|
||||
|
||||
export interface LiveDatabaseManifestTableEntry {
|
||||
interface LiveDatabaseManifestTableEntry {
|
||||
table: string;
|
||||
descriptions?: Record<string, string>;
|
||||
usage?: TableUsageOutput;
|
||||
|
|
|
|||
|
|
@ -1,428 +0,0 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { type LiveDatabaseSyncedSchema, planLiveDatabaseStructuralSync } from './structural-sync.js';
|
||||
|
||||
function idFactory(): () => string {
|
||||
let next = 1;
|
||||
return () => `id-${next++}`;
|
||||
}
|
||||
|
||||
describe('planLiveDatabaseStructuralSync', () => {
|
||||
it('plans table and column creates, updates, deletes, and metadata invalidation', () => {
|
||||
const current: LiveDatabaseSyncedSchema = {
|
||||
connectionId: 'conn-1',
|
||||
tables: [
|
||||
{
|
||||
id: 'tbl-orders',
|
||||
name: 'orders',
|
||||
catalog: null,
|
||||
db: 'public',
|
||||
enabled: true,
|
||||
descriptions: { ai: 'Old AI order text', db: 'Old DB order text' },
|
||||
columns: [
|
||||
{
|
||||
id: 'col-order-id',
|
||||
name: 'id',
|
||||
type: 'number',
|
||||
nullable: false,
|
||||
primaryKey: true,
|
||||
parentColumnId: null,
|
||||
descriptions: { db: 'Order id' },
|
||||
embedding: [1, 2, 3],
|
||||
sampleValues: null,
|
||||
cardinality: null,
|
||||
},
|
||||
{
|
||||
id: 'col-order-total',
|
||||
name: 'total',
|
||||
type: 'number',
|
||||
nullable: true,
|
||||
primaryKey: false,
|
||||
parentColumnId: null,
|
||||
descriptions: { ai: 'Old AI total text', db: 'Old total text' },
|
||||
embedding: [4, 5, 6],
|
||||
sampleValues: ['10'],
|
||||
cardinality: 12,
|
||||
},
|
||||
{
|
||||
id: 'col-order-removed',
|
||||
name: 'removed',
|
||||
type: 'string',
|
||||
nullable: true,
|
||||
primaryKey: false,
|
||||
parentColumnId: null,
|
||||
descriptions: {},
|
||||
embedding: null,
|
||||
sampleValues: null,
|
||||
cardinality: null,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'tbl-removed',
|
||||
name: 'removed_table',
|
||||
catalog: null,
|
||||
db: 'public',
|
||||
enabled: true,
|
||||
descriptions: {},
|
||||
columns: [
|
||||
{
|
||||
id: 'col-removed-id',
|
||||
name: 'id',
|
||||
type: 'number',
|
||||
nullable: false,
|
||||
primaryKey: true,
|
||||
parentColumnId: null,
|
||||
descriptions: {},
|
||||
embedding: null,
|
||||
sampleValues: null,
|
||||
cardinality: null,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
links: [
|
||||
{
|
||||
id: 'inferred-total-link',
|
||||
fromTableId: 'tbl-orders',
|
||||
fromColumnId: 'col-order-total',
|
||||
toTableId: 'tbl-orders',
|
||||
toColumnId: 'col-order-id',
|
||||
source: 'inferred',
|
||||
confidence: 0.7,
|
||||
relationshipType: 'MANY_TO_ONE',
|
||||
isPrimaryKeyReference: true,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const plan = planLiveDatabaseStructuralSync({
|
||||
connectionId: 'conn-1',
|
||||
current,
|
||||
extracted: {
|
||||
connectionId: 'conn-1',
|
||||
tables: [
|
||||
{
|
||||
name: 'orders',
|
||||
catalog: null,
|
||||
db: 'public',
|
||||
dbComment: 'Fresh DB order text',
|
||||
columns: [
|
||||
{
|
||||
name: 'id',
|
||||
type: 'number',
|
||||
nullable: false,
|
||||
primaryKey: true,
|
||||
dbComment: 'Order id',
|
||||
},
|
||||
{
|
||||
name: 'total',
|
||||
type: 'string',
|
||||
nullable: false,
|
||||
primaryKey: false,
|
||||
dbComment: 'Fresh total text',
|
||||
},
|
||||
{
|
||||
name: 'created_at',
|
||||
type: 'time',
|
||||
nullable: false,
|
||||
primaryKey: false,
|
||||
dbComment: 'Creation timestamp',
|
||||
},
|
||||
],
|
||||
foreignKeys: [],
|
||||
},
|
||||
{
|
||||
name: 'customers',
|
||||
catalog: null,
|
||||
db: 'public',
|
||||
dbComment: 'Customer table',
|
||||
columns: [
|
||||
{
|
||||
name: 'id',
|
||||
type: 'number',
|
||||
nullable: false,
|
||||
primaryKey: true,
|
||||
dbComment: null,
|
||||
},
|
||||
],
|
||||
foreignKeys: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
idFactory: idFactory(),
|
||||
});
|
||||
|
||||
expect(plan.stats).toEqual({
|
||||
tablesCreated: 1,
|
||||
tablesDeleted: 1,
|
||||
columnsCreated: 2,
|
||||
columnsDeleted: 2,
|
||||
columnsModified: 1,
|
||||
formalLinksCreated: 0,
|
||||
formalLinksDeleted: 0,
|
||||
});
|
||||
expect(plan.operations.deleteTableIds).toEqual(['tbl-removed']);
|
||||
expect(plan.operations.deleteColumnIds).toEqual(['col-order-removed']);
|
||||
expect(plan.operations.insertTables).toEqual([
|
||||
{
|
||||
id: 'id-2',
|
||||
connectionId: 'conn-1',
|
||||
name: 'customers',
|
||||
catalog: null,
|
||||
db: 'public',
|
||||
enabled: true,
|
||||
},
|
||||
]);
|
||||
expect(plan.operations.insertColumns).toEqual([
|
||||
{
|
||||
id: 'id-1',
|
||||
tableId: 'tbl-orders',
|
||||
name: 'created_at',
|
||||
parentColumnId: null,
|
||||
},
|
||||
{
|
||||
id: 'id-3',
|
||||
tableId: 'id-2',
|
||||
name: 'id',
|
||||
parentColumnId: null,
|
||||
},
|
||||
]);
|
||||
expect(plan.operations.touchColumnIds).toEqual(['col-order-total']);
|
||||
expect(plan.operations.invalidateColumnEmbeddingIds).toEqual(['col-order-total']);
|
||||
expect(plan.inferredLinksToValidate).toEqual(['inferred-total-link']);
|
||||
expect(plan.changes).toEqual({
|
||||
newTableIds: ['id-2'],
|
||||
newColumnIds: ['id-1', 'id-3'],
|
||||
tablesWithStructuralChanges: ['tbl-orders', 'id-2'],
|
||||
columnsWithTypeChange: ['col-order-total'],
|
||||
columnsWithDescriptionChange: ['col-order-total'],
|
||||
tablesWithDescriptionChange: ['tbl-orders'],
|
||||
});
|
||||
|
||||
const orders = plan.schema.tables.find((table) => table.name === 'orders');
|
||||
expect(orders?.descriptions).toEqual({ db: 'Fresh DB order text' });
|
||||
expect(orders?.columns.map((column) => column.name)).toEqual(['id', 'total', 'created_at']);
|
||||
expect(orders?.columns.find((column) => column.name === 'total')).toMatchObject({
|
||||
id: 'col-order-total',
|
||||
type: 'string',
|
||||
nullable: false,
|
||||
primaryKey: false,
|
||||
descriptions: { db: 'Fresh total text' },
|
||||
embedding: null,
|
||||
sampleValues: ['10'],
|
||||
cardinality: 12,
|
||||
});
|
||||
});
|
||||
|
||||
it('builds formal links from extracted foreign keys and preserves valid inferred links', () => {
|
||||
const current: LiveDatabaseSyncedSchema = {
|
||||
connectionId: 'conn-1',
|
||||
tables: [
|
||||
{
|
||||
id: 'tbl-orders',
|
||||
name: 'orders',
|
||||
catalog: null,
|
||||
db: 'public',
|
||||
enabled: true,
|
||||
descriptions: {},
|
||||
columns: [
|
||||
{
|
||||
id: 'col-orders-id',
|
||||
name: 'id',
|
||||
type: 'number',
|
||||
nullable: false,
|
||||
primaryKey: true,
|
||||
parentColumnId: null,
|
||||
descriptions: {},
|
||||
embedding: null,
|
||||
sampleValues: null,
|
||||
cardinality: null,
|
||||
},
|
||||
{
|
||||
id: 'col-orders-customer',
|
||||
name: 'customer_id',
|
||||
type: 'number',
|
||||
nullable: false,
|
||||
primaryKey: false,
|
||||
parentColumnId: null,
|
||||
descriptions: {},
|
||||
embedding: null,
|
||||
sampleValues: null,
|
||||
cardinality: null,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'tbl-customers',
|
||||
name: 'customers',
|
||||
catalog: null,
|
||||
db: 'public',
|
||||
enabled: true,
|
||||
descriptions: {},
|
||||
columns: [
|
||||
{
|
||||
id: 'col-customers-id',
|
||||
name: 'id',
|
||||
type: 'number',
|
||||
nullable: false,
|
||||
primaryKey: true,
|
||||
parentColumnId: null,
|
||||
descriptions: {},
|
||||
embedding: null,
|
||||
sampleValues: null,
|
||||
cardinality: null,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
links: [
|
||||
{
|
||||
id: 'formal-existing',
|
||||
fromTableId: 'tbl-orders',
|
||||
fromColumnId: 'col-orders-customer',
|
||||
toTableId: 'tbl-customers',
|
||||
toColumnId: 'col-customers-id',
|
||||
source: 'formal',
|
||||
confidence: 1,
|
||||
relationshipType: 'MANY_TO_ONE',
|
||||
isPrimaryKeyReference: true,
|
||||
},
|
||||
{
|
||||
id: 'inferred-existing',
|
||||
fromTableId: 'tbl-orders',
|
||||
fromColumnId: 'col-orders-id',
|
||||
toTableId: 'tbl-customers',
|
||||
toColumnId: 'col-customers-id',
|
||||
source: 'inferred',
|
||||
confidence: 0.6,
|
||||
relationshipType: 'MANY_TO_ONE',
|
||||
isPrimaryKeyReference: true,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const plan = planLiveDatabaseStructuralSync({
|
||||
connectionId: 'conn-1',
|
||||
current,
|
||||
extracted: {
|
||||
connectionId: 'conn-1',
|
||||
tables: [
|
||||
{
|
||||
name: 'orders',
|
||||
catalog: null,
|
||||
db: 'public',
|
||||
dbComment: null,
|
||||
columns: [
|
||||
{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null },
|
||||
{ name: 'customer_id', type: 'number', nullable: false, primaryKey: false, dbComment: null },
|
||||
],
|
||||
foreignKeys: [
|
||||
{
|
||||
fromTable: 'orders',
|
||||
fromColumn: 'customer_id',
|
||||
toTable: 'customers',
|
||||
toColumn: 'id',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'customers',
|
||||
catalog: null,
|
||||
db: 'public',
|
||||
dbComment: null,
|
||||
columns: [{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }],
|
||||
foreignKeys: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
idFactory: idFactory(),
|
||||
});
|
||||
|
||||
expect(plan.stats.formalLinksCreated).toBe(0);
|
||||
expect(plan.stats.formalLinksDeleted).toBe(0);
|
||||
expect(plan.schema.links.map((link) => link.id)).toEqual(['formal-existing', 'inferred-existing']);
|
||||
|
||||
const planAfterForeignKeyRemoval = planLiveDatabaseStructuralSync({
|
||||
connectionId: 'conn-1',
|
||||
current,
|
||||
extracted: {
|
||||
connectionId: 'conn-1',
|
||||
tables: [
|
||||
{
|
||||
name: 'orders',
|
||||
catalog: null,
|
||||
db: 'public',
|
||||
dbComment: null,
|
||||
columns: [
|
||||
{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null },
|
||||
{ name: 'customer_id', type: 'number', nullable: false, primaryKey: false, dbComment: null },
|
||||
],
|
||||
foreignKeys: [],
|
||||
},
|
||||
{
|
||||
name: 'customers',
|
||||
catalog: null,
|
||||
db: 'public',
|
||||
dbComment: null,
|
||||
columns: [{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }],
|
||||
foreignKeys: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
idFactory: idFactory(),
|
||||
});
|
||||
|
||||
expect(planAfterForeignKeyRemoval.stats.formalLinksDeleted).toBe(1);
|
||||
expect(planAfterForeignKeyRemoval.schema.links.map((link) => link.id)).toEqual(['inferred-existing']);
|
||||
|
||||
const planAfterForeignKeyCreation = planLiveDatabaseStructuralSync({
|
||||
connectionId: 'conn-1',
|
||||
current: { ...current, links: [current.links[1]] },
|
||||
extracted: {
|
||||
connectionId: 'conn-1',
|
||||
tables: [
|
||||
{
|
||||
name: 'orders',
|
||||
catalog: null,
|
||||
db: 'public',
|
||||
dbComment: null,
|
||||
columns: [
|
||||
{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null },
|
||||
{ name: 'customer_id', type: 'number', nullable: false, primaryKey: false, dbComment: null },
|
||||
],
|
||||
foreignKeys: [
|
||||
{
|
||||
fromTable: 'orders',
|
||||
fromColumn: 'customer_id',
|
||||
toTable: 'customers',
|
||||
toColumn: 'id',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'customers',
|
||||
catalog: null,
|
||||
db: 'public',
|
||||
dbComment: null,
|
||||
columns: [{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }],
|
||||
foreignKeys: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
idFactory: idFactory(),
|
||||
});
|
||||
|
||||
expect(planAfterForeignKeyCreation.stats.formalLinksCreated).toBe(1);
|
||||
expect(planAfterForeignKeyCreation.schema.links[0]).toMatchObject({
|
||||
id: 'id-1',
|
||||
fromTableId: 'tbl-orders',
|
||||
fromColumnId: 'col-orders-customer',
|
||||
toTableId: 'tbl-customers',
|
||||
toColumnId: 'col-customers-id',
|
||||
source: 'formal',
|
||||
confidence: 1,
|
||||
relationshipType: 'MANY_TO_ONE',
|
||||
isPrimaryKeyReference: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,525 +0,0 @@
|
|||
import type { LiveDatabaseExtractedSchema, LiveDatabaseExtractedTable } from './extracted-schema.js';
|
||||
import { buildLiveDatabaseTableNaturalKey } from './extracted-schema.js';
|
||||
|
||||
export interface LiveDatabaseSyncedColumn {
|
||||
id: string;
|
||||
name: string;
|
||||
type: string;
|
||||
nullable: boolean;
|
||||
primaryKey: boolean;
|
||||
parentColumnId: string | null;
|
||||
descriptions: Record<string, string>;
|
||||
embedding: number[] | null;
|
||||
sampleValues: string[] | null;
|
||||
cardinality: number | null;
|
||||
}
|
||||
|
||||
export interface LiveDatabaseSyncedTable {
|
||||
id: string;
|
||||
name: string;
|
||||
catalog: string | null;
|
||||
db: string | null;
|
||||
enabled: boolean;
|
||||
descriptions: Record<string, string>;
|
||||
columns: LiveDatabaseSyncedColumn[];
|
||||
}
|
||||
|
||||
export interface LiveDatabaseSyncedLink {
|
||||
id: string;
|
||||
fromTableId: string;
|
||||
fromColumnId: string;
|
||||
toTableId: string;
|
||||
toColumnId: string;
|
||||
source: 'formal' | 'inferred' | 'manual';
|
||||
confidence: number;
|
||||
relationshipType: string;
|
||||
isPrimaryKeyReference: boolean;
|
||||
}
|
||||
|
||||
export interface LiveDatabaseSyncedSchema {
|
||||
connectionId: string;
|
||||
tables: LiveDatabaseSyncedTable[];
|
||||
links: LiveDatabaseSyncedLink[];
|
||||
}
|
||||
|
||||
export interface LiveDatabaseStructuralChanges {
|
||||
newTableIds: string[];
|
||||
newColumnIds: string[];
|
||||
tablesWithStructuralChanges: string[];
|
||||
columnsWithTypeChange: string[];
|
||||
columnsWithDescriptionChange: string[];
|
||||
tablesWithDescriptionChange: string[];
|
||||
}
|
||||
|
||||
export interface LiveDatabaseStructuralSyncStats {
|
||||
tablesCreated: number;
|
||||
tablesDeleted: number;
|
||||
columnsCreated: number;
|
||||
columnsDeleted: number;
|
||||
columnsModified: number;
|
||||
formalLinksCreated: number;
|
||||
formalLinksDeleted: number;
|
||||
}
|
||||
|
||||
export interface LiveDatabaseStructuralSyncOperations {
|
||||
deleteTableIds: string[];
|
||||
deleteColumnIds: string[];
|
||||
insertTables: Array<{
|
||||
id: string;
|
||||
connectionId: string;
|
||||
name: string;
|
||||
catalog: string | null;
|
||||
db: string | null;
|
||||
enabled: boolean;
|
||||
}>;
|
||||
insertColumns: Array<{
|
||||
id: string;
|
||||
tableId: string;
|
||||
name: string;
|
||||
parentColumnId: string | null;
|
||||
}>;
|
||||
touchColumnIds: string[];
|
||||
invalidateColumnEmbeddingIds: string[];
|
||||
}
|
||||
|
||||
export interface LiveDatabaseStructuralSyncPlan {
|
||||
schema: LiveDatabaseSyncedSchema;
|
||||
inferredLinksToValidate: string[];
|
||||
stats: LiveDatabaseStructuralSyncStats;
|
||||
changes: LiveDatabaseStructuralChanges;
|
||||
operations: LiveDatabaseStructuralSyncOperations;
|
||||
}
|
||||
|
||||
export interface PlanLiveDatabaseStructuralSyncInput {
|
||||
connectionId: string;
|
||||
current: LiveDatabaseSyncedSchema | null;
|
||||
extracted: LiveDatabaseExtractedSchema;
|
||||
idFactory: () => string;
|
||||
}
|
||||
|
||||
interface UpdatedTableResult {
|
||||
table: LiveDatabaseSyncedTable;
|
||||
columnsCreated: number;
|
||||
columnsDeleted: number;
|
||||
columnsModified: number;
|
||||
newColumnIds: string[];
|
||||
columnsWithTypeChange: string[];
|
||||
columnsWithDescriptionChange: string[];
|
||||
tableDescriptionChanged: boolean;
|
||||
}
|
||||
|
||||
function updateDescription(
|
||||
descriptions: Record<string, string>,
|
||||
dbComment: string | null | undefined,
|
||||
changed: boolean,
|
||||
): Record<string, string> {
|
||||
const updated = { ...descriptions };
|
||||
if (dbComment) {
|
||||
updated.db = dbComment;
|
||||
} else {
|
||||
delete updated.db;
|
||||
}
|
||||
if (changed) {
|
||||
delete updated.ai;
|
||||
}
|
||||
return updated;
|
||||
}
|
||||
|
||||
function descriptionFromDbComment(dbComment: string | null | undefined): Record<string, string> {
|
||||
return dbComment ? { db: dbComment } : {};
|
||||
}
|
||||
|
||||
function planUpdatedTable(args: {
|
||||
currentTable: LiveDatabaseSyncedTable;
|
||||
extractedTable: LiveDatabaseExtractedTable;
|
||||
currentLinks: LiveDatabaseSyncedLink[];
|
||||
inferredLinksToValidate: string[];
|
||||
operations: LiveDatabaseStructuralSyncOperations;
|
||||
idFactory: () => string;
|
||||
}): UpdatedTableResult {
|
||||
const { currentTable, extractedTable, currentLinks, inferredLinksToValidate, operations, idFactory } = args;
|
||||
|
||||
let columnsCreated = 0;
|
||||
let columnsDeleted = 0;
|
||||
let columnsModified = 0;
|
||||
const newColumnIds: string[] = [];
|
||||
const columnsWithTypeChange: string[] = [];
|
||||
const columnsWithDescriptionChange: string[] = [];
|
||||
const updatedColumns: LiveDatabaseSyncedColumn[] = [];
|
||||
|
||||
const tableDescriptionChanged = (currentTable.descriptions.db ?? null) !== (extractedTable.dbComment ?? null);
|
||||
const currentColumnsByName = new Map(currentTable.columns.map((column) => [column.name, column]));
|
||||
const extractedColumnsByName = new Map(extractedTable.columns.map((column) => [column.name, column]));
|
||||
|
||||
for (const [name, currentColumn] of currentColumnsByName) {
|
||||
if (!extractedColumnsByName.has(name)) {
|
||||
operations.deleteColumnIds.push(currentColumn.id);
|
||||
columnsDeleted++;
|
||||
}
|
||||
}
|
||||
|
||||
for (const [name, extractedColumn] of extractedColumnsByName) {
|
||||
const currentColumn = currentColumnsByName.get(name);
|
||||
if (!currentColumn) {
|
||||
const columnId = idFactory();
|
||||
operations.insertColumns.push({
|
||||
id: columnId,
|
||||
tableId: currentTable.id,
|
||||
name: extractedColumn.name,
|
||||
parentColumnId: null,
|
||||
});
|
||||
columnsCreated++;
|
||||
newColumnIds.push(columnId);
|
||||
updatedColumns.push({
|
||||
id: columnId,
|
||||
name: extractedColumn.name,
|
||||
type: extractedColumn.type,
|
||||
nullable: extractedColumn.nullable,
|
||||
primaryKey: extractedColumn.primaryKey,
|
||||
descriptions: descriptionFromDbComment(extractedColumn.dbComment),
|
||||
parentColumnId: null,
|
||||
embedding: null,
|
||||
sampleValues: null,
|
||||
cardinality: null,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const typeChanged = currentColumn.type !== extractedColumn.type;
|
||||
const nullableChanged = currentColumn.nullable !== extractedColumn.nullable;
|
||||
const primaryKeyChanged = currentColumn.primaryKey !== extractedColumn.primaryKey;
|
||||
const dbDescriptionChanged = (currentColumn.descriptions.db ?? null) !== (extractedColumn.dbComment ?? null);
|
||||
|
||||
if (typeChanged || nullableChanged || primaryKeyChanged || dbDescriptionChanged) {
|
||||
operations.touchColumnIds.push(currentColumn.id);
|
||||
columnsModified++;
|
||||
|
||||
if (typeChanged || dbDescriptionChanged) {
|
||||
operations.invalidateColumnEmbeddingIds.push(currentColumn.id);
|
||||
}
|
||||
|
||||
if (typeChanged) {
|
||||
columnsWithTypeChange.push(currentColumn.id);
|
||||
const affectedLinks = currentLinks.filter(
|
||||
(link) =>
|
||||
link.source === 'inferred' &&
|
||||
(link.fromColumnId === currentColumn.id || link.toColumnId === currentColumn.id),
|
||||
);
|
||||
for (const link of affectedLinks) {
|
||||
if (!inferredLinksToValidate.includes(link.id)) {
|
||||
inferredLinksToValidate.push(link.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (dbDescriptionChanged) {
|
||||
columnsWithDescriptionChange.push(currentColumn.id);
|
||||
}
|
||||
}
|
||||
|
||||
updatedColumns.push({
|
||||
...currentColumn,
|
||||
type: extractedColumn.type,
|
||||
nullable: extractedColumn.nullable,
|
||||
primaryKey: extractedColumn.primaryKey,
|
||||
descriptions: updateDescription(currentColumn.descriptions, extractedColumn.dbComment, dbDescriptionChanged),
|
||||
embedding: typeChanged ? null : currentColumn.embedding,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
table: {
|
||||
...currentTable,
|
||||
descriptions: updateDescription(currentTable.descriptions, extractedTable.dbComment, tableDescriptionChanged),
|
||||
columns: updatedColumns,
|
||||
},
|
||||
columnsCreated,
|
||||
columnsDeleted,
|
||||
columnsModified,
|
||||
newColumnIds,
|
||||
columnsWithTypeChange,
|
||||
columnsWithDescriptionChange,
|
||||
tableDescriptionChanged,
|
||||
};
|
||||
}
|
||||
|
||||
function planCreatedTable(args: {
|
||||
connectionId: string;
|
||||
extractedTable: LiveDatabaseExtractedTable;
|
||||
operations: LiveDatabaseStructuralSyncOperations;
|
||||
idFactory: () => string;
|
||||
}): LiveDatabaseSyncedTable {
|
||||
const { connectionId, extractedTable, operations, idFactory } = args;
|
||||
const tableId = idFactory();
|
||||
operations.insertTables.push({
|
||||
id: tableId,
|
||||
connectionId,
|
||||
name: extractedTable.name,
|
||||
catalog: extractedTable.catalog,
|
||||
db: extractedTable.db,
|
||||
enabled: true,
|
||||
});
|
||||
|
||||
const columns: LiveDatabaseSyncedColumn[] = extractedTable.columns.map((extractedColumn) => {
|
||||
const columnId = idFactory();
|
||||
operations.insertColumns.push({
|
||||
id: columnId,
|
||||
tableId,
|
||||
name: extractedColumn.name,
|
||||
parentColumnId: null,
|
||||
});
|
||||
return {
|
||||
id: columnId,
|
||||
name: extractedColumn.name,
|
||||
type: extractedColumn.type,
|
||||
nullable: extractedColumn.nullable,
|
||||
primaryKey: extractedColumn.primaryKey,
|
||||
descriptions: descriptionFromDbComment(extractedColumn.dbComment),
|
||||
parentColumnId: null,
|
||||
embedding: null,
|
||||
sampleValues: null,
|
||||
cardinality: null,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
id: tableId,
|
||||
name: extractedTable.name,
|
||||
catalog: extractedTable.catalog,
|
||||
db: extractedTable.db,
|
||||
enabled: true,
|
||||
descriptions: descriptionFromDbComment(extractedTable.dbComment),
|
||||
columns,
|
||||
};
|
||||
}
|
||||
|
||||
function syncFormalLinks(args: {
|
||||
extracted: LiveDatabaseExtractedSchema;
|
||||
tables: LiveDatabaseSyncedTable[];
|
||||
tableNaturalKeyToId: Map<string, string>;
|
||||
currentLinks: LiveDatabaseSyncedLink[];
|
||||
idFactory: () => string;
|
||||
}): { links: LiveDatabaseSyncedLink[]; created: number; deleted: number } {
|
||||
const { extracted, tables, tableNaturalKeyToId, currentLinks, idFactory } = args;
|
||||
const columnKeyToId = new Map<string, string>();
|
||||
|
||||
for (const table of tables) {
|
||||
const tableKey = buildLiveDatabaseTableNaturalKey(table);
|
||||
for (const column of table.columns) {
|
||||
columnKeyToId.set(`${tableKey}.${column.name}`, column.id);
|
||||
}
|
||||
}
|
||||
|
||||
const extractedFormalLinks: Array<{
|
||||
fromTableId: string;
|
||||
fromColumnId: string;
|
||||
toTableId: string;
|
||||
toColumnId: string;
|
||||
}> = [];
|
||||
|
||||
for (const table of extracted.tables) {
|
||||
const fromTableKey = buildLiveDatabaseTableNaturalKey(table);
|
||||
const fromTableId = tableNaturalKeyToId.get(fromTableKey);
|
||||
if (!fromTableId) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const foreignKey of table.foreignKeys) {
|
||||
const toTableKey = buildLiveDatabaseTableNaturalKey({
|
||||
catalog: table.catalog,
|
||||
db: table.db,
|
||||
name: foreignKey.toTable,
|
||||
});
|
||||
const toTableId = tableNaturalKeyToId.get(toTableKey);
|
||||
if (!toTableId) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fromColumnId = columnKeyToId.get(`${fromTableKey}.${foreignKey.fromColumn}`);
|
||||
const toColumnId = columnKeyToId.get(`${toTableKey}.${foreignKey.toColumn}`);
|
||||
if (!fromColumnId || !toColumnId) {
|
||||
continue;
|
||||
}
|
||||
|
||||
extractedFormalLinks.push({ fromTableId, fromColumnId, toTableId, toColumnId });
|
||||
}
|
||||
}
|
||||
|
||||
const currentFormalLinks = currentLinks.filter((link) => link.source === 'formal');
|
||||
const extractedLinkKeys = new Set(extractedFormalLinks.map((link) => `${link.fromColumnId}->${link.toColumnId}`));
|
||||
const linksToDelete = currentFormalLinks.filter(
|
||||
(link) => !extractedLinkKeys.has(`${link.fromColumnId}->${link.toColumnId}`),
|
||||
);
|
||||
|
||||
const currentLinkKeys = new Set(currentFormalLinks.map((link) => `${link.fromColumnId}->${link.toColumnId}`));
|
||||
const linksToCreate = extractedFormalLinks.filter(
|
||||
(link) => !currentLinkKeys.has(`${link.fromColumnId}->${link.toColumnId}`),
|
||||
);
|
||||
|
||||
const newLinks = linksToCreate.map((linkData) => ({
|
||||
id: idFactory(),
|
||||
fromTableId: linkData.fromTableId,
|
||||
fromColumnId: linkData.fromColumnId,
|
||||
toTableId: linkData.toTableId,
|
||||
toColumnId: linkData.toColumnId,
|
||||
source: 'formal' as const,
|
||||
confidence: 1,
|
||||
relationshipType: 'MANY_TO_ONE',
|
||||
isPrimaryKeyReference: true,
|
||||
}));
|
||||
|
||||
const deletedLinkIds = new Set(linksToDelete.map((link) => link.id));
|
||||
const preservedFormalLinks = currentFormalLinks.filter((link) => !deletedLinkIds.has(link.id));
|
||||
|
||||
return {
|
||||
links: [...preservedFormalLinks, ...newLinks],
|
||||
created: linksToCreate.length,
|
||||
deleted: linksToDelete.length,
|
||||
};
|
||||
}
|
||||
|
||||
export function planLiveDatabaseStructuralSync(
|
||||
input: PlanLiveDatabaseStructuralSyncInput,
|
||||
): LiveDatabaseStructuralSyncPlan {
|
||||
const operations: LiveDatabaseStructuralSyncOperations = {
|
||||
deleteTableIds: [],
|
||||
deleteColumnIds: [],
|
||||
insertTables: [],
|
||||
insertColumns: [],
|
||||
touchColumnIds: [],
|
||||
invalidateColumnEmbeddingIds: [],
|
||||
};
|
||||
const stats: LiveDatabaseStructuralSyncStats = {
|
||||
tablesCreated: 0,
|
||||
tablesDeleted: 0,
|
||||
columnsCreated: 0,
|
||||
columnsDeleted: 0,
|
||||
columnsModified: 0,
|
||||
formalLinksCreated: 0,
|
||||
formalLinksDeleted: 0,
|
||||
};
|
||||
const changes: LiveDatabaseStructuralChanges = {
|
||||
newTableIds: [],
|
||||
newColumnIds: [],
|
||||
tablesWithStructuralChanges: [],
|
||||
columnsWithTypeChange: [],
|
||||
columnsWithDescriptionChange: [],
|
||||
tablesWithDescriptionChange: [],
|
||||
};
|
||||
const inferredLinksToValidate: string[] = [];
|
||||
|
||||
const currentTablesByKey = new Map<string, LiveDatabaseSyncedTable>();
|
||||
const extractedTablesByKey = new Map<string, LiveDatabaseExtractedTable>();
|
||||
|
||||
if (input.current) {
|
||||
for (const table of input.current.tables) {
|
||||
currentTablesByKey.set(buildLiveDatabaseTableNaturalKey(table), table);
|
||||
}
|
||||
}
|
||||
for (const table of input.extracted.tables) {
|
||||
extractedTablesByKey.set(buildLiveDatabaseTableNaturalKey(table), table);
|
||||
}
|
||||
|
||||
const tablesToDelete: LiveDatabaseSyncedTable[] = [];
|
||||
const tablesToUpdate: Array<{
|
||||
current: LiveDatabaseSyncedTable;
|
||||
extracted: LiveDatabaseExtractedTable;
|
||||
}> = [];
|
||||
const tablesToCreate: LiveDatabaseExtractedTable[] = [];
|
||||
|
||||
for (const [key, table] of currentTablesByKey) {
|
||||
const extractedTable = extractedTablesByKey.get(key);
|
||||
if (!extractedTable) {
|
||||
tablesToDelete.push(table);
|
||||
} else {
|
||||
tablesToUpdate.push({ current: table, extracted: extractedTable });
|
||||
}
|
||||
}
|
||||
|
||||
for (const [key, table] of extractedTablesByKey) {
|
||||
if (!currentTablesByKey.has(key)) {
|
||||
tablesToCreate.push(table);
|
||||
}
|
||||
}
|
||||
|
||||
for (const table of tablesToDelete) {
|
||||
operations.deleteTableIds.push(table.id);
|
||||
stats.tablesDeleted++;
|
||||
stats.columnsDeleted += table.columns.length;
|
||||
}
|
||||
|
||||
const updatedTables: LiveDatabaseSyncedTable[] = [];
|
||||
for (const { current, extracted } of tablesToUpdate) {
|
||||
const result = planUpdatedTable({
|
||||
currentTable: current,
|
||||
extractedTable: extracted,
|
||||
currentLinks: input.current?.links ?? [],
|
||||
inferredLinksToValidate,
|
||||
operations,
|
||||
idFactory: input.idFactory,
|
||||
});
|
||||
updatedTables.push(result.table);
|
||||
stats.columnsCreated += result.columnsCreated;
|
||||
stats.columnsDeleted += result.columnsDeleted;
|
||||
stats.columnsModified += result.columnsModified;
|
||||
changes.newColumnIds.push(...result.newColumnIds);
|
||||
changes.columnsWithTypeChange.push(...result.columnsWithTypeChange);
|
||||
changes.columnsWithDescriptionChange.push(...result.columnsWithDescriptionChange);
|
||||
if (result.tableDescriptionChanged) {
|
||||
changes.tablesWithDescriptionChange.push(current.id);
|
||||
}
|
||||
if (result.columnsCreated > 0 || result.columnsDeleted > 0 || result.columnsWithTypeChange.length > 0) {
|
||||
changes.tablesWithStructuralChanges.push(current.id);
|
||||
}
|
||||
}
|
||||
|
||||
const createdTables: LiveDatabaseSyncedTable[] = [];
|
||||
for (const extractedTable of tablesToCreate) {
|
||||
const table = planCreatedTable({
|
||||
connectionId: input.connectionId,
|
||||
extractedTable,
|
||||
operations,
|
||||
idFactory: input.idFactory,
|
||||
});
|
||||
createdTables.push(table);
|
||||
stats.tablesCreated++;
|
||||
stats.columnsCreated += table.columns.length;
|
||||
changes.newTableIds.push(table.id);
|
||||
changes.newColumnIds.push(...table.columns.map((column) => column.id));
|
||||
changes.tablesWithStructuralChanges.push(table.id);
|
||||
}
|
||||
|
||||
const allTables = [...updatedTables, ...createdTables];
|
||||
const tableNaturalKeyToId = new Map<string, string>();
|
||||
for (const table of allTables) {
|
||||
tableNaturalKeyToId.set(buildLiveDatabaseTableNaturalKey(table), table.id);
|
||||
}
|
||||
|
||||
const formalLinkResult = syncFormalLinks({
|
||||
extracted: input.extracted,
|
||||
tables: allTables,
|
||||
tableNaturalKeyToId,
|
||||
currentLinks: input.current?.links ?? [],
|
||||
idFactory: input.idFactory,
|
||||
});
|
||||
stats.formalLinksCreated = formalLinkResult.created;
|
||||
stats.formalLinksDeleted = formalLinkResult.deleted;
|
||||
|
||||
const deletedTableIds = new Set(tablesToDelete.map((table) => table.id));
|
||||
const preservedInferredLinks = (input.current?.links ?? []).filter(
|
||||
(link) =>
|
||||
link.source === 'inferred' && !deletedTableIds.has(link.fromTableId) && !deletedTableIds.has(link.toTableId),
|
||||
);
|
||||
|
||||
return {
|
||||
schema: {
|
||||
connectionId: input.connectionId,
|
||||
tables: allTables,
|
||||
links: [...formalLinkResult.links, ...preservedInferredLinks],
|
||||
},
|
||||
inferredLinksToValidate,
|
||||
stats,
|
||||
changes,
|
||||
operations,
|
||||
};
|
||||
}
|
||||
|
|
@ -7,6 +7,7 @@ export interface LookerCredentialResolver {
|
|||
resolve(lookerConnectionId: string): Promise<LookerConnectionParams>;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface LookerConnectionClientFactory {
|
||||
createClient(lookerConnectionId: string): Promise<LookerRuntimeClient>;
|
||||
}
|
||||
|
|
@ -23,6 +24,7 @@ export class DefaultLookerConnectionClientFactory implements LookerConnectionCli
|
|||
}
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export class DefaultLookerClientFactory implements LookerClientFactory {
|
||||
constructor(private readonly inner: LookerConnectionClientFactory) {}
|
||||
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ import {
|
|||
stagedUserFileSchema,
|
||||
} from './types.js';
|
||||
|
||||
export interface LookerEntityRef {
|
||||
interface LookerEntityRef {
|
||||
id: string;
|
||||
updatedAt?: string | null;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,8 @@
|
|||
import type { KtxLocalProject, KtxProjectConnectionConfig } from '../../../project/index.js';
|
||||
import type { LookerClientLogger } from './client.js';
|
||||
import type { KtxLocalProject } from '../../../../context/project/project.js';
|
||||
import type { KtxProjectConnectionConfig } from '../../../../context/project/config.js';
|
||||
import {
|
||||
DefaultLookerClientFactory,
|
||||
DefaultLookerConnectionClientFactory,
|
||||
type LookerCredentialResolver,
|
||||
} from './factory.js';
|
||||
import { LookerSourceAdapter } from './looker.adapter.js';
|
||||
|
||||
function stringField(value: unknown): string | null {
|
||||
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
|
||||
|
|
@ -54,16 +51,3 @@ export function createLocalLookerCredentialResolver(
|
|||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createLocalLookerSourceAdapter(
|
||||
project: KtxLocalProject,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
logger?: LookerClientLogger,
|
||||
): LookerSourceAdapter {
|
||||
const connectionFactory = new DefaultLookerConnectionClientFactory(createLocalLookerCredentialResolver(project, env), {
|
||||
...(logger ? { logger } : {}),
|
||||
});
|
||||
return new LookerSourceAdapter({
|
||||
clientFactory: new DefaultLookerClientFactory(connectionFactory),
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import type { LookerWarehouseConnectionInfo } from './client.js';
|
|||
import type { LookerConnectionMapping } from './mapping.js';
|
||||
import type { LookerRuntimeCursors } from './types.js';
|
||||
|
||||
export type LocalLookerMappingSource = 'ktx.yaml' | 'cli' | 'refresh';
|
||||
type LocalLookerMappingSource = 'ktx.yaml' | 'cli' | 'refresh';
|
||||
|
||||
interface LocalLookerRuntimeStoreOptions {
|
||||
dbPath: string;
|
||||
|
|
@ -41,7 +41,7 @@ export interface ClearLocalLookerMappingsInput {
|
|||
lookerConnectionName?: string;
|
||||
}
|
||||
|
||||
export interface LookerSourceStateReader {
|
||||
interface LookerSourceStateReader {
|
||||
readMappings(lookerConnectionId: string): Promise<LookerConnectionMapping[]>;
|
||||
readCursors(lookerConnectionId: string): Promise<LookerRuntimeCursors>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import type { ParsedTargetTable } from '../../parsed-target-table.js';
|
|||
import type { LookerWarehouseConnectionInfo } from './client.js';
|
||||
import type { LookerPullConfig, LookerRuntimeCursors, StagedExploreFile, StagedLookmlModelsFile } from './types.js';
|
||||
|
||||
export const LOOKER_DIALECT_TO_CONNECTION_TYPE = {
|
||||
const LOOKER_DIALECT_TO_CONNECTION_TYPE = {
|
||||
bigquery: 'BIGQUERY',
|
||||
bigquery_standard_sql: 'BIGQUERY',
|
||||
snowflake: 'SNOWFLAKE',
|
||||
|
|
@ -16,6 +16,7 @@ export const LOOKER_DIALECT_TO_CONNECTION_TYPE = {
|
|||
clickhouse: 'CLICKHOUSE',
|
||||
} as const;
|
||||
|
||||
/** @internal */
|
||||
export type LookerWarehouseTargetConnectionType =
|
||||
(typeof LOOKER_DIALECT_TO_CONNECTION_TYPE)[keyof typeof LOOKER_DIALECT_TO_CONNECTION_TYPE];
|
||||
|
||||
|
|
@ -33,6 +34,7 @@ export interface LookerTargetConnection {
|
|||
connection_params?: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface LookerMappingCandidateConnection extends LookerTargetConnection {}
|
||||
|
||||
export interface LookerMappingDrift {
|
||||
|
|
@ -89,6 +91,7 @@ export async function discoverLookerConnections(
|
|||
return client.listLookerConnections();
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function lookerDialectToConnectionType(dialect: string | null): LookerWarehouseTargetConnectionType | null {
|
||||
if (!dialect) {
|
||||
return null;
|
||||
|
|
@ -98,10 +101,12 @@ export function lookerDialectToConnectionType(dialect: string | null): LookerWar
|
|||
);
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function sqlglotDialectForConnectionType(connectionType: string): string | null {
|
||||
return SQLGLOT_DIALECT_BY_CONNECTION_TYPE[connectionType as LookerWarehouseTargetConnectionType] ?? null;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function validateLookerWarehouseTarget(connectionType: string): { ok: true } | { ok: false; reason: string } {
|
||||
return sqlglotDialectForConnectionType(connectionType)
|
||||
? { ok: true }
|
||||
|
|
@ -111,7 +116,7 @@ export function validateLookerWarehouseTarget(connectionType: string): { ok: tru
|
|||
};
|
||||
}
|
||||
|
||||
export function extractWarehouseHost(params: unknown, connectionType: string): string | null {
|
||||
function extractWarehouseHost(params: unknown, connectionType: string): string | null {
|
||||
const record = isRecord(params) ? params : {};
|
||||
switch (connectionType) {
|
||||
case 'POSTGRESQL':
|
||||
|
|
@ -126,7 +131,7 @@ export function extractWarehouseHost(params: unknown, connectionType: string): s
|
|||
}
|
||||
}
|
||||
|
||||
export function extractWarehouseDatabase(params: unknown, connectionType: string): string | null {
|
||||
function extractWarehouseDatabase(params: unknown, connectionType: string): string | null {
|
||||
const record = isRecord(params) ? params : {};
|
||||
switch (connectionType) {
|
||||
case 'POSTGRESQL':
|
||||
|
|
@ -142,14 +147,15 @@ export function extractWarehouseDatabase(params: unknown, connectionType: string
|
|||
}
|
||||
}
|
||||
|
||||
export function normalizeHost(value: string | null): string | null {
|
||||
function normalizeHost(value: string | null): string | null {
|
||||
return value ? value.toLowerCase().replace(/:\d+$/, '') : null;
|
||||
}
|
||||
|
||||
export function normalizeName(value: string | null): string | null {
|
||||
function normalizeName(value: string | null): string | null {
|
||||
return value ? value.toLowerCase() : null;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function suggestKtxConnectionForLookerConnection(args: {
|
||||
lookerConnection: LookerWarehouseConnectionInfo;
|
||||
candidateConnections: LookerMappingCandidateConnection[];
|
||||
|
|
@ -224,6 +230,7 @@ export function validateLookerMappings(args: {
|
|||
return errors.length === 0 ? { ok: true } : { ok: false, errors };
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function refreshLookerMappingPlaceholders(args: {
|
||||
stored: LookerConnectionMapping[];
|
||||
live: LookerWarehouseConnectionInfo[];
|
||||
|
|
@ -264,6 +271,7 @@ export function refreshLookerMappingPlaceholders(args: {
|
|||
return { mappings: [...byName.values()], changed };
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function collectExploreParseItems(args: {
|
||||
explore: StagedExploreFile;
|
||||
connectionMappings: Record<string, string>;
|
||||
|
|
@ -309,6 +317,7 @@ export function collectExploreParseItems(args: {
|
|||
return { parsedTargetTables, parseItems };
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function projectParsedIdentifier(row: LookerParsedIdentifier | undefined): ParsedTargetTable {
|
||||
if (!row) {
|
||||
return { ok: false, reason: 'parse_error', detail: 'Python parser response was missing this key.' };
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ export async function describeLookerScope(stagedDir: string): Promise<ScopeDescr
|
|||
};
|
||||
}
|
||||
|
||||
export async function readLookerScope(stagedDir: string): Promise<StagedLookerScopeFile> {
|
||||
async function readLookerScope(stagedDir: string): Promise<StagedLookerScopeFile> {
|
||||
try {
|
||||
const body = await readFile(join(stagedDir, STAGED_FILES.scope), 'utf-8');
|
||||
return stagedLookerScopeFileSchema.parse(JSON.parse(body));
|
||||
|
|
@ -27,6 +27,7 @@ export async function readLookerScope(stagedDir: string): Promise<StagedLookerSc
|
|||
}
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function hashLookerScope(scope: StagedLookerScopeFile): string {
|
||||
const canonical = JSON.stringify({
|
||||
mode: scope.mode,
|
||||
|
|
@ -36,6 +37,7 @@ export function hashLookerScope(scope: StagedLookerScopeFile): string {
|
|||
return createHash('sha256').update(canonical).digest('hex');
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function isPathInLookerScope(rawPath: string, scope: StagedLookerScopeFile): boolean {
|
||||
if (scope.mode === 'full') {
|
||||
return true;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type { ToolOutput } from '../../../../tools/index.js';
|
||||
import type { ToolOutput } from '../../../../../context/tools/base-tool.js';
|
||||
import { buildLookerSlProposal, createLookerQueryToSlTool, type LookerSlProposal } from './looker-query-to-sl.tool.js';
|
||||
|
||||
describe('buildLookerSlProposal', () => {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { tool } from 'ai';
|
||||
import { z } from 'zod';
|
||||
import type { ToolOutput } from '../../../../tools/index.js';
|
||||
import type { ToolOutput } from '../../../../../context/tools/base-tool.js';
|
||||
import type { ParsedTargetTable } from '../../../parsed-target-table.js';
|
||||
import { stagedLookerQuerySchema } from '../types.js';
|
||||
|
||||
|
|
@ -9,7 +9,7 @@ const lookerUsageInputSchema = z.object({
|
|||
uniqueUsers30d: z.number().int().nonnegative().default(0),
|
||||
});
|
||||
|
||||
export const lookerQueryToSlInputSchema = z.object({
|
||||
const lookerQueryToSlInputSchema = z.object({
|
||||
query: stagedLookerQuerySchema,
|
||||
contentTitle: z.string().min(1).optional(),
|
||||
contentType: z.enum(['look', 'dashboard_tile']).default('look'),
|
||||
|
|
@ -20,17 +20,17 @@ export type LookerQueryToSlInput = z.input<typeof lookerQueryToSlInputSchema>;
|
|||
|
||||
type LookerTargetStatus = 'mapped' | 'unmapped' | 'unparseable' | 'missing_target_table';
|
||||
|
||||
export interface LookerSlFieldProposal {
|
||||
interface LookerSlFieldProposal {
|
||||
name: string;
|
||||
lookerField: string;
|
||||
}
|
||||
|
||||
export interface LookerSlMeasureProposal extends LookerSlFieldProposal {
|
||||
interface LookerSlMeasureProposal extends LookerSlFieldProposal {
|
||||
expr: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
export interface LookerSlSegmentProposal {
|
||||
interface LookerSlSegmentProposal {
|
||||
name: string;
|
||||
filters: Record<string, unknown>;
|
||||
suggestedPredicate: string;
|
||||
|
|
@ -89,6 +89,7 @@ function targetNotes(status: LookerTargetStatus, targetTable: ParsedTargetTable
|
|||
];
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function buildLookerSlProposal(raw: LookerQueryToSlInput): LookerSlProposal {
|
||||
const input = lookerQueryToSlInputSchema.parse(raw);
|
||||
const sourceName = `looker__${toSlName(input.query.model)}__${toSlName(input.query.view)}`;
|
||||
|
|
@ -182,7 +183,7 @@ export function createLookerQueryToSlTool() {
|
|||
});
|
||||
}
|
||||
|
||||
export function formatLookerSlProposal(proposal: LookerSlProposal): string {
|
||||
function formatLookerSlProposal(proposal: LookerSlProposal): string {
|
||||
const lines = [
|
||||
'## Looker query SL proposal',
|
||||
'',
|
||||
|
|
|
|||
|
|
@ -5,9 +5,9 @@ import { parsedTargetTableSchema } from '../../parsed-target-table.js';
|
|||
const lookerIdSchema = z.union([z.string(), z.number().int()]).transform(String);
|
||||
const nullableLookerIdSchema = z.union([lookerIdSchema, z.null()]).default(null);
|
||||
|
||||
export const lookerConnectionIdSchema = z.string().min(1).regex(/^[A-Za-z0-9_-]+$/);
|
||||
const lookerConnectionIdSchema = z.string().min(1).regex(/^[A-Za-z0-9_-]+$/);
|
||||
|
||||
export const lookerRuntimeCursorsSchema = z.object({
|
||||
const lookerRuntimeCursorsSchema = z.object({
|
||||
dashboardsLastSyncedAt: z.iso.datetime().nullable().default(null),
|
||||
looksLastSyncedAt: z.iso.datetime().nullable().default(null),
|
||||
});
|
||||
|
|
@ -215,6 +215,7 @@ const stagedLookerFetchIssueKindSchema = z.enum([
|
|||
'lookml_connection_mismatch',
|
||||
]);
|
||||
|
||||
/** @internal */
|
||||
export const stagedLookerFetchIssueSchema = z.object({
|
||||
rawPath: z.string().min(1),
|
||||
entityType: z.enum(['dashboard', 'look', 'explore', 'signals', 'lookml_models', 'looker_connection_mapping']),
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ export interface MetabaseCardSummary {
|
|||
collection_id?: number | 'root' | null;
|
||||
}
|
||||
|
||||
export interface MetabaseResultMetadataColumn {
|
||||
interface MetabaseResultMetadataColumn {
|
||||
name: string;
|
||||
base_type: string;
|
||||
semantic_type?: string | null;
|
||||
|
|
@ -79,7 +79,7 @@ export interface MetabaseResultMetadataColumn {
|
|||
field_ref?: unknown[] | null;
|
||||
}
|
||||
|
||||
export interface MetabaseParameter {
|
||||
interface MetabaseParameter {
|
||||
id: string;
|
||||
name: string;
|
||||
type: string;
|
||||
|
|
@ -103,7 +103,7 @@ export interface MetabaseTemplateTag {
|
|||
'widget-type'?: string;
|
||||
}
|
||||
|
||||
export interface MetabaseResolvedTemplateTag {
|
||||
interface MetabaseResolvedTemplateTag {
|
||||
name: string;
|
||||
type: string;
|
||||
cardReference?: number | null;
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ class MetabaseApiError extends Error {
|
|||
* `[[`/`]]` literals in string values or regex predicates are preserved. Metabase's
|
||||
* grammar disallows nested optional blocks (per docs), so non-greedy matching is safe.
|
||||
*/
|
||||
/** @internal */
|
||||
export function stripOptionalClauses(sql: string): string {
|
||||
return sql.replace(/\[\[[\s\S]*?\]\]/g, (match) => (match.includes('{{') ? '' : match));
|
||||
}
|
||||
|
|
@ -163,6 +164,7 @@ function injectNativeSql(datasetQuery: MetabaseDatasetQuery, sql: string): Metab
|
|||
* format, number widgets need a string scalar, identifier/enum widgets accept `[string]`.
|
||||
* Sending `['placeholder']` for a date widget triggers a ClassCastException → HTTP 500.
|
||||
*/
|
||||
/** @internal */
|
||||
export function getDummyValueForWidgetType(widgetType: string | undefined): string | string[] {
|
||||
switch (widgetType) {
|
||||
case 'date/range':
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { parseMetabasePullConfig, type MetabasePullConfig } from './types.js';
|
||||
|
||||
export interface MetabaseFanoutMappingInput {
|
||||
interface MetabaseFanoutMappingInput {
|
||||
metabaseDatabaseId: number;
|
||||
targetConnectionId: string | null;
|
||||
syncEnabled: boolean;
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import type { KtxProjectConnectionConfig } from '../../../project/index.js';
|
||||
import type { KtxProjectConnectionConfig } from '../../../../context/project/config.js';
|
||||
import { metabaseRuntimeConfigFromLocalConnection } from './local-metabase.adapter.js';
|
||||
|
||||
describe('metabaseRuntimeConfigFromLocalConnection', () => {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import type { KtxLocalProject, KtxProjectConnectionConfig } from '../../../project/index.js';
|
||||
import { ktxLocalStateDbPath } from '../../../project/index.js';
|
||||
import type { KtxLocalProject } from '../../../../context/project/project.js';
|
||||
import type { KtxProjectConnectionConfig } from '../../../../context/project/config.js';
|
||||
import { ktxLocalStateDbPath } from '../../../../context/project/local-state-db.js';
|
||||
import { resolveKtxConfigReference } from '../../../core/config-reference.js';
|
||||
import {
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { buildDefaultKtxProjectConfig } from '../../../project/index.js';
|
||||
import { buildDefaultKtxProjectConfig } from '../../../../context/project/config.js';
|
||||
import { connectionConfigSchema } from '../../../project/driver-schemas.js';
|
||||
import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './local-source-state-store.js';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,15 +1,12 @@
|
|||
import { mkdirSync } from 'node:fs';
|
||||
import { dirname } from 'node:path';
|
||||
import Database from 'better-sqlite3';
|
||||
import {
|
||||
parseMetabaseMappingBootstrap,
|
||||
type KtxLocalProject,
|
||||
type MetabaseMappingBootstrap,
|
||||
} from '../../../project/index.js';
|
||||
import { parseMetabaseMappingBootstrap, type MetabaseMappingBootstrap } from '../../../../context/project/mappings-yaml-schema.js';
|
||||
import type { KtxLocalProject } from '../../../../context/project/project.js';
|
||||
import type { DiscoveredMetabaseDatabase } from './mapping.js';
|
||||
import type { MetabaseSourceState, MetabaseSourceStateReader, MetabaseSourceStateSelection } from './source-state-port.js';
|
||||
|
||||
export type LocalMetabaseMappingSource = 'ktx.yaml' | 'refresh';
|
||||
type LocalMetabaseMappingSource = 'ktx.yaml' | 'refresh';
|
||||
|
||||
interface LocalMetabaseDiscoveryCacheOptions {
|
||||
dbPath: string;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import type { MetabaseDatabase, MetabaseRuntimeClient } from './client-port.js';
|
||||
|
||||
/** @internal */
|
||||
export const METABASE_ENGINE_TO_CONNECTION_TYPE = {
|
||||
postgres: 'POSTGRESQL',
|
||||
bigquery: 'BIGQUERY',
|
||||
|
|
@ -9,8 +10,6 @@ export const METABASE_ENGINE_TO_CONNECTION_TYPE = {
|
|||
mysql: 'MYSQL',
|
||||
} as const;
|
||||
|
||||
export type MetabaseMappedConnectionType =
|
||||
(typeof METABASE_ENGINE_TO_CONNECTION_TYPE)[keyof typeof METABASE_ENGINE_TO_CONNECTION_TYPE];
|
||||
|
||||
export interface DiscoveredMetabaseDatabase {
|
||||
id: number;
|
||||
|
|
@ -42,26 +41,31 @@ export interface KtxConnectionPhysicalInfo {
|
|||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface PhysicalMismatchInput {
|
||||
mappingId: string;
|
||||
metabase: MappingPhysicalInfo;
|
||||
target: KtxConnectionPhysicalInfo;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface PhysicalMismatch {
|
||||
mappingId: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface MappingRefreshReport {
|
||||
drift: MetabaseMappingDrift;
|
||||
physicalMismatches: PhysicalMismatch[];
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export type MetabaseMappingValidationResult =
|
||||
| { ok: true }
|
||||
| { ok: false; errors: Array<{ key: string; reason: string }> };
|
||||
|
||||
/** @internal */
|
||||
export interface AutoMatchCandidate {
|
||||
id: string;
|
||||
name: string;
|
||||
|
|
@ -69,6 +73,7 @@ export interface AutoMatchCandidate {
|
|||
connection_params: unknown;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface AutoMatchResult {
|
||||
connectionId: string;
|
||||
connectionName: string;
|
||||
|
|
@ -170,6 +175,7 @@ export function computeMetabaseMappingDrift(args: {
|
|||
return { unmappedDiscovered, staleMappings, inSync };
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function validateMetabaseMappings(args: {
|
||||
mappings: Record<string, string | null | undefined>;
|
||||
knownKtxConnectionIds: Set<string>;
|
||||
|
|
@ -236,6 +242,7 @@ export function validateMappingPhysicalMatch(
|
|||
return null;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function computeMetabaseMappingPhysicalMismatches(inputs: PhysicalMismatchInput[]): PhysicalMismatch[] {
|
||||
const mismatches: PhysicalMismatch[] = [];
|
||||
for (const input of inputs) {
|
||||
|
|
@ -247,6 +254,7 @@ export function computeMetabaseMappingPhysicalMismatches(inputs: PhysicalMismatc
|
|||
return mismatches;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export async function refreshMetabaseMapping(args: {
|
||||
client: Pick<MetabaseRuntimeClient, 'getDatabases'>;
|
||||
currentMappings: Record<string, string | null | undefined>;
|
||||
|
|
@ -286,6 +294,7 @@ export async function refreshMetabaseMapping(args: {
|
|||
return { drift, physicalMismatches };
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function findBestMatch(mapping: MappingPhysicalInfo, candidates: AutoMatchCandidate[]): AutoMatchResult | null {
|
||||
const engine = mapping.metabaseEngine?.toLowerCase();
|
||||
if (!engine) {
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ export interface MetabaseSourceStateSelection {
|
|||
metabaseObjectId: number;
|
||||
}
|
||||
|
||||
export interface MetabaseSourceStateMapping {
|
||||
interface MetabaseSourceStateMapping {
|
||||
metabaseDatabaseId: number;
|
||||
metabaseDatabaseName: string | null;
|
||||
metabaseEngine: string | null;
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { z } from 'zod';
|
|||
const metabaseSyncModeSchema = z.enum(['ALL', 'ONLY', 'EXCEPT']);
|
||||
export type MetabaseSyncMode = z.infer<typeof metabaseSyncModeSchema>;
|
||||
|
||||
export const metabaseLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/);
|
||||
const metabaseLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/);
|
||||
|
||||
/**
|
||||
* The lean config the adapter needs at `fetch()` time. Lives in the BullMQ payload's
|
||||
|
|
@ -11,6 +11,7 @@ export const metabaseLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a-
|
|||
* job — the persisted state (enabled/disabled, auth, scheduling) lives on the
|
||||
* Metabase connection's `connections.config` JSONB.
|
||||
*/
|
||||
/** @internal */
|
||||
export const metabasePullConfigSchema = z.object({
|
||||
/** The Metabase connection (source) — the thing being swept. */
|
||||
metabaseConnectionId: metabaseLocalConnectionIdSchema,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { parse as parseYaml } from 'yaml';
|
||||
import { noopLogger, type KtxLogger } from '../../../core/index.js';
|
||||
import { noopLogger, type KtxLogger } from '../../../../context/core/config.js';
|
||||
|
||||
export interface DimensionDefinition {
|
||||
interface DimensionDefinition {
|
||||
name: string;
|
||||
column: string;
|
||||
type: string;
|
||||
|
|
@ -9,7 +9,7 @@ export interface DimensionDefinition {
|
|||
description?: string;
|
||||
}
|
||||
|
||||
export interface SimpleMeasureDefinition {
|
||||
interface SimpleMeasureDefinition {
|
||||
type: 'simple';
|
||||
name: string;
|
||||
column: string;
|
||||
|
|
@ -20,7 +20,7 @@ export interface SimpleMeasureDefinition {
|
|||
cumulative?: boolean;
|
||||
}
|
||||
|
||||
export type MeasureDefinition =
|
||||
type MeasureDefinition =
|
||||
| SimpleMeasureDefinition
|
||||
| {
|
||||
type: 'derived';
|
||||
|
|
@ -186,6 +186,7 @@ export function parseMetricflowFiles(
|
|||
return parser.parseFiles(files);
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function translateMetricflowJinjaFilter(filter: string): string {
|
||||
return new MetricflowDeepParser(noopLogger).translateJinjaFilter(filter);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,6 @@
|
|||
import type { SemanticLayerService, SemanticLayerSource } from '../../../sl/index.js';
|
||||
import {
|
||||
addTouchedSlSource,
|
||||
createTouchedSlSources,
|
||||
listTouchedSlSources,
|
||||
type TouchedSlSource,
|
||||
} from '../../../tools/index.js';
|
||||
import type { SemanticLayerService } from '../../../../context/sl/semantic-layer.service.js';
|
||||
import type { SemanticLayerSource } from '../../../../context/sl/types.js';
|
||||
import { addTouchedSlSource, createTouchedSlSources, listTouchedSlSources, type TouchedSlSource } from '../../../../context/tools/touched-sl-sources.js';
|
||||
import type { MetricFlowParseResult } from './deep-parse.js';
|
||||
import {
|
||||
buildMetricflowJoinsForModel,
|
||||
|
|
@ -29,12 +25,12 @@ export interface MetricFlowImportResult {
|
|||
touchedSources: TouchedSlSource[];
|
||||
}
|
||||
|
||||
export type MetricflowSemanticLayerWriter = Pick<
|
||||
type MetricflowSemanticLayerWriter = Pick<
|
||||
SemanticLayerService,
|
||||
'getManifestEntry' | 'isManifestBacked' | 'loadAllSources' | 'loadSource' | 'writeSource'
|
||||
>;
|
||||
|
||||
export type MetricflowSemanticLayerService = MetricflowSemanticLayerWriter & {
|
||||
type MetricflowSemanticLayerService = MetricflowSemanticLayerWriter & {
|
||||
forWorktree(workdir: string): MetricflowSemanticLayerWriter;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { z } from 'zod';
|
||||
import { parsedTargetTableSchema } from '../../parsed-target-table.js';
|
||||
|
||||
export const metricflowPullConfigSchema = z.object({
|
||||
const metricflowPullConfigSchema = z.object({
|
||||
repoUrl: z.string().url(),
|
||||
branch: z.string().default('main'),
|
||||
path: z.string().nullable().default(null),
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { composeOverlay, type SemanticLayerSource } from '../../../sl/index.js';
|
||||
import { composeOverlay } from '../../../../context/sl/semantic-layer.service.js';
|
||||
import type { SemanticLayerSource } from '../../../../context/sl/types.js';
|
||||
import type { ParsedCrossModelMetric, ParsedMetricflowRelationship, ParsedSemanticModel } from './deep-parse.js';
|
||||
import {
|
||||
buildMetricflowColumns,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { SemanticLayerSource } from '../../../sl/index.js';
|
||||
import type { SemanticLayerSource } from '../../../../context/sl/types.js';
|
||||
import type {
|
||||
ParsedCrossModelMetric,
|
||||
ParsedMetricflowRelationship,
|
||||
|
|
@ -25,6 +25,7 @@ export type MetricflowSemanticModelJoin = SemanticLayerSource['joins'][number];
|
|||
export type MetricflowWritableSemanticLayerSource = Pick<SemanticLayerSource, 'name'> &
|
||||
Partial<Omit<SemanticLayerSource, 'name'>>;
|
||||
|
||||
/** @internal */
|
||||
export function toKebabCaseMetricflowName(str: string): string {
|
||||
return str
|
||||
.toLowerCase()
|
||||
|
|
@ -32,6 +33,7 @@ export function toKebabCaseMetricflowName(str: string): string {
|
|||
.replace(/^-|-$/g, '');
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function mapSemanticModelToSource(model: ParsedSemanticModel, tableRef?: string): SemanticLayerSource {
|
||||
return {
|
||||
name: toKebabCaseMetricflowName(model.modelRef),
|
||||
|
|
@ -177,7 +179,7 @@ export function buildMetricflowSemanticModelSource(
|
|||
return mapMetricflowSemanticModelToStandalone(model, sourceName, matchedTable?.name ?? model.modelRef, joins);
|
||||
}
|
||||
|
||||
export function buildMetricflowMeasures(model: ParsedSemanticModel): SemanticLayerSource['measures'] {
|
||||
function buildMetricflowMeasures(model: ParsedSemanticModel): SemanticLayerSource['measures'] {
|
||||
return model.measures.map((measure) => {
|
||||
if (measure.type === 'simple') {
|
||||
return {
|
||||
|
|
@ -195,6 +197,7 @@ export function buildMetricflowMeasures(model: ParsedSemanticModel): SemanticLay
|
|||
});
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function buildMetricflowColumns(model: ParsedSemanticModel): SemanticLayerSource['columns'] {
|
||||
const columns: SemanticLayerSource['columns'] = model.dimensions.map((dimension) => ({
|
||||
name: dimension.column,
|
||||
|
|
@ -243,6 +246,7 @@ export function getMetricflowAvailableColumnNames(context: MetricflowSemanticMod
|
|||
return new Set(columns.map((column) => column.name.toLowerCase()));
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function countImportableMetricflowRelationships(
|
||||
relationships: ParsedMetricflowRelationship[],
|
||||
hostTables: MetricflowHostTable[],
|
||||
|
|
@ -336,10 +340,11 @@ function mergeMetricflowJoins(
|
|||
return [...baseJoins, ...newJoins];
|
||||
}
|
||||
|
||||
export function normalizeMetricflowJoinOn(on: string): string {
|
||||
function normalizeMetricflowJoinOn(on: string): string {
|
||||
return on.replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function rewriteMetricflowManifestJoins(
|
||||
joins: SemanticLayerSource['joins'],
|
||||
sourceNameByManifestName: Map<string, string>,
|
||||
|
|
@ -351,7 +356,7 @@ export function rewriteMetricflowManifestJoins(
|
|||
}));
|
||||
}
|
||||
|
||||
export function rewriteMetricflowJoinOn(on: string, sourceNameByManifestName: Map<string, string>): string {
|
||||
function rewriteMetricflowJoinOn(on: string, sourceNameByManifestName: Map<string, string>): string {
|
||||
const parts = on.split('=');
|
||||
if (parts.length !== 2) {
|
||||
return on;
|
||||
|
|
@ -366,7 +371,7 @@ export function rewriteMetricflowJoinOn(on: string, sourceNameByManifestName: Ma
|
|||
return `${leftTable}.${left.column} = ${rightTable}.${right.column}`;
|
||||
}
|
||||
|
||||
export function parseMetricflowJoinReference(ref: string): { table: string; column: string } | null {
|
||||
function parseMetricflowJoinReference(ref: string): { table: string; column: string } | null {
|
||||
const lastDot = ref.lastIndexOf('.');
|
||||
if (lastDot <= 0 || lastDot === ref.length - 1) {
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import type { ChunkResult, DiffSet, ScopeDescriptor, WorkUnit } from '../../type
|
|||
import { notionManifestSchema, notionMetadataSchema } from './types.js';
|
||||
|
||||
const MAX_NOTION_WORK_UNIT_CHARS = 40_000;
|
||||
/** @internal */
|
||||
export const NOTION_ORG_KNOWLEDGE_WARNING =
|
||||
'Anything accessible to this Notion integration can become organization knowledge.';
|
||||
const NOTION_SL_WRITE_GUIDANCE =
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import type { SemanticLayerService } from '../sl/index.js';
|
||||
import type { TouchedSlSource } from '../tools/index.js';
|
||||
import type { KnowledgeWikiService } from '../wiki/index.js';
|
||||
import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js';
|
||||
import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js';
|
||||
import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js';
|
||||
import { findMissingWikiRefs } from '../wiki/wiki-ref-validation.js';
|
||||
import { findInvalidWikiBodyRefs } from './wiki-body-refs.js';
|
||||
|
||||
export interface TouchedValidationResult {
|
||||
interface TouchedValidationResult {
|
||||
invalidSources: string[];
|
||||
validSources: string[];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { type KtxLogger, noopLogger } from '../../core/index.js';
|
||||
import { type KtxLogger, noopLogger } from '../../../context/core/config.js';
|
||||
import type { CandidateDedupResult, ContextCandidateForDedup, JsonValue } from '../ports.js';
|
||||
import { buildContextCandidateEmbeddingText } from './embedding-text.js';
|
||||
import type { ContextCandidateStorePort } from './store.js';
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { type KtxLogger, noopLogger } from '../../core/index.js';
|
||||
import { type KtxLogger, noopLogger } from '../../../context/core/config.js';
|
||||
import type { JsonValue } from '../ports.js';
|
||||
import type { ContextCandidateStorePort } from './store.js';
|
||||
import type {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import type { KtxModelRole } from '../../../llm/index.js';
|
||||
import { type KtxLogger, noopLogger } from '../../core/index.js';
|
||||
import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../llm/index.js';
|
||||
import type { MemoryAction } from '../../memory/index.js';
|
||||
import type { KtxModelRole } from '../../../llm/types.js';
|
||||
import { type KtxLogger, noopLogger } from '../../../context/core/config.js';
|
||||
import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js';
|
||||
import type { MemoryAction } from '../../../context/memory/types.js';
|
||||
import type { ContextCandidateForDedup, CuratorPaginationPort, CuratorPaginationReport } from '../ports.js';
|
||||
import type {
|
||||
ReconcileCandidateForPrompt,
|
||||
|
|
|
|||
|
|
@ -1,29 +0,0 @@
|
|||
export type { CandidateDedupServiceDeps } from './candidate-dedup.service.js';
|
||||
export { CandidateDedupService } from './candidate-dedup.service.js';
|
||||
export type {
|
||||
ContextCandidateCarryforwardArgs,
|
||||
ContextCandidateCarryforwardResult,
|
||||
ContextCandidateCarryforwardServiceDeps,
|
||||
} from './context-candidate-carryforward.service.js';
|
||||
export { ContextCandidateCarryforwardService } from './context-candidate-carryforward.service.js';
|
||||
export type { CuratorPaginationInput, CuratorPaginationServiceDeps } from './curator-pagination.service.js';
|
||||
export { CuratorPaginationService } from './curator-pagination.service.js';
|
||||
export { buildContextCandidateEmbeddingText } from './embedding-text.js';
|
||||
export type { ContextCandidateStorePort } from './store.js';
|
||||
export type {
|
||||
BudgetExhaustedCandidateForCarryForward,
|
||||
CandidateDedupSettings,
|
||||
ContextCandidateActionHint,
|
||||
ContextCandidateCarryforwardSettings,
|
||||
ContextCandidateEmbeddingPort,
|
||||
ContextCandidateForPrompt,
|
||||
ContextCandidateLane,
|
||||
ContextCandidateRejectionReason,
|
||||
ContextCandidateScoreAggregation,
|
||||
ContextCandidateStatus,
|
||||
ContextCandidateVerdictSummary,
|
||||
CuratorPaginationSettings,
|
||||
CurrentRunEvidenceChunkForCarryForward,
|
||||
InsertContextCandidateInput,
|
||||
MarkContextCandidateClusterInput,
|
||||
} from './types.js';
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import type { JsonValue } from '../ports.js';
|
||||
|
||||
export type ContextCandidateActionHint = 'create' | 'update' | 'merge' | 'conflict' | 'skip';
|
||||
export type ContextCandidateStatus = 'pending' | 'promoted' | 'merged' | 'rejected' | 'conflict';
|
||||
type ContextCandidateActionHint = 'create' | 'update' | 'merge' | 'conflict' | 'skip';
|
||||
type ContextCandidateStatus = 'pending' | 'promoted' | 'merged' | 'rejected' | 'conflict';
|
||||
export type ContextCandidateRejectionReason =
|
||||
| 'low_score'
|
||||
| 'duplicates_existing_wiki'
|
||||
|
|
@ -10,20 +10,9 @@ export type ContextCandidateRejectionReason =
|
|||
| 'exceeded_run_budget'
|
||||
| 'exceeded_curator_passes'
|
||||
| 'curator_pass_error';
|
||||
export type ContextCandidateLane = 'light' | 'full' | null;
|
||||
export type ContextCandidateScoreAggregation = 'max' | 'mean' | 'sum';
|
||||
type ContextCandidateLane = 'light' | 'full' | null;
|
||||
type ContextCandidateScoreAggregation = 'max' | 'mean' | 'sum';
|
||||
|
||||
export interface ContextCandidateForPrompt {
|
||||
candidateKey: string;
|
||||
topic: string;
|
||||
assertion: string;
|
||||
rationale: string;
|
||||
actionHint: string;
|
||||
status: string;
|
||||
promotionScore: number;
|
||||
suggestedPageKey: string | null;
|
||||
evidenceRefs: JsonValue;
|
||||
}
|
||||
|
||||
export interface ContextCandidateVerdictSummary {
|
||||
pending: number;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { readdir, readFile } from 'node:fs/promises';
|
||||
import { basename, dirname, join, relative } from 'node:path';
|
||||
import { noopLogger, type KtxLogger } from '../../core/index.js';
|
||||
import { noopLogger, type KtxLogger } from '../../../context/core/config.js';
|
||||
import type { JsonValue } from '../ports.js';
|
||||
import type { DiffSet } from '../types.js';
|
||||
import type { ContextEvidenceIndexStorePort } from './store.js';
|
||||
|
|
|
|||
|
|
@ -1,12 +0,0 @@
|
|||
export { ContextEvidenceIndexService } from './context-evidence-index.service.js';
|
||||
export { SqliteContextEvidenceStore } from './sqlite-context-evidence-store.js';
|
||||
export type {
|
||||
ContextEvidenceDocumentRef,
|
||||
ContextEvidenceEmbeddingPort,
|
||||
ContextEvidenceIndexSummary,
|
||||
EvidencePublishState,
|
||||
ReplaceContextEvidenceChunk,
|
||||
UpsertContextEvidenceDocument,
|
||||
} from './types.js';
|
||||
export type { ContextEvidenceIndexStorePort } from './store.js';
|
||||
export type { SqliteContextEvidenceStoreOptions } from './sqlite-context-evidence-store.js';
|
||||
|
|
@ -2,7 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import type { InsertContextCandidateInput } from '../context-candidates/index.js';
|
||||
import type { InsertContextCandidateInput } from '../../../context/ingest/context-candidates/types.js';
|
||||
import type { JsonValue } from '../ports.js';
|
||||
import { SqliteContextEvidenceStore } from './sqlite-context-evidence-store.js';
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@ import { randomUUID } from 'node:crypto';
|
|||
import { mkdirSync } from 'node:fs';
|
||||
import { dirname } from 'node:path';
|
||||
import Database from 'better-sqlite3';
|
||||
import { HybridSearchCore, type SearchCandidateGenerator, type SearchLaneBreakdown } from '../../search/index.js';
|
||||
import { HybridSearchCore } from '../../../context/search/hybrid-search-core.js';
|
||||
import type { SearchCandidateGenerator, SearchLaneBreakdown } from '../../../context/search/types.js';
|
||||
import type {
|
||||
ContextCandidateStatusResult,
|
||||
ContextEvidenceChunkForCandidate,
|
||||
|
|
@ -14,16 +15,9 @@ import type {
|
|||
ContextEvidenceSearchResult,
|
||||
ContextEvidenceToolStorePort,
|
||||
} from '../../tools/context-evidence-tool-store.js';
|
||||
import type {
|
||||
BudgetExhaustedCandidateForCarryForward,
|
||||
ContextCandidateRejectionReason,
|
||||
ContextCandidateStorePort,
|
||||
ContextCandidateVerdictSummary,
|
||||
CurrentRunEvidenceChunkForCarryForward,
|
||||
InsertContextCandidateInput,
|
||||
MarkContextCandidateClusterInput,
|
||||
} from '../context-candidates/index.js';
|
||||
import type { PageTriageEvidenceChunk, PageTriageStorePort } from '../page-triage/index.js';
|
||||
import type { BudgetExhaustedCandidateForCarryForward, ContextCandidateRejectionReason, ContextCandidateVerdictSummary, CurrentRunEvidenceChunkForCarryForward, InsertContextCandidateInput, MarkContextCandidateClusterInput } from '../../../context/ingest/context-candidates/types.js';
|
||||
import type { ContextCandidateStorePort } from '../../../context/ingest/context-candidates/store.js';
|
||||
import type { PageTriageEvidenceChunk, PageTriageStorePort } from '../../../context/ingest/page-triage/page-triage.service.js';
|
||||
import type { ContextCandidateForDedup, ContextCandidateSummary, JsonValue } from '../ports.js';
|
||||
import type { ContextEvidenceIndexStorePort } from './store.js';
|
||||
import type {
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ export interface ResolveJinjaVariablesResult {
|
|||
unresolvedVars: string[];
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function parseProjectVars(yamlContent: string): Map<string, string> {
|
||||
const variables = new Map<string, string>();
|
||||
const project = parseProjectYaml(yamlContent);
|
||||
|
|
@ -30,6 +31,7 @@ export function parseProjectVars(yamlContent: string): Map<string, string> {
|
|||
return variables;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function parseProjectName(yamlContent: string): string | null {
|
||||
const project = parseProjectYaml(yamlContent);
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ export async function loadDbtSchemaFiles(projectDir: string): Promise<DbtSchemaF
|
|||
);
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export async function findDbtSchemaFiles(projectDir: string): Promise<string[]> {
|
||||
const schemaFiles: string[] = [];
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { z } from 'zod';
|
||||
import type { AgentRunnerPort, KtxRuntimeToolSet } from '../llm/index.js';
|
||||
import type { TouchedSlSource } from '../tools/index.js';
|
||||
import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../context/llm/runtime-port.js';
|
||||
import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js';
|
||||
import type { IngestTraceWriter } from './ingest-trace.js';
|
||||
import { traceTimed } from './ingest-trace.js';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import type { SemanticLayerSource } from '../sl/index.js';
|
||||
import type { TouchedSlSource } from '../tools/index.js';
|
||||
import type { SemanticLayerSource } from '../../context/sl/types.js';
|
||||
import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js';
|
||||
import type { IngestReportFinalizationMismatch } from './reports.js';
|
||||
|
||||
interface DeriveTouchedSourcesInput {
|
||||
|
|
|
|||
|
|
@ -1,670 +0,0 @@
|
|||
export { DbtSourceAdapter } from './adapters/dbt/dbt.adapter.js';
|
||||
export { FakeSourceAdapter } from './adapters/fake/fake.adapter.js';
|
||||
export type {
|
||||
DaemonLiveDatabaseIntrospectionOptions,
|
||||
KtxDaemonDatabaseHttpJsonRunner,
|
||||
KtxDaemonDatabaseIntrospectionCommand,
|
||||
KtxDaemonDatabaseJsonRunner,
|
||||
} from './adapters/live-database/daemon-introspection.js';
|
||||
export { createDaemonLiveDatabaseIntrospection } from './adapters/live-database/daemon-introspection.js';
|
||||
export type {
|
||||
LiveDatabaseExtractedColumn,
|
||||
LiveDatabaseExtractedForeignKey,
|
||||
LiveDatabaseExtractedSchema,
|
||||
LiveDatabaseExtractedTable,
|
||||
} from './adapters/live-database/extracted-schema.js';
|
||||
export {
|
||||
buildLiveDatabaseTableNaturalKey,
|
||||
ktxSchemaSnapshotToExtractedSchema,
|
||||
} from './adapters/live-database/extracted-schema.js';
|
||||
export {
|
||||
assertSemanticLayerTargetPathsAllowed,
|
||||
findDisallowedSemanticLayerTargetPaths,
|
||||
semanticLayerConnectionIdFromPath,
|
||||
} from './semantic-layer-target-policy.js';
|
||||
export { LiveDatabaseSourceAdapter } from './adapters/live-database/live-database.adapter.js';
|
||||
export type {
|
||||
BuildLiveDatabaseManifestShardsInput,
|
||||
BuildLiveDatabaseManifestShardsResult,
|
||||
LiveDatabaseManifestColumn,
|
||||
LiveDatabaseManifestExistingDescriptions,
|
||||
LiveDatabaseManifestJoinData,
|
||||
LiveDatabaseManifestJoinEntry,
|
||||
LiveDatabaseManifestShard,
|
||||
LiveDatabaseManifestTableData,
|
||||
LiveDatabaseManifestTableEntry,
|
||||
} from './adapters/live-database/manifest.js';
|
||||
export { buildLiveDatabaseManifestShards } from './adapters/live-database/manifest.js';
|
||||
export type {
|
||||
LiveDatabaseStructuralChanges,
|
||||
LiveDatabaseStructuralSyncOperations,
|
||||
LiveDatabaseStructuralSyncPlan,
|
||||
LiveDatabaseStructuralSyncStats,
|
||||
LiveDatabaseSyncedColumn,
|
||||
LiveDatabaseSyncedLink,
|
||||
LiveDatabaseSyncedSchema,
|
||||
LiveDatabaseSyncedTable,
|
||||
PlanLiveDatabaseStructuralSyncInput,
|
||||
} from './adapters/live-database/structural-sync.js';
|
||||
export { planLiveDatabaseStructuralSync } from './adapters/live-database/structural-sync.js';
|
||||
export type {
|
||||
LiveDatabaseIntrospectionPort,
|
||||
LiveDatabaseSourceAdapterDeps,
|
||||
} from './adapters/live-database/types.js';
|
||||
export { getLookerTriageSignals, writeLookerEvidenceDocuments } from './adapters/looker/evidence-documents.js';
|
||||
export { LookerClient } from './adapters/looker/client.js';
|
||||
export type {
|
||||
LookerClientDeps,
|
||||
LookerClientLogger,
|
||||
LookerConnectionParams,
|
||||
LookerSdkPort,
|
||||
LookerWarehouseConnectionInfo,
|
||||
TestConnectionResult as LookerTestConnectionResult,
|
||||
} from './adapters/looker/client.js';
|
||||
export type {
|
||||
LookerClientFactory,
|
||||
LookerEntityRef,
|
||||
LookerRuntimeClient,
|
||||
} from './adapters/looker/fetch.js';
|
||||
export {
|
||||
DefaultLookerClientFactory,
|
||||
DefaultLookerConnectionClientFactory,
|
||||
} from './adapters/looker/factory.js';
|
||||
export {
|
||||
createDaemonLookerTableIdentifierParser,
|
||||
type DaemonLookerTableIdentifierParserOptions,
|
||||
type KtxDaemonTableIdentifierHttpJsonRunner,
|
||||
} from './adapters/looker/daemon-table-identifier-parser.js';
|
||||
export type {
|
||||
LookerConnectionClientFactory,
|
||||
LookerCredentialResolver,
|
||||
} from './adapters/looker/factory.js';
|
||||
export {
|
||||
createLocalLookerCredentialResolver,
|
||||
createLocalLookerSourceAdapter,
|
||||
lookerCredentialsFromLocalConnection,
|
||||
} from './adapters/looker/local-looker.adapter.js';
|
||||
export {
|
||||
LocalLookerRuntimeStore,
|
||||
type ClearLocalLookerMappingsInput,
|
||||
type LocalLookerConnectionMappingListRow,
|
||||
type LocalLookerMappingSource,
|
||||
type LookerSourceStateReader,
|
||||
type RefreshLocalLookerDiscoveredConnectionsInput,
|
||||
type UpsertLocalLookerConnectionMappingInput,
|
||||
} from './adapters/looker/local-runtime-store.js';
|
||||
export {
|
||||
LOOKER_DIALECT_TO_CONNECTION_TYPE,
|
||||
buildLookerPullConfigFromInputs,
|
||||
collectExploreParseItems,
|
||||
computeLookerMappingDrift,
|
||||
discoverLookerConnections,
|
||||
extractWarehouseDatabase,
|
||||
extractWarehouseHost,
|
||||
lookerDialectToConnectionType,
|
||||
normalizeHost,
|
||||
normalizeName,
|
||||
projectParsedIdentifier,
|
||||
refreshLookerMappingPlaceholders,
|
||||
sqlglotDialectForConnectionType,
|
||||
suggestKtxConnectionForLookerConnection,
|
||||
validateLookerMappings,
|
||||
validateLookerWarehouseTarget,
|
||||
} from './adapters/looker/mapping.js';
|
||||
export type {
|
||||
LookerConnectionMapping as KtxLookerConnectionMapping,
|
||||
LookerMappingCandidateConnection,
|
||||
LookerMappingClient,
|
||||
LookerMappingDrift,
|
||||
LookerMappingValidationResult,
|
||||
LookerParsedIdentifier,
|
||||
LookerTableIdentifierParseItem,
|
||||
LookerTableIdentifierParser,
|
||||
LookerTargetConnection,
|
||||
LookerWarehouseTargetConnectionType,
|
||||
} from './adapters/looker/mapping.js';
|
||||
export {
|
||||
readLookerFetchReport,
|
||||
writeLookerFetchReport,
|
||||
} from './adapters/looker/fetch-report.js';
|
||||
export { LookerSourceAdapter, type LookerSourceAdapterDeps } from './adapters/looker/looker.adapter.js';
|
||||
export {
|
||||
describeLookerScope,
|
||||
hashLookerScope,
|
||||
isPathInLookerScope,
|
||||
readLookerScope,
|
||||
} from './adapters/looker/scope.js';
|
||||
export type {
|
||||
LookerQueryToSlInput,
|
||||
LookerSlFieldProposal,
|
||||
LookerSlMeasureProposal,
|
||||
LookerSlProposal,
|
||||
LookerSlSegmentProposal,
|
||||
} from './adapters/looker/tools/looker-query-to-sl.tool.js';
|
||||
export {
|
||||
buildLookerSlProposal,
|
||||
createLookerQueryToSlTool,
|
||||
formatLookerSlProposal,
|
||||
lookerQueryToSlInputSchema,
|
||||
} from './adapters/looker/tools/looker-query-to-sl.tool.js';
|
||||
export type {
|
||||
LookerPullConfig,
|
||||
LookerRuntimeCursors,
|
||||
StagedDashboardFile,
|
||||
StagedExploreFile,
|
||||
StagedFoldersTreeFile,
|
||||
StagedGroupFile,
|
||||
StagedLookerFetchIssue,
|
||||
StagedLookerFetchReport,
|
||||
StagedLookerQuery,
|
||||
StagedLookerScopeFile,
|
||||
StagedLookerSignalsFile,
|
||||
StagedLookFile,
|
||||
StagedLookmlModelsFile,
|
||||
StagedUserFile,
|
||||
} from './adapters/looker/types.js';
|
||||
export {
|
||||
lookerConnectionIdSchema,
|
||||
lookerRuntimeCursorsSchema,
|
||||
stagedLookerFetchIssueSchema,
|
||||
stagedLookerFetchReportSchema,
|
||||
stagedLookerScopeFileSchema,
|
||||
stagedSyncConfigSchema,
|
||||
} from './adapters/looker/types.js';
|
||||
export { LookmlSourceAdapter } from './adapters/lookml/lookml.adapter.js';
|
||||
export { parseLookmlStagedDir } from './adapters/lookml/parse.js';
|
||||
export type { ParsedLookmlProject } from './adapters/lookml/parse.js';
|
||||
export {
|
||||
DEFAULT_METABASE_CLIENT_CONFIG,
|
||||
DefaultMetabaseConnectionClientFactory,
|
||||
MetabaseClient,
|
||||
getDummyValueForWidgetType,
|
||||
stripOptionalClauses,
|
||||
} from './adapters/metabase/client.js';
|
||||
export { CardReferenceCycleError, expandCardReferences } from './adapters/metabase/card-references.js';
|
||||
export { IngestMetabaseClientFactory } from './adapters/metabase/client-port.js';
|
||||
export type { MetabaseClientLogger } from './adapters/metabase/client.js';
|
||||
export type {
|
||||
MetabaseCard,
|
||||
MetabaseCardSummary,
|
||||
MetabaseClientConfig,
|
||||
MetabaseClientFactory,
|
||||
MetabaseClientRuntimeConfig,
|
||||
MetabaseCollection,
|
||||
MetabaseCollectionItem,
|
||||
MetabaseConnectionClientFactory,
|
||||
MetabaseDatabase,
|
||||
MetabaseDatasetQuery,
|
||||
MetabaseNativeQueryResult,
|
||||
MetabaseParameter,
|
||||
MetabaseResolvedTemplateTag,
|
||||
MetabaseResultMetadataColumn,
|
||||
MetabaseRuntimeClient,
|
||||
MetabaseTemplateTag,
|
||||
MetabaseUser,
|
||||
ResolvedSqlResult,
|
||||
TestConnectionResult,
|
||||
} from './adapters/metabase/client-port.js';
|
||||
export type {
|
||||
MetabaseSourceState,
|
||||
MetabaseSourceStateMapping,
|
||||
MetabaseSourceStateReader,
|
||||
MetabaseSourceStateSelection,
|
||||
} from './adapters/metabase/source-state-port.js';
|
||||
export {
|
||||
METABASE_ENGINE_TO_CONNECTION_TYPE,
|
||||
computeMetabaseMappingDrift,
|
||||
computeMetabaseMappingPhysicalMismatches,
|
||||
discoverMetabaseDatabases,
|
||||
findBestMatch,
|
||||
refreshMetabaseMapping,
|
||||
validateMappingPhysicalMatch,
|
||||
validateMetabaseMappings,
|
||||
} from './adapters/metabase/mapping.js';
|
||||
export type {
|
||||
AutoMatchCandidate,
|
||||
AutoMatchResult as MetabaseAutoMatchResult,
|
||||
DiscoveredMetabaseDatabase,
|
||||
KtxConnectionPhysicalInfo,
|
||||
MappingPhysicalInfo,
|
||||
MappingRefreshReport,
|
||||
MetabaseMappedConnectionType,
|
||||
MetabaseMappingDrift,
|
||||
MetabaseMappingValidationResult,
|
||||
PhysicalMismatch,
|
||||
PhysicalMismatchInput,
|
||||
} from './adapters/metabase/mapping.js';
|
||||
export { planMetabaseFanoutChildren } from './adapters/metabase/fanout-planner.js';
|
||||
export type {
|
||||
MetabaseFanoutChildPlan,
|
||||
MetabaseFanoutMappingInput,
|
||||
PlanMetabaseFanoutChildrenInput,
|
||||
} from './adapters/metabase/fanout-planner.js';
|
||||
export { MetabaseSourceAdapter } from './adapters/metabase/metabase.adapter.js';
|
||||
export {
|
||||
createLocalMetabaseSourceAdapter,
|
||||
metabaseRuntimeConfigFromLocalConnection,
|
||||
} from './adapters/metabase/local-metabase.adapter.js';
|
||||
export {
|
||||
KtxYamlMetabaseSourceStateReader,
|
||||
LocalMetabaseDiscoveryCache,
|
||||
} from './adapters/metabase/local-source-state-store.js';
|
||||
export type {
|
||||
LocalMetabaseDiscoveredDatabaseRow,
|
||||
LocalMetabaseMappingListRow,
|
||||
LocalMetabaseMappingSource,
|
||||
RefreshLocalMetabaseDiscoveredDatabasesInput,
|
||||
} from './adapters/metabase/local-source-state-store.js';
|
||||
export { metabaseLocalConnectionIdSchema, metabasePullConfigSchema, parseMetabasePullConfig } from './adapters/metabase/types.js';
|
||||
export type { MetabasePullConfig, MetabaseSyncMode } from './adapters/metabase/types.js';
|
||||
export {
|
||||
fetchMetricflowRepo,
|
||||
} from './adapters/metricflow/fetch.js';
|
||||
export type { FetchMetricflowRepoParams, FetchMetricflowRepoResult } from './adapters/metricflow/fetch.js';
|
||||
export {
|
||||
parseMetricflowFiles,
|
||||
translateMetricflowJinjaFilter,
|
||||
} from './adapters/metricflow/deep-parse.js';
|
||||
export type {
|
||||
DimensionDefinition,
|
||||
MeasureDefinition,
|
||||
MetricFlowParseResult,
|
||||
MetricflowParseOptions,
|
||||
ParsedCrossModelMetric,
|
||||
ParsedMetricflowRelationship,
|
||||
ParsedSemanticModel,
|
||||
SimpleMeasureDefinition,
|
||||
} from './adapters/metricflow/deep-parse.js';
|
||||
export {
|
||||
buildMetricflowColumns,
|
||||
buildMetricflowJoinsForModel,
|
||||
buildMetricflowMeasures,
|
||||
buildMetricflowSemanticModelSource,
|
||||
countImportableMetricflowRelationships,
|
||||
filterValidMetricflowRelationships,
|
||||
findMatchingMetricflowTable,
|
||||
getMetricflowAvailableColumnNames,
|
||||
mapCrossModelMetricToSource,
|
||||
mapSemanticModelToSource,
|
||||
normalizeMetricflowJoinOn,
|
||||
parseMetricflowJoinReference,
|
||||
resolveMetricflowSemanticModelSourceName,
|
||||
rewriteMetricflowJoinOn,
|
||||
rewriteMetricflowManifestJoins,
|
||||
toKebabCaseMetricflowName,
|
||||
} from './adapters/metricflow/semantic-models.js';
|
||||
export { importMetricflowSemanticModels } from './adapters/metricflow/import-semantic-models.js';
|
||||
export type {
|
||||
ImportMetricflowSemanticModelsDeps,
|
||||
ImportMetricflowSemanticModelsInput,
|
||||
MetricFlowImportResult,
|
||||
MetricflowSemanticLayerService,
|
||||
MetricflowSemanticLayerWriter,
|
||||
} from './adapters/metricflow/import-semantic-models.js';
|
||||
export type {
|
||||
MetricflowHostTable,
|
||||
MetricflowSemanticModelImportContext,
|
||||
MetricflowSemanticModelJoin,
|
||||
MetricflowWritableSemanticLayerSource,
|
||||
} from './adapters/metricflow/semantic-models.js';
|
||||
export { MetricflowSourceAdapter, type MetricflowSourceAdapterDeps } from './adapters/metricflow/metricflow.adapter.js';
|
||||
export {
|
||||
metricflowPullConfigSchema,
|
||||
parseMetricflowPullConfig,
|
||||
pullConfigFromMetricflowIntegration,
|
||||
} from './adapters/metricflow/pull-config.js';
|
||||
export type {
|
||||
MetricflowIntegrationLike,
|
||||
MetricflowPullConfig,
|
||||
} from './adapters/metricflow/pull-config.js';
|
||||
export { NOTION_ORG_KNOWLEDGE_WARNING } from './adapters/notion/chunk.js';
|
||||
export { NOTION_DEFAULT_MAX_KNOWLEDGE_CREATES_PER_RUN } from './adapters/notion/types.js';
|
||||
export { LocalNotionRuntimeStore } from './adapters/notion/local-state-store.js';
|
||||
export { NotionSourceAdapter, type NotionSourceAdapterDeps } from './adapters/notion/notion.adapter.js';
|
||||
export { NotionClient, type NotionApi, type NotionBotInfo } from './adapters/notion/notion-client.js';
|
||||
export { bucketDistinctUsers, bucketErrorRate, bucketExecutions, bucketP95Runtime, bucketRecency } from './adapters/historic-sql/buckets.js';
|
||||
export { chunkHistoricSqlUnifiedStagedDir, describeHistoricSqlUnifiedScope } from './adapters/historic-sql/chunk-unified.js';
|
||||
export { detectHistoricSqlStagedDir } from './adapters/historic-sql/detect.js';
|
||||
export {
|
||||
HistoricSqlExtensionMissingError,
|
||||
HistoricSqlGrantsMissingError,
|
||||
HistoricSqlVersionUnsupportedError,
|
||||
} from './adapters/historic-sql/errors.js';
|
||||
export { HistoricSqlSourceAdapter } from './adapters/historic-sql/historic-sql.adapter.js';
|
||||
export { BigQueryHistoricSqlQueryHistoryReader } from './adapters/historic-sql/bigquery-query-history-reader.js';
|
||||
export type { BigQueryHistoricSqlQueryHistoryReaderOptions } from './adapters/historic-sql/bigquery-query-history-reader.js';
|
||||
export { PostgresPgssReader } from './adapters/historic-sql/postgres-pgss-reader.js';
|
||||
export { SnowflakeHistoricSqlQueryHistoryReader } from './adapters/historic-sql/snowflake-query-history-reader.js';
|
||||
export { stageHistoricSqlAggregatedSnapshot } from './adapters/historic-sql/stage-unified.js';
|
||||
export {
|
||||
historicSqlEvidenceEnvelopeSchema,
|
||||
historicSqlEvidencePath,
|
||||
historicSqlPatternEvidenceSchema,
|
||||
historicSqlTableUsageEvidenceSchema,
|
||||
serializeHistoricSqlEvidence,
|
||||
} from './adapters/historic-sql/evidence.js';
|
||||
export type {
|
||||
HistoricSqlEvidenceEnvelope,
|
||||
HistoricSqlPatternEvidence,
|
||||
HistoricSqlTableUsageEvidence,
|
||||
} from './adapters/historic-sql/evidence.js';
|
||||
export { createEmitHistoricSqlEvidenceTool } from './adapters/historic-sql/evidence-tool.js';
|
||||
export { projectHistoricSqlEvidence } from './adapters/historic-sql/projection.js';
|
||||
export type { HistoricSqlProjectionInput, HistoricSqlProjectionResult } from './adapters/historic-sql/projection.js';
|
||||
export {
|
||||
patternOutputSchema,
|
||||
patternsArraySchema,
|
||||
tableUsageOutputSchema,
|
||||
} from './adapters/historic-sql/skill-schemas.js';
|
||||
export type {
|
||||
PatternOutput,
|
||||
TableUsageOutput,
|
||||
} from './adapters/historic-sql/skill-schemas.js';
|
||||
export type {
|
||||
AggregatedTemplate,
|
||||
HistoricSqlDialect,
|
||||
HistoricSqlProbeResult,
|
||||
HistoricSqlReader,
|
||||
HistoricSqlSourceAdapterDeps,
|
||||
HistoricSqlTimeWindow,
|
||||
HistoricSqlUnifiedPullConfig,
|
||||
KtxPostgresQueryClient,
|
||||
PostgresPgssProbeResult,
|
||||
StagedManifest,
|
||||
StagedPatternsInput,
|
||||
StagedTableInput,
|
||||
} from './adapters/historic-sql/types.js';
|
||||
export {
|
||||
HISTORIC_SQL_SOURCE_KEY,
|
||||
aggregatedTemplateSchema,
|
||||
historicSqlUnifiedPullConfigSchema,
|
||||
stagedManifestSchema,
|
||||
stagedPatternsInputSchema,
|
||||
stagedTableInputSchema,
|
||||
} from './adapters/historic-sql/types.js';
|
||||
export type { CanonicalPin } from './canonical-pins.js';
|
||||
export { buildCanonicalPinsPromptBlock, selectRelevantCanonicalPins } from './canonical-pins.js';
|
||||
export type {
|
||||
BudgetExhaustedCandidateForCarryForward,
|
||||
CandidateDedupServiceDeps,
|
||||
CandidateDedupSettings,
|
||||
ContextCandidateActionHint,
|
||||
ContextCandidateCarryforwardArgs,
|
||||
ContextCandidateCarryforwardResult,
|
||||
ContextCandidateCarryforwardServiceDeps,
|
||||
ContextCandidateCarryforwardSettings,
|
||||
ContextCandidateEmbeddingPort,
|
||||
ContextCandidateForPrompt,
|
||||
ContextCandidateLane,
|
||||
ContextCandidateRejectionReason,
|
||||
ContextCandidateScoreAggregation,
|
||||
ContextCandidateStatus,
|
||||
ContextCandidateStorePort,
|
||||
ContextCandidateVerdictSummary,
|
||||
CuratorPaginationInput,
|
||||
CuratorPaginationServiceDeps,
|
||||
CuratorPaginationSettings,
|
||||
CurrentRunEvidenceChunkForCarryForward,
|
||||
InsertContextCandidateInput,
|
||||
MarkContextCandidateClusterInput,
|
||||
} from './context-candidates/index.js';
|
||||
export {
|
||||
buildContextCandidateEmbeddingText,
|
||||
CandidateDedupService,
|
||||
ContextCandidateCarryforwardService,
|
||||
CuratorPaginationService,
|
||||
} from './context-candidates/index.js';
|
||||
export type {
|
||||
ContextEvidenceDocumentRef,
|
||||
ContextEvidenceEmbeddingPort,
|
||||
ContextEvidenceIndexStorePort,
|
||||
ContextEvidenceIndexSummary as PackageContextEvidenceIndexSummary,
|
||||
EvidencePublishState,
|
||||
ReplaceContextEvidenceChunk,
|
||||
SqliteContextEvidenceStoreOptions,
|
||||
UpsertContextEvidenceDocument,
|
||||
} from './context-evidence/index.js';
|
||||
export {
|
||||
ContextEvidenceIndexService,
|
||||
SqliteContextEvidenceStore,
|
||||
} from './context-evidence/index.js';
|
||||
export { DiffSetService } from './diff-set.service.js';
|
||||
export { IngestBundleRunner } from './ingest-bundle.runner.js';
|
||||
export type { DefaultLocalIngestAdaptersOptions } from './local-adapters.js';
|
||||
export { createDefaultLocalIngestAdapters, localPullConfigForAdapter } from './local-adapters.js';
|
||||
export type {
|
||||
LocalIngestMcpOptions,
|
||||
LocalIngestResult,
|
||||
LocalMetabaseFanoutChild,
|
||||
LocalMetabaseFanoutProgress,
|
||||
LocalMetabaseFanoutProgressChild,
|
||||
LocalMetabaseFanoutResult,
|
||||
RunLocalIngestOptions,
|
||||
RunLocalMetabaseIngestOptions,
|
||||
} from './local-ingest.js';
|
||||
export { getLatestLocalIngestStatus, getLocalIngestStatus, runLocalIngest, runLocalMetabaseIngest } from './local-ingest.js';
|
||||
export { seedLocalMappingStateFromKtxYaml } from './local-mapping-reconcile.js';
|
||||
export type {
|
||||
CreateLocalBundleIngestRuntimeOptions,
|
||||
LocalBundleIngestRuntime,
|
||||
} from './local-bundle-runtime.js';
|
||||
export { createLocalBundleIngestRuntime } from './local-bundle-runtime.js';
|
||||
export type {
|
||||
LocalIngestDiffPaths,
|
||||
LocalIngestRunRecord,
|
||||
LocalIngestStatus,
|
||||
RunLocalStageOnlyIngestOptions,
|
||||
} from './local-stage-ingest.js';
|
||||
export { getLocalStageOnlyIngestStatus, runLocalStageOnlyIngest } from './local-stage-ingest.js';
|
||||
export {
|
||||
ingestReportToMemoryFlowReplay,
|
||||
localIngestRunToMemoryFlowReplay,
|
||||
} from './memory-flow/events.js';
|
||||
export {
|
||||
buildAuthenticatedUrl,
|
||||
cleanupRepoDir,
|
||||
cloneOrPull,
|
||||
RepoConfigError,
|
||||
RepoFetchError,
|
||||
repoDirExists,
|
||||
sanitizeRepoError,
|
||||
testRepoConnection,
|
||||
validateRepoConfig,
|
||||
} from './repo-fetch.js';
|
||||
export type { RepoFetchConfig } from './repo-fetch.js';
|
||||
export {
|
||||
loadProjectInfo,
|
||||
parseProjectName,
|
||||
parseProjectVars,
|
||||
resolveJinjaVariables,
|
||||
} from './dbt-shared/project-vars.js';
|
||||
export type { DbtProjectInfo, ResolveJinjaVariablesResult } from './dbt-shared/project-vars.js';
|
||||
export { findDbtSchemaFiles, loadDbtSchemaFiles } from './dbt-shared/schema-files.js';
|
||||
export {
|
||||
computeDbtSchemaHash,
|
||||
parseDbtSchemaFile,
|
||||
parseDbtSchemaFiles,
|
||||
} from './adapters/dbt-descriptions/parse-schema.js';
|
||||
export type {
|
||||
DbtParsedColumn,
|
||||
DbtColumnConstraints,
|
||||
DbtDataTestRef,
|
||||
DbtParsedRelationship,
|
||||
DbtParsedTable,
|
||||
DbtSchemaFile,
|
||||
DbtSchemaParseResult,
|
||||
} from './adapters/dbt-descriptions/parse-schema.js';
|
||||
export { findMatchingKtxTable, matchDbtTables } from './adapters/dbt-descriptions/match-tables.js';
|
||||
export type { DbtHostTableLite, DbtTableMatch } from './adapters/dbt-descriptions/match-tables.js';
|
||||
export { toDescriptionUpdates } from './adapters/dbt-descriptions/to-description-updates.js';
|
||||
export type { DbtDescriptionUpdates } from './adapters/dbt-descriptions/to-description-updates.js';
|
||||
export { toRelationshipUpdates } from './adapters/dbt-descriptions/to-relationship-updates.js';
|
||||
export type { DbtRelationshipUpdates } from './adapters/dbt-descriptions/to-relationship-updates.js';
|
||||
export { toMetadataUpdates } from './adapters/dbt-descriptions/to-metadata-updates.js';
|
||||
export { mergeSemanticModelTables } from './adapters/dbt-descriptions/merge-semantic-model-tables.js';
|
||||
export type { KtxJoinUpdate, KtxMetadataUpdate } from '../scan/enrichment-types.js';
|
||||
export {
|
||||
createInitialMemoryFlowInteractionState,
|
||||
findMemoryFlowSearchMatches,
|
||||
reduceMemoryFlowInteractionState,
|
||||
selectedMemoryFlowColumn,
|
||||
selectedMemoryFlowDetails,
|
||||
selectMemoryFlowChip,
|
||||
selectMemoryFlowColumn,
|
||||
visibleMemoryFlowChips,
|
||||
} from './memory-flow/interaction.js';
|
||||
export { renderMemoryFlowInteractive } from './memory-flow/interactive-render.js';
|
||||
export { createMemoryFlowLiveBuffer, sanitizeMemoryFlowError } from './memory-flow/live-buffer.js';
|
||||
export { renderMemoryFlowReplay } from './memory-flow/render.js';
|
||||
export { formatMemoryFlowFinalSummary } from './memory-flow/summary.js';
|
||||
export type { MemoryFlowStreamEvent } from './memory-flow/schema.js';
|
||||
export {
|
||||
memoryFlowActionDetailSchema,
|
||||
memoryFlowDetailSectionsSchema,
|
||||
memoryFlowEventSchema,
|
||||
memoryFlowPlannedWorkUnitSchema,
|
||||
memoryFlowReplayInputSchema,
|
||||
memoryFlowRunStatusSchema,
|
||||
memoryFlowStreamEventSchema,
|
||||
parseMemoryFlowReplayInput,
|
||||
} from './memory-flow/schema.js';
|
||||
export type {
|
||||
MemoryFlowChip,
|
||||
MemoryFlowColumnId,
|
||||
MemoryFlowColumnView,
|
||||
MemoryFlowDisplayStatus,
|
||||
MemoryFlowEvent,
|
||||
MemoryFlowEventSink,
|
||||
MemoryFlowFilterMode,
|
||||
MemoryFlowInteractionCommand,
|
||||
MemoryFlowInteractionState,
|
||||
MemoryFlowLiveBufferOptions,
|
||||
MemoryFlowPaneId,
|
||||
MemoryFlowPlannedWorkUnit,
|
||||
MemoryFlowRenderOptions,
|
||||
MemoryFlowReplayInput,
|
||||
MemoryFlowReplayPatch,
|
||||
MemoryFlowRunStatus,
|
||||
MemoryFlowViewModel,
|
||||
} from './memory-flow/types.js';
|
||||
export { buildMemoryFlowViewModel } from './memory-flow/view-model.js';
|
||||
export type {
|
||||
MemoryFlowStatusBadge,
|
||||
MemoryFlowVisualColumn,
|
||||
MemoryFlowVisualModel,
|
||||
} from './memory-flow/visuals.js';
|
||||
export {
|
||||
buildMemoryFlowVisualModel,
|
||||
memoryFlowStatusBadge,
|
||||
renderMemoryFlowConnectorLine,
|
||||
} from './memory-flow/visuals.js';
|
||||
export type {
|
||||
PageTriageEvidenceChunk,
|
||||
PageTriageReport,
|
||||
PageTriageRunArgs,
|
||||
PageTriageServiceDeps,
|
||||
PageTriageSettings,
|
||||
PageTriageStorePort,
|
||||
} from './page-triage/index.js';
|
||||
export { PageTriageService } from './page-triage/index.js';
|
||||
export type {
|
||||
CandidateDedupPort,
|
||||
CandidateDedupResult,
|
||||
ContextCandidateCarryforwardPort,
|
||||
ContextCandidateForDedup,
|
||||
ContextCandidateSummary,
|
||||
ContextEvidenceCandidatesPort,
|
||||
ContextEvidenceIndexPort,
|
||||
ContextEvidenceIndexSummary,
|
||||
CreateIngestRunArgs,
|
||||
CuratorPaginationPort,
|
||||
CuratorPaginationReport,
|
||||
DiffSetComputerPort,
|
||||
IngestBundleRunnerDeps,
|
||||
IngestCanonicalPinsPort,
|
||||
IngestCommitMessagePort,
|
||||
IngestFileStorePort,
|
||||
IngestGitAuthor,
|
||||
IngestKnowledgeIndexPort,
|
||||
IngestLockPort,
|
||||
IngestProvenanceInsert,
|
||||
IngestProvenancePort,
|
||||
IngestProvenanceRow,
|
||||
IngestReportsPort,
|
||||
IngestRunnerJob,
|
||||
IngestRunRecord,
|
||||
IngestRunsPort,
|
||||
IngestSessionWorktree,
|
||||
IngestSessionWorktreePort,
|
||||
IngestSettingsPort,
|
||||
IngestStoragePort,
|
||||
IngestToolsetFactoryPort,
|
||||
IngestToolsetLike,
|
||||
PageTriagePort,
|
||||
PageTriageRunResult,
|
||||
ProvenanceActionType,
|
||||
SourceAdapterRegistryPort,
|
||||
} from './ports.js';
|
||||
export {
|
||||
buildSyncId,
|
||||
provenanceMarker,
|
||||
rawSourcesDirForSync,
|
||||
rawSourcesRoot,
|
||||
} from './raw-sources-paths.js';
|
||||
export { ingestReportSnapshotSchema, parseIngestReportSnapshot } from './report-snapshot.js';
|
||||
export type { IngestReportBody, IngestReportSnapshot } from './reports.js';
|
||||
export * from './artifact-gates.js';
|
||||
export * from './ingest-trace.js';
|
||||
export * from './isolated-diff/git-patch.js';
|
||||
export * from './isolated-diff/patch-integrator.js';
|
||||
export * from './isolated-diff/work-unit-executor.js';
|
||||
export * from './reports.js';
|
||||
export { SourceAdapterRegistry } from './source-adapter-registry.js';
|
||||
export type { SqliteBundleIngestStoreOptions } from './sqlite-bundle-ingest-store.js';
|
||||
export { SqliteBundleIngestStore } from './sqlite-bundle-ingest-store.js';
|
||||
export type {
|
||||
SaveCompletedLocalIngestRunInput,
|
||||
SqliteLocalIngestStoreOptions,
|
||||
} from './sqlite-local-ingest-store.js';
|
||||
export { SqliteLocalIngestStore } from './sqlite-local-ingest-store.js';
|
||||
export type {
|
||||
ReconcileCandidateForPrompt,
|
||||
ReconcileCandidateSummary,
|
||||
ReconcilePromptRunState,
|
||||
WikiPageRef,
|
||||
} from './stages/build-reconcile-context.js';
|
||||
export {
|
||||
buildReconcileSystemPrompt,
|
||||
buildReconcileToolSet,
|
||||
buildReconcileUserPrompt,
|
||||
} from './stages/build-reconcile-context.js';
|
||||
export type { ReconciliationOutcome } from './stages/stage-4-reconciliation.js';
|
||||
export { runReconciliationStage4 } from './stages/stage-4-reconciliation.js';
|
||||
export type { StageIndex } from './stages/stage-index.types.js';
|
||||
export type {
|
||||
ChunkResult,
|
||||
DiffSet,
|
||||
EvictionUnit,
|
||||
FetchContext,
|
||||
IngestBundleJob,
|
||||
IngestBundleRef,
|
||||
IngestBundleResult,
|
||||
IngestDiffSummary,
|
||||
IngestJobContext,
|
||||
IngestJobPhase,
|
||||
IngestTrigger,
|
||||
ScopeDescriptor,
|
||||
SourceAdapter,
|
||||
SourceFetchIssue,
|
||||
SourceFetchReport,
|
||||
TriageLane,
|
||||
TriageSignals,
|
||||
UnresolvedCardInfo,
|
||||
WorkUnit,
|
||||
DeterministicProjectionContext,
|
||||
ProjectionResult,
|
||||
DeterministicFinalizationContext,
|
||||
FinalizationOverrideReplay,
|
||||
FinalizationResult,
|
||||
} from './types.js';
|
||||
export * from './wiki-body-refs.js';
|
||||
|
|
@ -2,9 +2,10 @@ import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promis
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { GitService, SessionWorktreeService } from '../core/index.js';
|
||||
import { GitService } from '../../context/core/git.service.js';
|
||||
import { SessionWorktreeService } from '../../context/core/session-worktree.service.js';
|
||||
import { LocalGitFileStore } from '../project/local-git-file-store.js';
|
||||
import { addTouchedSlSource } from '../tools/index.js';
|
||||
import { addTouchedSlSource } from '../../context/tools/touched-sl-sources.js';
|
||||
import { IngestBundleRunner } from './ingest-bundle.runner.js';
|
||||
import type { IngestBundleRunnerDeps } from './ports.js';
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { addTouchedSlSource } from '../tools/index.js';
|
||||
import { addTouchedSlSource } from '../../context/tools/touched-sl-sources.js';
|
||||
import { IngestBundleRunner } from './ingest-bundle.runner.js';
|
||||
import { createMemoryFlowLiveBuffer } from './memory-flow/live-buffer.js';
|
||||
import type { MemoryFlowReplayInput } from './memory-flow/types.js';
|
||||
|
|
|
|||
|
|
@ -2,12 +2,17 @@ import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|||
import { dirname, join } from 'node:path';
|
||||
import pLimit from 'p-limit';
|
||||
import { z } from 'zod';
|
||||
import { type KtxLogger, noopLogger } from '../core/index.js';
|
||||
import { createRuntimeToolDescriptorFromAiTool, type KtxRuntimeToolSet } from '../llm/index.js';
|
||||
import type { CaptureSession, MemoryAction } from '../memory/index.js';
|
||||
import type { SemanticLayerService, SemanticLayerSource, SlValidationDeps } from '../sl/index.js';
|
||||
import { createTouchedSlSources, type ToolContext, type ToolSession, type TouchedSlSource } from '../tools/index.js';
|
||||
import type { KnowledgeWikiService } from '../wiki/index.js';
|
||||
import { type KtxLogger, noopLogger } from '../../context/core/config.js';
|
||||
import { createRuntimeToolDescriptorFromAiTool } from '../../context/llm/runtime-tools.js';
|
||||
import type { KtxRuntimeToolSet } from '../../context/llm/runtime-port.js';
|
||||
import type { CaptureSession, MemoryAction } from '../../context/memory/types.js';
|
||||
import type { SemanticLayerService } from '../../context/sl/semantic-layer.service.js';
|
||||
import type { SemanticLayerSource } from '../../context/sl/types.js';
|
||||
import type { SlValidationDeps } from '../../context/sl/tools/sl-warehouse-validation.js';
|
||||
import { createTouchedSlSources, type TouchedSlSource } from '../../context/tools/touched-sl-sources.js';
|
||||
import type { ToolContext } from '../../context/tools/base-tool.js';
|
||||
import type { ToolSession } from '../../context/tools/tool-session.js';
|
||||
import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js';
|
||||
import { findDanglingWikiRefsForActions } from '../wiki/wiki-ref-validation.js';
|
||||
import { actionTargetConnectionId } from './action-identity.js';
|
||||
import { NOTION_DEFAULT_MAX_KNOWLEDGE_CREATES_PER_RUN } from './adapters/notion/types.js';
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ import { readFile } from 'node:fs/promises';
|
|||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { PromptService } from '../prompts/index.js';
|
||||
import { SkillsRegistryService } from '../skills/index.js';
|
||||
import { PromptService } from '../../context/prompts/prompt.service.js';
|
||||
import { SkillsRegistryService } from '../../context/skills/skills-registry.service.js';
|
||||
|
||||
const promptsDir = fileURLToPath(new URL('../../prompts', import.meta.url));
|
||||
const skillsDir = fileURLToPath(new URL('../../skills', import.meta.url));
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ export interface IngestTraceContext {
|
|||
level?: IngestTraceLevel;
|
||||
}
|
||||
|
||||
export interface IngestTraceEvent {
|
||||
interface IngestTraceEvent {
|
||||
schemaVersion: 1;
|
||||
at: string;
|
||||
level: IngestTraceLevel;
|
||||
|
|
@ -121,22 +121,6 @@ export class FileIngestTraceWriter implements IngestTraceWriter {
|
|||
}
|
||||
}
|
||||
|
||||
export class NoopIngestTraceWriter implements IngestTraceWriter {
|
||||
readonly tracePath = '';
|
||||
readonly context: IngestTraceContext = {
|
||||
tracePath: '',
|
||||
jobId: '',
|
||||
connectionId: '',
|
||||
sourceKey: '',
|
||||
level: 'error',
|
||||
};
|
||||
|
||||
withContext(): IngestTraceWriter {
|
||||
return this;
|
||||
}
|
||||
|
||||
async event(): Promise<void> {}
|
||||
}
|
||||
|
||||
export async function traceTimed<T>(
|
||||
trace: IngestTraceWriter,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { assertSemanticLayerTargetPathsAllowed } from '../semantic-layer-target-policy.js';
|
||||
|
||||
/** @internal */
|
||||
export const textArtifactRoots = ['wiki/', 'semantic-layer/'] as const;
|
||||
|
||||
export interface PatchTouchedPath {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { GitService } from '../../core/index.js';
|
||||
import { GitService } from '../../../context/core/git.service.js';
|
||||
import { FileIngestTraceWriter } from '../ingest-trace.js';
|
||||
import { integrateWorkUnitPatch } from './patch-integrator.js';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
import { readFile } from 'node:fs/promises';
|
||||
import type { GitService } from '../../core/index.js';
|
||||
import type { GitService } from '../../../context/core/git.service.js';
|
||||
import type { FinalGateRepairResult } from '../final-gate-repair.js';
|
||||
import type { IngestTraceWriter } from '../ingest-trace.js';
|
||||
import { traceTimed } from '../ingest-trace.js';
|
||||
import { assertPatchAllowedForWorkUnit, parsePatchTouchedPaths } from './git-patch.js';
|
||||
import type { TextualConflictResolutionResult } from './textual-conflict-resolver.js';
|
||||
|
||||
export type PatchIntegrationTextualResolution =
|
||||
type PatchIntegrationTextualResolution =
|
||||
| { status: 'repaired'; attempts: number; changedPaths: string[] }
|
||||
| { status: 'failed'; attempts: number; reason: string };
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { z } from 'zod';
|
||||
import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../llm/index.js';
|
||||
import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js';
|
||||
import type { IngestTraceWriter } from '../ingest-trace.js';
|
||||
import { traceTimed } from '../ingest-trace.js';
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { GitService } from '../../core/index.js';
|
||||
import { GitService } from '../../../context/core/git.service.js';
|
||||
import { FileIngestTraceWriter } from '../ingest-trace.js';
|
||||
import { runIsolatedWorkUnit } from './work-unit-executor.js';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { mkdir, readFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import type { SessionOutcome } from '../../core/index.js';
|
||||
import type { SessionOutcome } from '../../../context/core/session-worktree.service.js';
|
||||
import type { IngestSessionWorktree, IngestSessionWorktreePort } from '../ports.js';
|
||||
import type { WorkUnit } from '../types.js';
|
||||
import type { IngestTraceWriter } from '../ingest-trace.js';
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js';
|
||||
import type { SqlAnalysisPort } from '../sql-analysis/index.js';
|
||||
import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js';
|
||||
import type { SqlAnalysisPort } from '../../context/sql-analysis/ports.js';
|
||||
import type { HistoricSqlReader } from './adapters/historic-sql/types.js';
|
||||
import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js';
|
||||
import { LocalNotionRuntimeStore } from './adapters/notion/local-state-store.js';
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
import { join } from 'node:path';
|
||||
import { localConnectionToWarehouseDescriptor, notionConnectionToPullConfig, parseNotionConnectionConfig } from '../connections/index.js';
|
||||
import { localConnectionToWarehouseDescriptor } from '../../context/connections/local-warehouse-descriptor.js';
|
||||
import { notionConnectionToPullConfig, parseNotionConnectionConfig } from '../../context/connections/notion-config.js';
|
||||
import { resolveKtxConfigReference } from '../core/config-reference.js';
|
||||
import { ktxLocalStateDbPath, type KtxLocalProject } from '../project/index.js';
|
||||
import type { SqlAnalysisPort } from '../sql-analysis/index.js';
|
||||
import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js';
|
||||
import type { KtxLocalProject } from '../../context/project/project.js';
|
||||
import type { SqlAnalysisPort } from '../../context/sql-analysis/ports.js';
|
||||
import { DbtSourceAdapter } from './adapters/dbt/dbt.adapter.js';
|
||||
import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js';
|
||||
import { HistoricSqlSourceAdapter } from './adapters/historic-sql/historic-sql.adapter.js';
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@ import { tmpdir } from 'node:os';
|
|||
import { join } from 'node:path';
|
||||
import Database from 'better-sqlite3';
|
||||
import YAML from 'yaml';
|
||||
import type { AgentRunnerPort, RunLoopParams } from '../llm/index.js';
|
||||
import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js';
|
||||
import type { AgentRunnerPort, RunLoopParams } from '../../context/llm/runtime-port.js';
|
||||
import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js';
|
||||
import { makeLocalGitRepo } from '../test/make-local-git-repo.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js';
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { AgentRunnerPort } from '../llm/index.js';
|
||||
import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js';
|
||||
import type { AgentRunnerPort } from '../../context/llm/runtime-port.js';
|
||||
import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../../context/project/project.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js';
|
||||
import { createLocalBundleIngestRuntime } from './local-bundle-runtime.js';
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue