apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-10 08:05:14 +02:00
Code Issues Projects Releases Packages Wiki Activity

refactor(cli): delete internal barrel index.ts files

Browse source 
The 34 `index.ts` re-export barrels inside `packages/cli/src/` were
holdovers from the pre-fold multi-workspace structure. Post-fold-in they
served no production purpose: external consumers go through the single
package main entry, and in-repo callers mostly imported through them
only because the path was short. Internally, knip flagged most barrel
re-exports as production-dead (only reached via tests).

This change:
- Deletes every internal barrel except `packages/cli/src/index.ts`
  (the published package entry).
- Rewrites ~270 source/test files to import each name directly from
  the file that defines it.
- Moves `tools/warehouse-verification/index.ts` to
  `create-warehouse-verification-tools.ts` (the function it defined
  locally) and updates its single consumer.
- Renames `search/backend-conformance.ts` → `.test-utils.ts` to match
  the existing test-helper file convention.
- Deletes 13 dead test-only chains (dbt-descriptions/*,
  live-database/extracted-schema, live-database/structural-sync,
  relationship-* feedback/review chain) plus their tests and a
  cascading orphan integration test.
- Updates test mocks that pointed at deleted barrel paths
  (notion-client, connector barrels in scan/local-scan-connectors
  tests) to mock the source files instead.
- Points the maintainer benchmark script
  (`scripts/relationship-benchmark-report.mjs`) at source files
  instead of `dist/context/scan/index.js`.
- Drops the barrel `!` entries from `knip.json`; adds explicit
  production entries only for the benchmark code reached via dist by
  the maintainer script.

Net: 413 files changed, ~1.2k insertions, ~9.4k deletions.

`pnpm run dead-code` (Biome + knip default + knip production) and
`pnpm run type-check` are clean; 2277 tests pass.
This commit is contained in:
Andrey Avtomonov 2026-05-21 12:41:20 +02:00
parent b690e6988b
commit 34d4a1e9e1
413 changed files with 1260 additions and 8739 deletions
Download patch file Download diff file Expand all files Collapse all files

51
knip.json
View file

@ -2,48 +2,41 @@
"$schema": "https://unpkg.com/knip@6/schema.json",
"workspaces": {
".": {
"entry": ["scripts/**/*.mjs"],
"project": ["scripts/**/*.mjs"],
"ignoreDependencies": [
"@semantic-release/commit-analyzer",
"@semantic-release/github",
"@semantic-release/npm",
"@semantic-release/release-notes-generator",
"conventional-changelog-conventionalcommits"
"entry": [
"scripts/**/*.mjs",
"scripts/**/*.cjs",
".releaserc.cjs!"
]
},
"packages/cli": {
"entry": [
"src/llm/index.ts!",
"src/context/**/index.ts!",
"src/connectors/*/index.ts!",
"src/print-command-tree.ts!",
"scripts/**/*.mjs",
"src/**/*.test-utils.ts",
"src/**/acceptance-fixtures.ts"
],
"project": ["src/**/*.{ts,tsx}!", "scripts/**/*.mjs", "vitest.config.ts"]
"src/**/acceptance-fixtures.ts",
"src/context/scan/relationship-benchmarks.ts!",
"src/context/scan/relationship-benchmark-report.ts!"
]
},
"docs-site": {
"entry": [
"components/**/*.{ts,tsx}!",
"source.config.ts!"
],
"project": [
"app/**/*.{ts,tsx}!",
"components/**/*.{ts,tsx}!",
"lib/**/*.{ts,tsx}!",
"*.ts!",
"*.mjs",
"tests/**/*.mjs"
],
"ignoreDependencies": ["tailwindcss"]
"ignoreDependencies": [
"tailwindcss"
]
}
},
"ignore": [
"**/dist/**",
"**/coverage/**",
"**/.next/**",
"**/node_modules/**"
"ignoreDependencies": [
"@semantic-release/commit-analyzer",
"@semantic-release/github",
"@semantic-release/npm",
"@semantic-release/release-notes-generator",
"conventional-changelog-conventionalcommits"
],
"ignoreBinaries": ["uv", "lsof"]
"ignoreBinaries": [
"uv",
"lsof"
]
}

2
packages/cli/src/admin-reindex.test.ts
View file

@ -1,6 +1,6 @@
import { createRequire } from 'node:module';
import type { ReindexSummary } from './context/index-sync/index.js';
import type { ReindexSummary } from './context/index-sync/types.js';
import { describe, expect, it, vi } from 'vitest';
import { renderReindexJson, renderReindexPlain, reindexHasErrors } from './admin-reindex.js';
import { runKtxCli } from './index.js';

8
packages/cli/src/admin-reindex.ts
View file

@ -1,6 +1,8 @@
import { KtxIngestEmbeddingPortAdapter, type KtxEmbeddingPort } from './context/index.js';
import { reindexLocalIndexes, type ReindexScopeResult, type ReindexSummary } from './context/index-sync/index.js';
import { loadKtxProject } from './context/project/index.js';
import { KtxIngestEmbeddingPortAdapter } from './context/llm/embedding-port.js';
import type { KtxEmbeddingPort } from './context/core/embedding.js';
import { reindexLocalIndexes } from './context/index-sync/reindex.js';
import type { ReindexScopeResult, ReindexSummary } from './context/index-sync/types.js';
import { loadKtxProject } from './context/project/project.js';
import { Option, type Command } from '@commander-js/extra-typings';
import { cancel, intro, log, note, outro } from '@clack/prompts';
import type { KtxCliCommandContext } from './cli-program.js';

2
packages/cli/src/admin.ts
View file

@ -50,7 +50,7 @@ export function registerAdminCommands(program: Command, context: KtxCliCommandCo
.description('Print a JSON Schema describing ktx.yaml (for editors and LLM agents)')
.option('--output <file>', 'Write the schema to a file instead of stdout')
.action(async (options: { output?: string }) => {
const { generateKtxProjectConfigJsonSchema } = await import('./context/project/index.js');
const { generateKtxProjectConfigJsonSchema } = await import('./context/project/config.js');;
const json = `${JSON.stringify(generateKtxProjectConfigJsonSchema(), null, 2)}\n`;
if (options.output) {
const { writeFile } = await import('node:fs/promises');

2
packages/cli/src/claude-code-prompt-caching.ts
View file

@ -1,4 +1,4 @@
import type { KtxProjectLlmConfig } from './context/project/index.js';
import type { KtxProjectLlmConfig } from './context/project/config.js';
const CLAUDE_CODE_IGNORED_PROMPT_CACHING_FIELDS = [
'systemTtl',

2
packages/cli/src/cli-runtime.ts
View file

@ -70,7 +70,7 @@ export function packageInfoFromJson(packageJson: unknown): KtxCliPackageInfo {
}
async function runInit(args: { projectDir: string; force: boolean }, io: KtxCliIo): Promise<number> {
const { initKtxProject } = await import('./context/project/index.js');
const { initKtxProject } = await import('./context/project/project.js');;
const result = await initKtxProject({
projectDir: args.projectDir,
force: args.force,

9
packages/cli/src/connection.test.ts
View file

@ -1,9 +1,12 @@
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { LookerClient, MetabaseRuntimeClient, NotionClient } from './context/ingest/index.js';
import { initKtxProject, parseKtxProjectConfig, serializeKtxProjectConfig } from './context/project/index.js';
import type { KtxConnectionDriver, KtxScanConnector } from './context/scan/index.js';
import type { LookerClient } from './context/ingest/adapters/looker/client.js';
import type { MetabaseRuntimeClient } from './context/ingest/adapters/metabase/client-port.js';
import type { NotionClient } from './context/ingest/adapters/notion/notion-client.js';
import { initKtxProject } from './context/project/project.js';
import { parseKtxProjectConfig, serializeKtxProjectConfig } from './context/project/config.js';
import type { KtxConnectionDriver, KtxScanConnector } from './context/scan/types.js';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runKtxConnection } from './connection.js';

28
packages/cli/src/connection.ts
View file

@ -1,19 +1,15 @@
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultLookerConnectionClientFactory,
DefaultMetabaseConnectionClientFactory,
type LookerClient,
type MetabaseRuntimeClient,
type NotionBotInfo,
NotionClient,
createLocalLookerCredentialResolver,
metabaseRuntimeConfigFromLocalConnection,
testRepoConnection,
} from './context/ingest/index.js';
import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from './context/connections/index.js';
import { resolveKtxConfigReference } from './context/core/index.js';
import { type KtxLocalProject, loadKtxProject } from './context/project/index.js';
import type { KtxScanConnector } from './context/scan/index.js';
import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory } from './context/ingest/adapters/metabase/client.js';
import { DefaultLookerConnectionClientFactory } from './context/ingest/adapters/looker/factory.js';
import type { LookerClient } from './context/ingest/adapters/looker/client.js';
import type { MetabaseRuntimeClient } from './context/ingest/adapters/metabase/client-port.js';
import { type NotionBotInfo, NotionClient } from './context/ingest/adapters/notion/notion-client.js';
import { createLocalLookerCredentialResolver } from './context/ingest/adapters/looker/local-looker.adapter.js';
import { metabaseRuntimeConfigFromLocalConnection } from './context/ingest/adapters/metabase/local-metabase.adapter.js';
import { testRepoConnection } from './context/ingest/repo-fetch.js';
import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from './context/connections/notion-config.js';
import { resolveKtxConfigReference } from './context/core/config-reference.js';
import { type KtxLocalProject, loadKtxProject } from './context/project/project.js';
import type { KtxScanConnector } from './context/scan/types.js';
import type { KtxCliIo } from './index.js';
import { bold, dim, green, red, SYMBOLS } from './io/symbols.js';
import { createKtxCliScanConnector } from './local-scan-connectors.js';

13
packages/cli/src/connectors/bigquery/connector.test.ts
View file

@ -1,15 +1,6 @@
import { describe, expect, it, vi } from 'vitest';
import {
bigQueryConnectionConfigFromConfig,
createBigQueryLiveDatabaseIntrospection,
isKtxBigQueryConnectionConfig,
type KtxBigQueryClient,
KtxBigQueryScanConnector,
type KtxBigQueryClientFactory,
type KtxBigQueryDataset,
type KtxBigQueryQueryJob,
type KtxBigQueryTableRef,
} from './index.js';
import { bigQueryConnectionConfigFromConfig, isKtxBigQueryConnectionConfig, type KtxBigQueryClient, KtxBigQueryScanConnector, type KtxBigQueryClientFactory, type KtxBigQueryDataset, type KtxBigQueryQueryJob, type KtxBigQueryTableRef } from '../../connectors/bigquery/connector.js';
import { createBigQueryLiveDatabaseIntrospection } from '../../connectors/bigquery/live-database-introspection.js';
function fakeClientFactory(): KtxBigQueryClientFactory {
const queryResults = vi.fn(async (): ReturnType<KtxBigQueryQueryJob['getQueryResults']> => [

26
packages/cli/src/connectors/bigquery/connector.ts
View file

@ -1,24 +1,6 @@
import { BigQuery, type TableField } from '@google-cloud/bigquery';
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/index.js';
import {
createKtxConnectorCapabilities,
type KtxColumnSampleInput,
type KtxColumnSampleResult,
type KtxColumnStatsInput,
type KtxColumnStatsResult,
type KtxQueryResult,
type KtxReadOnlyQueryInput,
type KtxScanConnector,
type KtxScanContext,
type KtxScanInput,
type KtxSchemaColumn,
type KtxSchemaSnapshot,
type KtxSchemaTable,
type KtxTableListEntry,
type KtxTableRef,
type KtxTableSampleInput,
type KtxTableSampleResult,
} from '../../context/scan/index.js';
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
import { readFileSync } from 'node:fs';
import { homedir } from 'node:os';
import { resolve } from 'node:path';
@ -57,12 +39,14 @@ export interface KtxBigQueryColumnDistinctValuesResult {
cardinality: number;
}
/** @internal */
export interface KtxBigQueryQueryJob {
getQueryResults(): Promise<
[Array<Record<string, unknown>>, unknown, { schema?: { fields?: TableField[] } }?, ...unknown[]]
>;
}
/** @internal */
export interface KtxBigQueryTableRef {
id?: string;
metadata?: { type?: string };
@ -81,6 +65,7 @@ export interface KtxBigQueryTableRef {
>;
}
/** @internal */
export interface KtxBigQueryDataset {
get(): Promise<unknown>;
getTables(): Promise<[KtxBigQueryTableRef[], ...unknown[]]>;
@ -223,6 +208,7 @@ export function isKtxBigQueryConnectionConfig(
return String(connection?.driver ?? '').toLowerCase() === 'bigquery';
}
/** @internal */
export function bigQueryConnectionConfigFromConfig(input: {
connectionId: string;
connection: KtxBigQueryConnectionConfig | undefined;

2
packages/cli/src/connectors/bigquery/dialect.ts
View file

@ -1,4 +1,4 @@
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/index.js';
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
type BigQueryTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;

18
packages/cli/src/connectors/bigquery/index.ts
View file

@ -1,18 +0,0 @@
export { KtxBigQueryDialect } from './dialect.js';
export {
bigQueryConnectionConfigFromConfig,
isKtxBigQueryConnectionConfig,
KtxBigQueryScanConnector,
type KtxBigQueryClient,
type KtxBigQueryClientFactory,
type KtxBigQueryColumnDistinctValuesOptions,
type KtxBigQueryColumnDistinctValuesResult,
type KtxBigQueryConnectionConfig,
type KtxBigQueryDataset,
type KtxBigQueryQueryJob,
type KtxBigQueryReadOnlyQueryInput,
type KtxBigQueryResolvedConnectionConfig,
type KtxBigQueryScanConnectorOptions,
type KtxBigQueryTableRef,
} from './connector.js';
export { createBigQueryLiveDatabaseIntrospection } from './live-database-introspection.js';

4
packages/cli/src/connectors/bigquery/live-database-introspection.ts
View file

@ -1,5 +1,5 @@
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/index.js';
import type { KtxProjectConnectionConfig } from '../../context/project/index.js';
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js';
import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
import {
KtxBigQueryScanConnector,
type KtxBigQueryClientFactory,

9
packages/cli/src/connectors/clickhouse/connector.test.ts
View file

@ -1,11 +1,6 @@
import { describe, expect, it, vi } from 'vitest';
import {
clickHouseClientConfigFromConfig,
createClickHouseLiveDatabaseIntrospection,
isKtxClickHouseConnectionConfig,
KtxClickHouseScanConnector,
type KtxClickHouseClientFactory,
} from './index.js';
import { clickHouseClientConfigFromConfig, isKtxClickHouseConnectionConfig, KtxClickHouseScanConnector, type KtxClickHouseClientFactory } from '../../connectors/clickhouse/connector.js';
import { createClickHouseLiveDatabaseIntrospection } from '../../connectors/clickhouse/live-database-introspection.js';
function result<T>(payload: T) {
return {

23
packages/cli/src/connectors/clickhouse/connector.ts
View file

@ -1,24 +1,6 @@
import { createClient } from '@clickhouse/client';
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/index.js';
import {
createKtxConnectorCapabilities,
type KtxColumnSampleInput,
type KtxColumnSampleResult,
type KtxColumnStatsInput,
type KtxColumnStatsResult,
type KtxQueryResult,
type KtxReadOnlyQueryInput,
type KtxScanConnector,
type KtxScanContext,
type KtxScanInput,
type KtxSchemaColumn,
type KtxSchemaSnapshot,
type KtxSchemaTable,
type KtxTableRef,
type KtxTableSampleInput,
type KtxTableListEntry,
type KtxTableSampleResult,
} from '../../context/scan/index.js';
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js';
import { readFileSync } from 'node:fs';
import { Agent as HttpsAgent } from 'node:https';
import { homedir } from 'node:os';
@ -198,6 +180,7 @@ export function isKtxClickHouseConnectionConfig(
return String(connection?.driver ?? '').toLowerCase() === 'clickhouse';
}
/** @internal */
export function clickHouseClientConfigFromConfig(input: {
connectionId: string;
connection: KtxClickHouseConnectionConfig | undefined;

2
packages/cli/src/connectors/clickhouse/dialect.ts
View file

@ -1,4 +1,4 @@
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/index.js';
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
type ClickHouseTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;

16
packages/cli/src/connectors/clickhouse/index.ts
View file

@ -1,16 +0,0 @@
export { KtxClickHouseDialect } from './dialect.js';
export {
clickHouseClientConfigFromConfig,
isKtxClickHouseConnectionConfig,
KtxClickHouseScanConnector,
type KtxClickHouseClient,
type KtxClickHouseClientFactory,
type KtxClickHouseColumnDistinctValuesOptions,
type KtxClickHouseColumnDistinctValuesResult,
type KtxClickHouseConnectionConfig,
type KtxClickHouseEndpointResolver,
type KtxClickHouseReadOnlyQueryInput,
type KtxClickHouseResolvedClientConfig,
type KtxClickHouseScanConnectorOptions,
} from './connector.js';
export { createClickHouseLiveDatabaseIntrospection } from './live-database-introspection.js';

4
packages/cli/src/connectors/clickhouse/live-database-introspection.ts
View file

@ -1,5 +1,5 @@
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/index.js';
import type { KtxProjectConnectionConfig } from '../../context/project/index.js';
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js';
import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
import {
KtxClickHouseScanConnector,
type KtxClickHouseClientFactory,

9
packages/cli/src/connectors/mysql/connector.test.ts
View file

@ -1,12 +1,7 @@
import { describe, expect, it, vi } from 'vitest';
import type { FieldPacket, RowDataPacket } from 'mysql2/promise';
import {
createMysqlLiveDatabaseIntrospection,
isKtxMysqlConnectionConfig,
KtxMysqlScanConnector,
mysqlConnectionPoolConfigFromConfig,
type KtxMysqlPoolFactory,
} from './index.js';
import { createMysqlLiveDatabaseIntrospection } from '../../connectors/mysql/live-database-introspection.js';
import { isKtxMysqlConnectionConfig, KtxMysqlScanConnector, mysqlConnectionPoolConfigFromConfig, type KtxMysqlPoolFactory } from '../../connectors/mysql/connector.js';
function mysqlResult(rows: Record<string, unknown>[], fields: Array<{ name: string; type?: number }>): [RowDataPacket[], FieldPacket[]] {
return [rows as RowDataPacket[], fields as FieldPacket[]];

24
packages/cli/src/connectors/mysql/connector.ts
View file

@ -2,27 +2,8 @@ import mysql, { type FieldPacket, type Pool, type RowDataPacket } from 'mysql2/p
import { readFileSync } from 'node:fs';
import { homedir } from 'node:os';
import { resolve } from 'node:path';
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/index.js';
import {
createKtxConnectorCapabilities,
type KtxColumnSampleInput,
type KtxColumnSampleResult,
type KtxColumnStatsInput,
type KtxColumnStatsResult,
type KtxQueryResult,
type KtxReadOnlyQueryInput,
type KtxScanConnector,
type KtxScanContext,
type KtxScanInput,
type KtxSchemaColumn,
type KtxTableListEntry,
type KtxSchemaForeignKey,
type KtxSchemaSnapshot,
type KtxSchemaTable,
type KtxTableRef,
type KtxTableSampleInput,
type KtxTableSampleResult,
} from '../../context/scan/index.js';
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxTableListEntry, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
import { KtxMysqlDialect } from './dialect.js';
export interface KtxMysqlConnectionConfig {
@ -237,6 +218,7 @@ export function isKtxMysqlConnectionConfig(
return String(connection?.driver ?? '').toLowerCase() === 'mysql';
}
/** @internal */
export function mysqlConnectionPoolConfigFromConfig(input: {
connectionId: string;
connection: KtxMysqlConnectionConfig | undefined;

2
packages/cli/src/connectors/mysql/dialect.ts
View file

@ -1,4 +1,4 @@
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/index.js';
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
type MysqlTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;

15
packages/cli/src/connectors/mysql/index.ts
View file

@ -1,15 +0,0 @@
export { KtxMysqlDialect } from './dialect.js';
export {
isKtxMysqlConnectionConfig,
KtxMysqlScanConnector,
mysqlConnectionPoolConfigFromConfig,
type KtxMysqlColumnDistinctValuesOptions,
type KtxMysqlColumnDistinctValuesResult,
type KtxMysqlConnectionConfig,
type KtxMysqlEndpointResolver,
type KtxMysqlPoolConfig,
type KtxMysqlPoolFactory,
type KtxMysqlReadOnlyQueryInput,
type KtxMysqlScanConnectorOptions,
} from './connector.js';
export { createMysqlLiveDatabaseIntrospection } from './live-database-introspection.js';

4
packages/cli/src/connectors/mysql/live-database-introspection.ts
View file

@ -1,5 +1,5 @@
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/index.js';
import type { KtxProjectConnectionConfig } from '../../context/project/index.js';
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js';
import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
import {
KtxMysqlScanConnector,
type KtxMysqlConnectionConfig,

9
packages/cli/src/connectors/postgres/connector.test.ts
View file

@ -1,11 +1,6 @@
import { describe, expect, it, vi } from 'vitest';
import {
createPostgresLiveDatabaseIntrospection,
isKtxPostgresConnectionConfig,
KtxPostgresScanConnector,
postgresPoolConfigFromConfig,
type KtxPostgresPoolFactory,
} from './index.js';
import { createPostgresLiveDatabaseIntrospection } from '../../connectors/postgres/live-database-introspection.js';
import { isKtxPostgresConnectionConfig, KtxPostgresScanConnector, postgresPoolConfigFromConfig, type KtxPostgresPoolFactory } from '../../connectors/postgres/connector.js';
interface FakeQueryResult {
rows: Record<string, unknown>[];

24
packages/cli/src/connectors/postgres/connector.ts
View file

@ -1,27 +1,8 @@
import { readFileSync } from 'node:fs';
import { homedir } from 'node:os';
import { resolve } from 'node:path';
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/index.js';
import {
createKtxConnectorCapabilities,
type KtxColumnSampleInput,
type KtxColumnSampleResult,
type KtxColumnStatsInput,
type KtxColumnStatsResult,
type KtxQueryResult,
type KtxReadOnlyQueryInput,
type KtxScanConnector,
type KtxScanContext,
type KtxScanInput,
type KtxSchemaColumn,
type KtxSchemaForeignKey,
type KtxSchemaSnapshot,
type KtxSchemaTable,
type KtxTableListEntry,
type KtxTableRef,
type KtxTableSampleInput,
type KtxTableSampleResult,
} from '../../context/scan/index.js';
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
import { Pool } from 'pg';
import { KtxPostgresDialect } from './dialect.js';
@ -297,6 +278,7 @@ export function isKtxPostgresConnectionConfig(
return driver === 'postgres' || driver === 'postgresql';
}
/** @internal */
export function postgresPoolConfigFromConfig(input: {
connectionId: string;
connection: KtxPostgresConnectionConfig | undefined;

2
packages/cli/src/connectors/postgres/dialect.ts
View file

@ -1,4 +1,4 @@
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/index.js';
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
type PostgresTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;

2
packages/cli/src/connectors/postgres/historic-sql-query-client.ts
View file

@ -1,4 +1,4 @@
import type { KtxPostgresQueryClient } from '../../context/ingest/index.js';
import type { KtxPostgresQueryClient } from '../../context/ingest/adapters/historic-sql/types.js';
import { KtxPostgresScanConnector, type KtxPostgresScanConnectorOptions } from './connector.js';
export type KtxPostgresHistoricSqlQueryClientOptions = KtxPostgresScanConnectorOptions;

21
packages/cli/src/connectors/postgres/index.ts
View file

@ -1,21 +0,0 @@
export { KtxPostgresDialect } from './dialect.js';
export {
isKtxPostgresConnectionConfig,
KtxPostgresScanConnector,
postgresPoolConfigFromConfig,
type KtxPostgresColumnDistinctValuesOptions,
type KtxPostgresColumnDistinctValuesResult,
type KtxPostgresColumnStatisticsResult,
type KtxPostgresConnectionConfig,
type KtxPostgresEndpointResolver,
type KtxPostgresPoolConfig,
type KtxPostgresPoolFactory,
type KtxPostgresReadOnlyQueryInput,
type KtxPostgresScanConnectorOptions,
type KtxPostgresTableSampleResult,
} from './connector.js';
export {
KtxPostgresHistoricSqlQueryClient,
type KtxPostgresHistoricSqlQueryClientOptions,
} from './historic-sql-query-client.js';
export { createPostgresLiveDatabaseIntrospection } from './live-database-introspection.js';

4
packages/cli/src/connectors/postgres/live-database-introspection.ts
View file

@ -1,5 +1,5 @@
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/index.js';
import type { KtxProjectConnectionConfig } from '../../context/project/index.js';
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js';
import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
import {
KtxPostgresScanConnector,
type KtxPostgresConnectionConfig,

10
packages/cli/src/connectors/snowflake/connector.test.ts
View file

@ -1,12 +1,6 @@
import { describe, expect, it, vi } from 'vitest';
import {
createSnowflakeLiveDatabaseIntrospection,
isKtxSnowflakeConnectionConfig,
KtxSnowflakeScanConnector,
snowflakeConnectionConfigFromConfig,
type KtxSnowflakeDriver,
type KtxSnowflakeDriverFactory,
} from './index.js';
import { createSnowflakeLiveDatabaseIntrospection } from '../../connectors/snowflake/live-database-introspection.js';
import { isKtxSnowflakeConnectionConfig, KtxSnowflakeScanConnector, snowflakeConnectionConfigFromConfig, type KtxSnowflakeDriver, type KtxSnowflakeDriverFactory } from '../../connectors/snowflake/connector.js';
function fakeDriverFactory(): KtxSnowflakeDriverFactory {
const driver: KtxSnowflakeDriver = {

23
packages/cli/src/connectors/snowflake/connector.ts
View file

@ -2,26 +2,8 @@ import { createPrivateKey } from 'node:crypto';
import { readFileSync } from 'node:fs';
import { homedir } from 'node:os';
import { resolve } from 'node:path';
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/index.js';
import {
createKtxConnectorCapabilities,
type KtxColumnSampleInput,
type KtxColumnSampleResult,
type KtxColumnStatsInput,
type KtxColumnStatsResult,
type KtxQueryResult,
type KtxReadOnlyQueryInput,
type KtxScanConnector,
type KtxScanContext,
type KtxScanInput,
type KtxSchemaColumn,
type KtxSchemaSnapshot,
type KtxSchemaTable,
type KtxTableRef,
type KtxTableSampleInput,
type KtxTableListEntry,
type KtxTableSampleResult,
} from '../../context/scan/index.js';
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js';
import * as snowflake from 'snowflake-sdk';
import { KtxSnowflakeDialect } from './dialect.js';
@ -196,6 +178,7 @@ export function isKtxSnowflakeConnectionConfig(
return String(connection?.driver ?? '').toLowerCase() === 'snowflake';
}
/** @internal */
export function snowflakeConnectionConfigFromConfig(input: {
connectionId: string;
connection: KtxSnowflakeConnectionConfig | undefined;

2
packages/cli/src/connectors/snowflake/dialect.ts
View file

@ -1,4 +1,4 @@
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/index.js';
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
type SnowflakeTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;

18
packages/cli/src/connectors/snowflake/index.ts
View file

@ -1,18 +0,0 @@
export { KtxSnowflakeDialect } from './dialect.js';
export {
isKtxSnowflakeConnectionConfig,
KtxSnowflakeScanConnector,
snowflakeConnectionConfigFromConfig,
type KtxSnowflakeColumnDistinctValuesOptions,
type KtxSnowflakeColumnDistinctValuesResult,
type KtxSnowflakeConnectionConfig,
type KtxSnowflakeDriver,
type KtxSnowflakeDriverFactory,
type KtxSnowflakeRawColumnMetadata,
type KtxSnowflakeRawTableMetadata,
type KtxSnowflakeReadOnlyQueryInput,
type KtxSnowflakeResolvedConnectionConfig,
type KtxSnowflakeScanConnectorOptions,
type KtxSnowflakeSdkOptionsProvider,
} from './connector.js';
export { createSnowflakeLiveDatabaseIntrospection } from './live-database-introspection.js';

4
packages/cli/src/connectors/snowflake/live-database-introspection.ts
View file

@ -1,5 +1,5 @@
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/index.js';
import type { KtxProjectConnectionConfig } from '../../context/project/index.js';
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js';
import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
import {
KtxSnowflakeScanConnector,
type KtxSnowflakeConnectionConfig,

8
packages/cli/src/connectors/sqlite/connector.test.ts
View file

@ -4,12 +4,8 @@ import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import {
createSqliteLiveDatabaseIntrospection,
isKtxSqliteConnectionConfig,
KtxSqliteScanConnector,
sqliteDatabasePathFromConfig,
} from './index.js';
import { createSqliteLiveDatabaseIntrospection } from '../../connectors/sqlite/live-database-introspection.js';
import { isKtxSqliteConnectionConfig, KtxSqliteScanConnector, sqliteDatabasePathFromConfig } from '../../connectors/sqlite/connector.js';
describe('KtxSqliteScanConnector', () => {
let tempDir: string;

24
packages/cli/src/connectors/sqlite/connector.ts
View file

@ -3,25 +3,9 @@ import { existsSync, readFileSync, statSync } from 'node:fs';
import { homedir } from 'node:os';
import { isAbsolute, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { assertReadOnlySql, limitSqlForExecution, normalizeQueryRows } from '../../context/connections/index.js';
import {
createKtxConnectorCapabilities,
type KtxColumnSampleInput,
type KtxColumnSampleResult,
type KtxColumnStatsInput,
type KtxColumnStatsResult,
type KtxQueryResult,
type KtxReadOnlyQueryInput,
type KtxScanConnector,
type KtxScanContext,
type KtxScanInput,
type KtxSchemaForeignKey,
type KtxSchemaSnapshot,
type KtxSchemaTable,
type KtxTableRef,
type KtxTableSampleInput,
type KtxTableSampleResult,
} from '../../context/scan/index.js';
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
import { normalizeQueryRows } from '../../context/connections/query-executor.js';
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
import { KtxSqliteDialect } from './dialect.js';
export interface KtxSqliteConnectionConfig {
@ -31,6 +15,7 @@ export interface KtxSqliteConnectionConfig {
[key: string]: unknown;
}
/** @internal */
export interface SqliteDatabasePathInput {
connectionId: string;
projectDir?: string;
@ -142,6 +127,7 @@ export function isKtxSqliteConnectionConfig(
return driver === 'sqlite' || driver === 'sqlite3';
}
/** @internal */
export function sqliteDatabasePathFromConfig(input: SqliteDatabasePathInput): string {
const inputDriver = input.connection?.driver ?? 'unknown';
if (!isKtxSqliteConnectionConfig(input.connection)) {

2
packages/cli/src/connectors/sqlite/dialect.ts
View file

@ -1,4 +1,4 @@
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/index.js';
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
type SqliteTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;

16
packages/cli/src/connectors/sqlite/index.ts
View file

@ -1,16 +0,0 @@
export { KtxSqliteDialect } from './dialect.js';
export {
isKtxSqliteConnectionConfig,
KtxSqliteScanConnector,
sqliteDatabasePathFromConfig,
type KtxSqliteColumnDistinctValuesOptions,
type KtxSqliteColumnDistinctValuesResult,
type KtxSqliteConnectionConfig,
type KtxSqliteReadOnlyQueryInput,
type KtxSqliteScanConnectorOptions,
type SqliteDatabasePathInput,
} from './connector.js';
export {
createSqliteLiveDatabaseIntrospection,
type CreateSqliteLiveDatabaseIntrospectionOptions,
} from './live-database-introspection.js';

4
packages/cli/src/connectors/sqlite/live-database-introspection.ts
View file

@ -1,5 +1,5 @@
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/index.js';
import type { KtxProjectConnectionConfig } from '../../context/project/index.js';
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js';
import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
import { KtxSqliteScanConnector, type KtxSqliteConnectionConfig } from './connector.js';
export interface CreateSqliteLiveDatabaseIntrospectionOptions {

10
packages/cli/src/connectors/sqlserver/connector.test.ts
View file

@ -1,12 +1,6 @@
import { describe, expect, it, vi } from 'vitest';
import {
createSqlServerLiveDatabaseIntrospection,
isKtxSqlServerConnectionConfig,
KtxSqlServerScanConnector,
sqlServerConnectionPoolConfigFromConfig,
type KtxSqlServerPoolFactory,
type KtxSqlServerQueryResult,
} from './index.js';
import { createSqlServerLiveDatabaseIntrospection } from '../../connectors/sqlserver/live-database-introspection.js';
import { isKtxSqlServerConnectionConfig, KtxSqlServerScanConnector, sqlServerConnectionPoolConfigFromConfig, type KtxSqlServerPoolFactory, type KtxSqlServerQueryResult } from '../../connectors/sqlserver/connector.js';
function recordset<T extends Record<string, unknown>>(
rows: T[],

25
packages/cli/src/connectors/sqlserver/connector.ts
View file

@ -1,24 +1,5 @@
import { assertReadOnlySql } from '../../context/connections/index.js';
import {
createKtxConnectorCapabilities,
type KtxColumnSampleInput,
type KtxColumnSampleResult,
type KtxColumnStatsInput,
type KtxColumnStatsResult,
type KtxQueryResult,
type KtxReadOnlyQueryInput,
type KtxScanConnector,
type KtxScanContext,
type KtxScanInput,
type KtxSchemaColumn,
type KtxSchemaForeignKey,
type KtxSchemaSnapshot,
type KtxSchemaTable,
type KtxTableListEntry,
type KtxTableRef,
type KtxTableSampleInput,
type KtxTableSampleResult,
} from '../../context/scan/index.js';
import { assertReadOnlySql } from '../../context/connections/read-only-sql.js';
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
import { readFileSync } from 'node:fs';
import { homedir } from 'node:os';
import { resolve } from 'node:path';
@ -50,6 +31,7 @@ export interface KtxSqlServerPoolConfig {
pool: { max: number; min: number; idleTimeoutMillis: number };
}
/** @internal */
export interface KtxSqlServerQueryResult {
recordset?: Array<Record<string, unknown>> & { columns?: Record<string, { type?: { declaration?: string } }> };
}
@ -239,6 +221,7 @@ export function isKtxSqlServerConnectionConfig(
return String(connection?.driver ?? '').toLowerCase() === 'sqlserver';
}
/** @internal */
export function sqlServerConnectionPoolConfigFromConfig(input: {
connectionId: string;
connection: KtxSqlServerConnectionConfig | undefined;

2
packages/cli/src/connectors/sqlserver/dialect.ts
View file

@ -1,4 +1,4 @@
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/index.js';
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
type SqlServerTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;

17
packages/cli/src/connectors/sqlserver/index.ts
View file

@ -1,17 +0,0 @@
export { KtxSqlServerDialect } from './dialect.js';
export {
isKtxSqlServerConnectionConfig,
KtxSqlServerScanConnector,
sqlServerConnectionPoolConfigFromConfig,
type KtxSqlServerColumnDistinctValuesOptions,
type KtxSqlServerColumnDistinctValuesResult,
type KtxSqlServerConnectionConfig,
type KtxSqlServerEndpointResolver,
type KtxSqlServerPool,
type KtxSqlServerPoolConfig,
type KtxSqlServerPoolFactory,
type KtxSqlServerQueryResult,
type KtxSqlServerReadOnlyQueryInput,
type KtxSqlServerScanConnectorOptions,
} from './connector.js';
export { createSqlServerLiveDatabaseIntrospection } from './live-database-introspection.js';

4
packages/cli/src/connectors/sqlserver/live-database-introspection.ts
View file

@ -1,5 +1,5 @@
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/index.js';
import type { KtxProjectConnectionConfig } from '../../context/project/index.js';
import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js';
import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
import {
KtxSqlServerScanConnector,
type KtxSqlServerConnectionConfig,

2
packages/cli/src/context-build-view.test.ts
View file

@ -1,4 +1,4 @@
import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/index.js';
import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from './context/project/config.js';
import { describe, expect, it, vi } from 'vitest';
import type { KtxPublicIngestProject, KtxPublicIngestTargetResult } from './public-ingest.js';
import {

2
packages/cli/src/context-build-view.ts
View file

@ -1,4 +1,4 @@
import type { KtxProgressPort, KtxProgressUpdateOptions } from './context/scan/index.js';
import type { KtxProgressPort, KtxProgressUpdateOptions } from './context/scan/types.js';
import type { KtxCliIo } from './index.js';
import type { KtxIngestProgressUpdate } from './ingest.js';
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';

9
packages/cli/src/context/agent/index.ts
View file

@ -1,9 +0,0 @@
export type {
AgentRunnerPort,
RunLoopParams,
RunLoopResult,
RunLoopStepInfo,
RunLoopStopReason,
} from '../llm/runtime-port.js';
export { RuntimeAgentRunner } from '../llm/runtime-port.js';
export type { AgentTelemetryPort } from '../llm/ai-sdk-runtime.js';

2
packages/cli/src/context/connections/dialects.ts
View file

@ -1,6 +1,6 @@
import type { KtxSchemaDimensionType, KtxTableRef } from '../scan/types.js';
export type SupportedDriver =
type SupportedDriver =
| 'postgres'
| 'postgresql'
| 'mysql'

30
packages/cli/src/context/connections/index.ts
View file

@ -1,30 +0,0 @@
export type {
KtxSqlQueryExecutionInput,
KtxSqlQueryExecutionResult,
KtxSqlQueryExecutorPort,
} from './query-executor.js';
export type { KtxDialect, SupportedDriver } from './dialects.js';
export { createDefaultLocalQueryExecutor, type DefaultLocalQueryExecutorOptions } from './local-query-executor.js';
export { getDialectForDriver } from './dialects.js';
export { normalizeQueryRows } from './query-executor.js';
export { createPostgresQueryExecutor } from './postgres-query-executor.js';
export { assertReadOnlySql, limitSqlForExecution } from './read-only-sql.js';
export { createSqliteQueryExecutor, sqliteDatabasePathFromConnection } from './sqlite-query-executor.js';
export { connectionTypeSchema, type ConnectionType } from './connection-type.js';
export {
localConnectionInfoFromConfig,
localConnectionToWarehouseDescriptor,
localConnectionTypeForConfig,
type LocalConnectionInfo,
type LocalWarehouseDescriptor,
} from './local-warehouse-descriptor.js';
export {
KTX_NOTION_ORG_KNOWLEDGE_WARNING,
notionConnectionToPullConfig,
parseNotionConnectionConfig,
redactNotionConnectionConfig,
resolveNotionConnectionAuthToken,
resolveNotionAuthToken,
type KtxNotionConnectionConfig,
type RedactedKtxNotionConnectionConfig,
} from './notion-config.js';

5
packages/cli/src/context/connections/notion-config.ts
View file

@ -8,7 +8,7 @@ import {
} from '../ingest/adapters/notion/types.js';
import type { KtxProjectConnectionConfig } from '../project/config.js';
export const KTX_NOTION_ORG_KNOWLEDGE_WARNING =
const KTX_NOTION_ORG_KNOWLEDGE_WARNING =
'Anything accessible to this Notion integration can become organization knowledge.';
type KtxNotionCrawlMode = 'all_accessible' | 'selected_roots';
@ -39,6 +39,7 @@ export type KtxNotionConnectionConfig = Omit<
max_knowledge_updates_per_run: number;
};
/** @internal */
export interface RedactedKtxNotionConnectionConfig {
driver: 'notion';
hasAuthToken: boolean;
@ -152,6 +153,7 @@ export function parseNotionConnectionConfig(raw: unknown): KtxNotionConnectionCo
};
}
/** @internal */
export function redactNotionConnectionConfig(config: KtxNotionConnectionConfig): RedactedKtxNotionConnectionConfig {
return {
driver: 'notion',
@ -171,6 +173,7 @@ function expandHome(path: string): string {
return path === '~' || path.startsWith('~/') ? resolve(homedir(), path.slice(2)) : path;
}
/** @internal */
export async function resolveNotionAuthToken(
authTokenRef: string,
options: ResolveNotionTokenOptions = {},

2
packages/cli/src/context/connections/query-executor.ts
View file

@ -1,4 +1,4 @@
import type { KtxProjectConnectionConfig } from '../project/index.js';
import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
export interface KtxSqlQueryExecutionInput {
connectionId: string;

1
packages/cli/src/context/connections/sqlite-query-executor.ts
View file

@ -49,6 +49,7 @@ function sqlitePathFromUrl(url: string): string {
return url;
}
/** @internal */
export function sqliteDatabasePathFromConnection(input: KtxSqlQueryExecutionInput): string {
const driver = connectionDriver(input);
if (driver !== 'sqlite' && driver !== 'sqlite3') {

1
packages/cli/src/context/core/config-reference.ts
View file

@ -2,6 +2,7 @@ import { readFileSync } from 'node:fs';
import { homedir } from 'node:os';
import { resolve } from 'node:path';
/** @internal */
export function resolveKtxHomePath(path: string): string {
if (path === '~') {
return homedir();

4
packages/cli/src/context/core/config.ts
View file

@ -1,10 +1,10 @@
export interface KtxStorageConfig {
interface KtxStorageConfig {
configDir?: string;
homeDir?: string;
worktreesDir?: string;
}
export interface KtxGitConfig {
interface KtxGitConfig {
userName: string;
userEmail: string;
bootstrapMessage?: string;

27
packages/cli/src/context/core/index.ts
View file

@ -1,27 +0,0 @@
export type { KtxCoreConfig, KtxGitConfig, KtxLogger, KtxStorageConfig } from './config.js';
export { noopLogger, resolveConfigDir, resolveWorktreesDir } from './config.js';
export { resolveKtxConfigReference, resolveKtxHomePath } from './config-reference.js';
export type { KtxEmbeddingPort } from './embedding.js';
export {
REDACTED_KTX_CREDENTIAL_VALUE,
redactKtxSensitiveMetadata,
redactKtxSensitiveText,
redactKtxSensitiveValue,
} from './redaction.js';
export type {
KtxFileHistoryEntry,
KtxFileListResult,
KtxFileReadResult,
KtxFileStorePort,
KtxFileWriteResult,
} from './file-store.js';
export type { GitCommitInfo, SquashMergeResult, WorktreeEntry } from './git.service.js';
export { GitService } from './git.service.js';
export type {
SentinelPayload,
SessionOutcome,
SessionWorktree,
SessionWorktreeServiceDeps,
WorktreeConfigPort,
} from './session-worktree.service.js';
export { SessionWorktreeService } from './session-worktree.service.js';

1
packages/cli/src/context/core/redaction.ts
View file

@ -1,3 +1,4 @@
/** @internal */
export const REDACTED_KTX_CREDENTIAL_VALUE = '<redacted>';
const SENSITIVE_FIELD_NAME = /(password|secret|token|api[_-]?key|private[_-]?key|passphrase|credential|authorization|url)/i;

2
packages/cli/src/context/core/session-worktree.service.ts
View file

@ -5,7 +5,7 @@ import { GitService } from './git.service.js';
export type SessionOutcome = 'success' | 'empty' | 'conflict' | 'crash';
export interface SentinelPayload {
interface SentinelPayload {
outcome: SessionOutcome;
at: string;
chatId: string;

1
packages/cli/src/context/daemon/index.ts
View file

@ -1 +0,0 @@
export * from './semantic-layer-compute.js';

22
packages/cli/src/context/daemon/semantic-layer-compute.ts
View file

@ -4,21 +4,21 @@ import { URL } from 'node:url';
import { spawn } from 'node:child_process';
import type { ResolvedSemanticLayerSource, SemanticLayerQueryInput } from '../sl/types.js';
export interface KtxSemanticLayerComputeQueryResult {
interface KtxSemanticLayerComputeQueryResult {
sql: string;
dialect: string;
columns: Array<Record<string, unknown>>;
plan: Record<string, unknown>;
}
export interface KtxSemanticLayerComputeValidationResult {
interface KtxSemanticLayerComputeValidationResult {
valid: boolean;
errors: string[];
warnings: string[];
perSourceWarnings: Record<string, string[]>;
}
export interface KtxSemanticLayerSourceGenerationColumnInput {
interface KtxSemanticLayerSourceGenerationColumnInput {
name: string;
type: string;
primaryKey?: boolean;
@ -26,7 +26,7 @@ export interface KtxSemanticLayerSourceGenerationColumnInput {
comment?: string | null;
}
export interface KtxSemanticLayerSourceGenerationTableInput {
interface KtxSemanticLayerSourceGenerationTableInput {
name: string;
catalog?: string | null;
db?: string | null;
@ -34,7 +34,7 @@ export interface KtxSemanticLayerSourceGenerationTableInput {
columns: KtxSemanticLayerSourceGenerationColumnInput[];
}
export interface KtxSemanticLayerSourceGenerationLinkInput {
interface KtxSemanticLayerSourceGenerationLinkInput {
fromTable: string;
fromColumn: string;
toTable: string;
@ -42,13 +42,13 @@ export interface KtxSemanticLayerSourceGenerationLinkInput {
relationshipType: string;
}
export interface KtxSemanticLayerSourceGenerationInput {
interface KtxSemanticLayerSourceGenerationInput {
tables: KtxSemanticLayerSourceGenerationTableInput[];
links: KtxSemanticLayerSourceGenerationLinkInput[];
dialect?: string;
}
export interface KtxSemanticLayerSourceGenerationResult {
interface KtxSemanticLayerSourceGenerationResult {
sources: Array<Record<string, unknown>>;
sourceCount: number;
}
@ -75,14 +75,14 @@ export interface KtxSemanticLayerComputePort {
generateSources(input: KtxSemanticLayerSourceGenerationInput): Promise<KtxSemanticLayerSourceGenerationResult>;
}
export type KtxDaemonCommand = 'semantic-query' | 'semantic-validate' | 'semantic-generate-sources';
type KtxDaemonCommand = 'semantic-query' | 'semantic-validate' | 'semantic-generate-sources';
export type KtxDaemonJsonRunner = (
type KtxDaemonJsonRunner = (
subcommand: KtxDaemonCommand,
payload: Record<string, unknown>,
) => Promise<Record<string, unknown>>;
export type KtxDaemonHttpJsonRunner = (path: string, payload: Record<string, unknown>) => Promise<Record<string, unknown>>;
type KtxDaemonHttpJsonRunner = (path: string, payload: Record<string, unknown>) => Promise<Record<string, unknown>>;
export interface PythonSemanticLayerComputeOptions {
command?: string;
@ -92,6 +92,7 @@ export interface PythonSemanticLayerComputeOptions {
runJson?: KtxDaemonJsonRunner;
}
/** @internal */
export interface HttpSemanticLayerComputeOptions {
baseUrl: string;
requestJson?: KtxDaemonHttpJsonRunner;
@ -272,6 +273,7 @@ export function createPythonSemanticLayerComputePort(
};
}
/** @internal */
export function createHttpSemanticLayerComputePort(
options: HttpSemanticLayerComputeOptions,
): KtxSemanticLayerComputePort {

2
packages/cli/src/context/index-sync/index.ts
View file

@ -1,2 +0,0 @@
export type { ReindexOptions, ReindexScopeResult, ReindexSummary, ReindexWorkResult } from './types.js';
export { discoverReindexScopes, reindexLocalIndexes } from './reindex.js';

4
packages/cli/src/context/index-sync/reindex.test.ts
View file

@ -2,8 +2,8 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import type { KtxEmbeddingPort } from '../core/index.js';
import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../project/index.js';
import type { KtxEmbeddingPort } from '../../context/core/embedding.js';
import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../context/project/project.js';
import { SqliteKnowledgeIndex } from '../wiki/sqlite-knowledge-index.js';
import { reindexLocalIndexes } from './reindex.js';

12
packages/cli/src/context/index-sync/reindex.ts
View file

@ -1,8 +1,12 @@
import { readdir, stat } from 'node:fs/promises';
import { join, relative } from 'node:path';
import { ktxLocalStateDbPath, type KtxLocalProject } from '../project/index.js';
import { loadLocalSlSourceRecords, SlSearchService, SqliteSlSourcesIndex } from '../sl/index.js';
import { KnowledgeWikiService, SqliteKnowledgeIndex } from '../wiki/index.js';
import { ktxLocalStateDbPath } from '../../context/project/local-state-db.js';
import type { KtxLocalProject } from '../../context/project/project.js';
import { loadLocalSlSourceRecords } from '../../context/sl/local-sl.js';
import { SlSearchService } from '../../context/sl/sl-search.service.js';
import { SqliteSlSourcesIndex } from '../../context/sl/sqlite-sl-sources-index.js';
import { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js';
import { SqliteKnowledgeIndex } from '../../context/wiki/sqlite-knowledge-index.js';
import type { ReindexOptions, ReindexScopeResult, ReindexSummary, ReindexWorkResult } from './types.js';
type DiscoveredScope =
@ -41,7 +45,7 @@ async function childDirectories(path: string): Promise<string[]> {
}
}
export async function discoverReindexScopes(project: KtxLocalProject): Promise<DiscoveredScope[]> {
async function discoverReindexScopes(project: KtxLocalProject): Promise<DiscoveredScope[]> {
const scopes: DiscoveredScope[] = [];
if (await directoryExists(join(project.projectDir, 'wiki/global'))) {
scopes.push({ kind: 'wiki', scope: 'GLOBAL', scopeId: null, label: 'global' });

2
packages/cli/src/context/index-sync/types.ts
View file

@ -1,4 +1,4 @@
import type { KtxEmbeddingPort } from '../core/index.js';
import type { KtxEmbeddingPort } from '../../context/core/embedding.js';
export interface ReindexOptions {
force: boolean;

128
packages/cli/src/context/index.ts
View file

@ -1,128 +0,0 @@
export * from './agent/index.js';
export * from './core/index.js';
export * from './daemon/index.js';
export * from './ingest/index.js';
export * from './index-sync/index.js';
export * from './llm/index.js';
export type {
CaptureSession,
CaptureSignals,
MemoryAgentInput,
MemoryAgentResult,
MemoryAgentServiceDeps,
MemoryAgentSettings,
MemoryAgentSourceType,
MemoryCommitMessagePort,
MemoryConnectionPort,
MemoryFileStorePort,
MemoryKnowledgeSlRefsPort,
MemoryLockPort,
MemorySlSourceReconcilerPort,
MemoryTelemetryPort,
MemoryToolSetLike,
MemoryToolsetFactoryPort,
} from './memory/index.js';
export * from './project/index.js';
export * from './prompts/index.js';
export * from './search/index.js';
export * from './sql-analysis/index.js';
export type {
KtxColumnAnalysisResult,
KtxColumnDescriptionPromptInput,
KtxColumnEmbeddingForeignKeys,
KtxColumnEmbeddingTextInput,
KtxColumnSampleInput,
KtxColumnSampleResult,
KtxColumnSampleUpdate,
KtxColumnStatsInput,
KtxColumnStatsResult,
KtxConnectionDriver,
KtxConnectorCapabilities,
KtxCredentialEnvelope,
KtxCredentialEnvReference,
KtxCredentialFileReference,
KtxDataDictionaryColumnState,
KtxDataDictionarySampleDecision,
KtxDataDictionarySettings,
KtxDataDictionarySkipReason,
KtxDataSourceDescriptionPromptInput,
KtxDescriptionCachePort,
KtxDescriptionColumn,
KtxDescriptionColumnTable,
KtxDescriptionGenerationSettings,
KtxDescriptionGeneratorOptions,
KtxDescriptionSource,
KtxDescriptionTableInput,
KtxDescriptionUpdate,
KtxEmbeddingPort as KtxScanEmbeddingPort,
KtxEmbeddingUpdate,
KtxEnrichedColumn,
KtxEnrichedRelationship,
KtxEnrichedSchema,
KtxEnrichedTable,
KtxGenerateColumnDescriptionsInput,
KtxGenerateDataSourceDescriptionInput,
KtxGenerateTableDescriptionInput,
KtxOptionalConnectorCapabilities,
KtxProgressPort,
KtxQueryResult as KtxScanQueryResult,
KtxReadOnlyQueryInput,
KtxRelationshipEndpoint,
KtxRelationshipSource,
KtxRelationshipType,
KtxRelationshipUpdate,
KtxResolvedCredentialEnvelope,
KtxScanArtifactPaths,
KtxScanConnector,
KtxScanContext,
KtxScanDiffSummary,
KtxScanEnrichmentSummary,
KtxScanInput,
KtxScanLoggerPort,
KtxScanMetadataStore,
KtxScanMode,
KtxScanRelationshipSummary,
KtxScanReport,
KtxScanTrigger,
KtxScanWarning,
KtxScanWarningCode,
KtxSchemaColumn,
KtxSchemaDimensionType,
KtxSchemaForeignKey,
KtxSchemaScope,
KtxSchemaSnapshot,
KtxSchemaTable,
KtxSchemaTableKind,
KtxSkippedRelationship,
KtxStructuralSyncPlan,
KtxStructuralSyncStats,
KtxTableDescriptionPromptInput,
KtxTableRef,
KtxTableSampleInput,
KtxTableSampleResult,
KtxColumnTypeMapping,
} from './scan/index.js';
export {
buildKtxColumnDescriptionPrompt,
buildKtxColumnEmbeddingText,
buildKtxDataSourceDescriptionPrompt,
buildKtxTableDescriptionPrompt,
createKtxConnectorCapabilities,
defaultKtxDataDictionarySettings,
inferKtxDimensionType,
isKtxDataDictionaryCandidate,
ktxColumnTypeMappingFromNative,
KtxDescriptionGenerator,
normalizeKtxNativeType,
REDACTED_KTX_CREDENTIAL_VALUE,
redactKtxCredentialEnvelope,
redactKtxCredentialValue,
redactKtxScanMetadata,
redactKtxScanReport,
redactKtxScanWarning,
shouldKtxSampleColumnForDictionary,
} from './scan/index.js';
export * from './skills/index.js';
export * from './sl/index.js';
export * from './tools/index.js';
export * from './wiki/index.js';

2
packages/cli/src/context/ingest/action-identity.ts
View file

@ -1,4 +1,4 @@
import type { MemoryAction } from '../memory/index.js';
import type { MemoryAction } from '../../context/memory/types.js';
export function actionTargetConnectionId(action: MemoryAction, runConnectionId: string): string {
return action.target === 'sl' ? (action.targetConnectionId ?? runConnectionId) : runConnectionId;

75
packages/cli/src/context/ingest/adapters/dbt-descriptions/match-tables.test.ts
View file

@ -1,75 +0,0 @@
import { describe, expect, it } from 'vitest';
import type { DbtParsedTable } from './parse-schema.js';
import { findMatchingKtxTable, matchDbtTables, type DbtHostTableLite } from './match-tables.js';
const hostTables: DbtHostTableLite[] = [
{ id: '1', name: 'orders', catalog: 'warehouse', db: 'analytics', columns: [{ id: 'c1', name: 'id' }] },
{ id: '2', name: 'orders', catalog: 'warehouse', db: 'staging', columns: [{ id: 'c2', name: 'id' }] },
{ id: '3', name: 'customers', catalog: null, db: null, columns: [{ id: 'c3', name: 'id' }] },
];
function table(input: Partial<DbtParsedTable>): DbtParsedTable {
return {
name: 'orders',
description: null,
database: null,
schema: null,
columns: [],
resourceType: 'model',
...input,
};
}
describe('dbt descriptions table matching', () => {
it('uses schema plus name first and checks catalog when dbt database is present', () => {
expect(
findMatchingKtxTable(table({ database: 'warehouse', schema: 'analytics' }), hostTables, null)?.id,
).toBe('1');
});
it('does not fall back to name-only for source tables', () => {
expect(findMatchingKtxTable(table({ resourceType: 'source' }), hostTables, null)).toBeUndefined();
});
it('uses targetSchema for models and name-only only when unique', () => {
expect(findMatchingKtxTable(table({ resourceType: 'model' }), hostTables, 'staging')?.id).toBe('2');
expect(findMatchingKtxTable(table({ name: 'customers', resourceType: 'model' }), hostTables, null)?.id).toBe(
'3',
);
expect(findMatchingKtxTable(table({ resourceType: 'model' }), hostTables, null)).toBeUndefined();
});
it('summarizes matched columns and descriptions', () => {
const matches = matchDbtTables(
[
table({
name: 'customers',
description: 'Customers',
columns: [
{ name: 'id', description: 'Primary key', dataType: null },
{ name: 'missing', description: 'Missing', dataType: null },
],
}),
],
hostTables,
null,
);
expect(matches).toEqual([
{
dbtTable: 'customers',
dbtSchema: null,
dbtDatabase: null,
hostTableId: '3',
hostTableName: 'customers',
matched: true,
tableDescriptionAction: 'import',
tableDescriptionFound: true,
columnsToImport: 1,
columnsMatched: 1,
columnsTotal: 2,
columnDescriptionsFound: 1,
},
]);
});
});

127
packages/cli/src/context/ingest/adapters/dbt-descriptions/match-tables.ts
View file

@ -1,127 +0,0 @@
import type { DbtParsedTable } from './parse-schema.js';
export interface DbtHostTableLite {
id: string;
name: string;
catalog: string | null;
db: string | null;
columns: Array<{ id: string; name: string }>;
}
export interface DbtTableMatch {
dbtTable: string;
dbtSchema: string | null;
dbtDatabase: string | null;
hostTableId: string | null;
hostTableName: string | null;
matched: boolean;
tableDescriptionAction: 'skip' | 'import';
tableDescriptionFound: boolean;
columnsToImport: number;
columnsMatched: number;
columnsTotal: number;
columnDescriptionsFound: number;
}
export function matchDbtTables(
dbtTables: DbtParsedTable[],
hostTables: DbtHostTableLite[],
targetSchema?: string | null,
): DbtTableMatch[] {
return dbtTables.map((dbtTable) => {
const hostTable = findMatchingKtxTable(dbtTable, hostTables, targetSchema);
if (!hostTable) {
return {
dbtTable: dbtTable.name,
dbtSchema: dbtTable.schema,
dbtDatabase: dbtTable.database,
hostTableId: null,
hostTableName: null,
matched: false,
tableDescriptionAction: 'skip',
tableDescriptionFound: Boolean(dbtTable.description),
columnsToImport: 0,
columnsMatched: 0,
columnsTotal: dbtTable.columns.length,
columnDescriptionsFound: dbtTable.columns.filter((column) => Boolean(column.description)).length,
};
}
const analysis = analyzeColumns(dbtTable, hostTable);
return {
dbtTable: dbtTable.name,
dbtSchema: dbtTable.schema,
dbtDatabase: dbtTable.database,
hostTableId: hostTable.id,
hostTableName: hostTable.name,
matched: true,
tableDescriptionAction: dbtTable.description ? 'import' : 'skip',
tableDescriptionFound: Boolean(dbtTable.description),
...analysis,
};
});
}
export function findMatchingKtxTable(
dbtTable: DbtParsedTable,
hostTables: DbtHostTableLite[],
targetSchema?: string | null,
): DbtHostTableLite | undefined {
const dbtName = dbtTable.name.toLowerCase();
const effectiveSchema = dbtTable.schema ?? targetSchema ?? null;
if (effectiveSchema) {
const strictMatch = hostTables.find((table) => {
const nameMatches = table.name.toLowerCase() === dbtName;
const schemaMatches = table.db?.toLowerCase() === effectiveSchema.toLowerCase();
if (!nameMatches || !schemaMatches) {
return false;
}
if (dbtTable.database && table.catalog) {
return table.catalog.toLowerCase() === dbtTable.database.toLowerCase();
}
return true;
});
if (strictMatch) {
return strictMatch;
}
}
if (dbtTable.resourceType === 'source') {
return undefined;
}
const nameMatches = hostTables.filter((table) => table.name.toLowerCase() === dbtName);
return nameMatches.length === 1 ? nameMatches[0] : undefined;
}
function analyzeColumns(
dbtTable: DbtParsedTable,
hostTable: DbtHostTableLite,
): Pick<DbtTableMatch, 'columnsToImport' | 'columnsMatched' | 'columnsTotal' | 'columnDescriptionsFound'> {
let columnsToImport = 0;
let columnsMatched = 0;
let columnDescriptionsFound = 0;
for (const dbtColumn of dbtTable.columns) {
const hostColumn = hostTable.columns.find(
(column) => column.name.toLowerCase() === dbtColumn.name.toLowerCase(),
);
if (!hostColumn) {
continue;
}
columnsMatched++;
if (dbtColumn.description) {
columnDescriptionsFound++;
columnsToImport++;
}
}
return {
columnsToImport,
columnsMatched,
columnsTotal: dbtTable.columns.length,
columnDescriptionsFound,
};
}

62
packages/cli/src/context/ingest/adapters/dbt-descriptions/merge-semantic-model-tables.test.ts
View file

@ -1,62 +0,0 @@
import { describe, expect, it } from 'vitest';
import type { ParsedSemanticModel } from '../metricflow/deep-parse.js';
import { mergeSemanticModelTables } from './merge-semantic-model-tables.js';
import type { DbtSchemaParseResult } from './parse-schema.js';
const semanticModel: ParsedSemanticModel = {
name: 'orders_semantic',
description: 'Order facts',
modelRef: 'fct_orders',
dimensions: [
{ name: 'status', column: 'status', type: 'categorical', description: 'Order status' },
{ name: 'ordered_at', column: 'ordered_at', type: 'time' },
],
measures: [],
entities: [],
defaultTimeDimension: null,
};
describe('mergeSemanticModelTables', () => {
it('adds missing MetricFlow model refs as dbt model tables', () => {
const input: DbtSchemaParseResult = { projectName: null, dbtVersion: null, tables: [], relationships: [] };
expect(mergeSemanticModelTables(input, [semanticModel])).toEqual({
projectName: null,
dbtVersion: null,
relationships: [],
tables: [
{
name: 'fct_orders',
description: 'Order facts',
database: null,
schema: null,
resourceType: 'model',
columns: [
{ name: 'status', description: 'Order status', dataType: null },
{ name: 'ordered_at', description: null, dataType: 'TIMESTAMP' },
],
},
],
});
});
it('does not add a duplicate table when schema parsing already found the model ref', () => {
const input: DbtSchemaParseResult = {
projectName: null,
dbtVersion: null,
relationships: [],
tables: [
{
name: 'FCT_ORDERS',
description: 'Existing',
database: null,
schema: null,
resourceType: 'model',
columns: [],
},
],
};
expect(mergeSemanticModelTables(input, [semanticModel]).tables).toHaveLength(1);
});
});

37
packages/cli/src/context/ingest/adapters/dbt-descriptions/merge-semantic-model-tables.ts
View file

@ -1,37 +0,0 @@
import type { ParsedSemanticModel } from '../metricflow/deep-parse.js';
import type { DbtSchemaParseResult } from './parse-schema.js';
export function mergeSemanticModelTables(
parseResult: DbtSchemaParseResult,
semanticModels: ParsedSemanticModel[],
): DbtSchemaParseResult {
const merged: DbtSchemaParseResult = {
...parseResult,
tables: [...parseResult.tables],
relationships: [...parseResult.relationships],
};
const existingTableNames = new Set(merged.tables.map((table) => table.name.toLowerCase()));
for (const model of semanticModels) {
const tableName = model.modelRef;
if (existingTableNames.has(tableName.toLowerCase())) {
continue;
}
merged.tables.push({
name: tableName,
description: model.description,
database: null,
schema: null,
columns: model.dimensions.map((dimension) => ({
name: dimension.column,
description: dimension.description ?? null,
dataType: dimension.type === 'time' ? 'TIMESTAMP' : null,
})),
resourceType: 'model',
});
existingTableNames.add(tableName.toLowerCase());
}
return merged;
}

20
packages/cli/src/context/ingest/adapters/dbt-descriptions/parse-schema.ts
View file

@ -1,9 +1,9 @@
import { createHash } from 'node:crypto';
import { parse as parseYaml } from 'yaml';
import { type KtxLogger, noopLogger } from '../../../core/index.js';
import { type KtxLogger, noopLogger } from '../../../../context/core/config.js';
import { resolveJinjaVariables } from '../../dbt-shared/project-vars.js';
export interface DbtParsedColumn {
interface DbtParsedColumn {
name: string;
description: string | null;
dataType: string | null;
@ -12,20 +12,20 @@ export interface DbtParsedColumn {
enumValuesDbt?: string[];
}
export interface DbtDataTestRef {
interface DbtDataTestRef {
name: string;
package: string;
kwargs?: Record<string, unknown>;
}
export interface DbtColumnConstraints {
interface DbtColumnConstraints {
dbt: {
not_null?: boolean;
unique?: boolean;
};
}
export interface DbtParsedRelationship {
interface DbtParsedRelationship {
fromTable: string;
fromColumn: string;
toTable: string;
@ -35,7 +35,7 @@ export interface DbtParsedRelationship {
description?: string;
}
export interface DbtParsedTable {
interface DbtParsedTable {
name: string;
description: string | null;
database: string | null;
@ -126,6 +126,7 @@ type DbtSchemaDataTest =
[key: string]: unknown;
};
/** @internal */
export function parseDbtSchemaFile(content: string, options: ParseDbtSchemaOptions = {}): DbtSchemaParseResult {
return new DbtSchemaParser(options.logger ?? noopLogger).parseFile(content, options);
}
@ -138,13 +139,6 @@ export function parseDbtSchemaFiles(
return new DbtSchemaParser(options.logger ?? noopLogger).parseFiles(files, variables, options.projectName ?? null);
}
export function computeDbtSchemaHash(files: DbtSchemaFile[]): string {
const combined = [...files]
.sort((a, b) => a.path.localeCompare(b.path))
.map((file) => `${file.path}:${file.content}`)
.join('\n');
return createHash('sha256').update(combined).digest('hex').substring(0, 16);
}
class DbtSchemaParser {
constructor(private readonly logger: KtxLogger) {}

102
packages/cli/src/context/ingest/adapters/dbt-descriptions/to-description-updates.test.ts
View file

@ -1,102 +0,0 @@
import { describe, expect, it } from 'vitest';
import type { DbtSchemaParseResult } from './parse-schema.js';
import { toDescriptionUpdates } from './to-description-updates.js';
import type { DbtHostTableLite } from './match-tables.js';
const hostTables: DbtHostTableLite[] = [
{
id: '1',
name: 'orders',
catalog: 'warehouse',
db: 'analytics',
columns: [
{ id: 'c1', name: 'id' },
{ id: 'c2', name: 'amount' },
],
},
];
function parseResult(description: string | null, columnDescription: string | null): DbtSchemaParseResult {
return {
projectName: null,
dbtVersion: null,
relationships: [],
tables: [
{
name: 'orders',
description,
database: 'warehouse',
schema: 'analytics',
resourceType: 'model',
columns: [
{ name: 'id', description: columnDescription, dataType: null },
{ name: 'missing', description: 'not imported', dataType: null },
],
},
],
};
}
describe('dbt descriptions update payloads', () => {
it('emits dbt writes and matching ai invalidations when descriptions exist', () => {
expect(
toDescriptionUpdates({
connectionId: 'conn-1',
parseResult: parseResult('Orders table', 'Primary key'),
hostTables,
targetSchema: null,
}),
).toEqual({
dbt: [
{
connectionId: 'conn-1',
table: { catalog: 'warehouse', db: 'analytics', name: 'orders' },
source: 'dbt',
tableDescription: 'Orders table',
columnDescriptions: { id: 'Primary key' },
},
],
aiInvalidations: [
{
connectionId: 'conn-1',
table: { catalog: 'warehouse', db: 'analytics', name: 'orders' },
source: 'ai',
},
],
});
});
it('does not emit spurious dbt writes or ai invalidations when no descriptions exist', () => {
expect(
toDescriptionUpdates({
connectionId: 'conn-1',
parseResult: parseResult(null, null),
hostTables,
targetSchema: null,
}),
).toEqual({ dbt: [], aiInvalidations: [] });
});
it('emits ai invalidation without a dbt description write when only structural metadata exists', () => {
const result = parseResult(null, null);
result.tables[0]!.tagsDbt = ['finance'];
expect(
toDescriptionUpdates({
connectionId: 'conn-1',
parseResult: result,
hostTables,
targetSchema: null,
}),
).toEqual({
dbt: [],
aiInvalidations: [
{
connectionId: 'conn-1',
table: { catalog: 'warehouse', db: 'analytics', name: 'orders' },
source: 'ai',
},
],
});
});
});

70
packages/cli/src/context/ingest/adapters/dbt-descriptions/to-description-updates.ts
View file

@ -1,70 +0,0 @@
import type { KtxDescriptionUpdate } from '../../../scan/enrichment-types.js';
import { findMatchingKtxTable, type DbtHostTableLite } from './match-tables.js';
import type { DbtSchemaParseResult } from './parse-schema.js';
export interface DbtDescriptionUpdates {
dbt: KtxDescriptionUpdate[];
aiInvalidations: KtxDescriptionUpdate[];
}
export function toDescriptionUpdates(input: {
connectionId: string;
parseResult: DbtSchemaParseResult;
hostTables: DbtHostTableLite[];
targetSchema: string | null;
}): DbtDescriptionUpdates {
const dbt: KtxDescriptionUpdate[] = [];
const aiInvalidations: KtxDescriptionUpdate[] = [];
for (const dbtTable of input.parseResult.tables) {
const hostTable = findMatchingKtxTable(dbtTable, input.hostTables, input.targetSchema);
if (!hostTable) {
continue;
}
const tableDescription = dbtTable.description ?? undefined;
const columnDescriptions: Record<string, string | null> = {};
for (const dbtColumn of dbtTable.columns) {
if (!dbtColumn.description) {
continue;
}
const hostColumn = hostTable.columns.find(
(column) => column.name.toLowerCase() === dbtColumn.name.toLowerCase(),
);
if (hostColumn) {
columnDescriptions[hostColumn.name] = dbtColumn.description;
}
}
const hasColumnDescriptions = Object.keys(columnDescriptions).length > 0;
const hasDescriptionChange = tableDescription !== undefined || hasColumnDescriptions;
const hasMetadataChange =
!!dbtTable.tagsDbt?.length ||
dbtTable.freshnessDbt !== undefined ||
dbtTable.columns.some(
(column) => column.constraints !== undefined || !!column.enumValuesDbt?.length || !!column.dataTests?.length,
);
if (!hasDescriptionChange && !hasMetadataChange) {
continue;
}
const tableRef = { catalog: hostTable.catalog, db: hostTable.db, name: hostTable.name };
if (hasDescriptionChange) {
dbt.push({
connectionId: input.connectionId,
table: tableRef,
source: 'dbt',
...(tableDescription !== undefined ? { tableDescription } : {}),
...(hasColumnDescriptions ? { columnDescriptions } : {}),
});
}
aiInvalidations.push({
connectionId: input.connectionId,
table: tableRef,
source: 'ai',
});
}
return { dbt, aiInvalidations };
}

70
packages/cli/src/context/ingest/adapters/dbt-descriptions/to-metadata-updates.test.ts
View file

@ -1,70 +0,0 @@
import { describe, expect, it } from 'vitest';
import { toMetadataUpdates } from './to-metadata-updates.js';
describe('toMetadataUpdates', () => {
it('emits source-keyed dbt metadata updates for matched tables and columns', () => {
const updates = toMetadataUpdates({
connectionId: 'conn_1',
targetSchema: 'analytics',
hostTables: [
{
id: 'orders-id',
name: 'orders',
catalog: 'warehouse',
db: 'analytics',
columns: [
{ id: 'status-id', name: 'status' },
{ id: 'created-id', name: 'created_at' },
],
},
],
parseResult: {
projectName: null,
dbtVersion: null,
relationships: [],
tables: [
{
name: 'orders',
description: null,
database: 'warehouse',
schema: 'analytics',
resourceType: 'model',
tagsDbt: ['finance'],
freshnessDbt: { loadedAtField: 'created_at' },
columns: [
{
name: 'status',
description: null,
dataType: null,
enumValuesDbt: ['placed', 'shipped'],
constraints: { dbt: { not_null: true } },
dataTests: [{ name: 'accepted_values', package: 'dbt', kwargs: { values: ['placed', 'shipped'] } }],
},
],
},
],
},
});
expect(updates).toEqual([
{
connectionId: 'conn_1',
table: { catalog: 'warehouse', db: 'analytics', name: 'orders' },
source: 'dbt',
tableFields: {
tags: ['finance'],
freshness: { loaded_at_field: 'created_at' },
},
columnFields: {
status: {
constraints: { not_null: true },
enum_values: ['placed', 'shipped'],
tests: [
{ name: 'accepted_values', package: 'dbt', kwargs: { values: ['placed', 'shipped'] } },
],
},
},
},
]);
});
});

74
packages/cli/src/context/ingest/adapters/dbt-descriptions/to-metadata-updates.ts
View file

@ -1,74 +0,0 @@
import type { KtxMetadataUpdate } from '../../../scan/enrichment-types.js';
import { findMatchingKtxTable, type DbtHostTableLite } from './match-tables.js';
import type { DbtSchemaParseResult } from './parse-schema.js';
export function toMetadataUpdates(input: {
connectionId: string;
parseResult: DbtSchemaParseResult;
hostTables: DbtHostTableLite[];
targetSchema: string | null;
}): KtxMetadataUpdate[] {
const updates: KtxMetadataUpdate[] = [];
for (const dbtTable of input.parseResult.tables) {
const hostTable = findMatchingKtxTable(dbtTable, input.hostTables, input.targetSchema);
if (!hostTable) {
continue;
}
const tableFields: Record<string, unknown> = {};
if (dbtTable.tagsDbt?.length) {
tableFields.tags = dbtTable.tagsDbt;
}
if (dbtTable.freshnessDbt) {
tableFields.freshness = {
...(dbtTable.freshnessDbt.raw !== undefined ? { raw: dbtTable.freshnessDbt.raw } : {}),
...(dbtTable.freshnessDbt.loadedAtField !== undefined
? { loaded_at_field: dbtTable.freshnessDbt.loadedAtField }
: {}),
};
}
const columnFields: Record<string, Record<string, unknown>> = {};
for (const dbtColumn of dbtTable.columns) {
const hostColumn = hostTable.columns.find(
(column) => column.name.toLowerCase() === dbtColumn.name.toLowerCase(),
);
if (!hostColumn) {
continue;
}
const fields: Record<string, unknown> = {};
if (dbtColumn.constraints) {
fields.constraints = dbtColumn.constraints.dbt;
}
if (dbtColumn.enumValuesDbt?.length) {
fields.enum_values = dbtColumn.enumValuesDbt;
}
if (dbtColumn.dataTests?.length) {
fields.tests = dbtColumn.dataTests.map((test) => ({
name: test.name,
package: test.package,
...(test.kwargs ? { kwargs: test.kwargs } : {}),
}));
}
if (Object.keys(fields).length > 0) {
columnFields[hostColumn.name] = fields;
}
}
if (Object.keys(tableFields).length === 0 && Object.keys(columnFields).length === 0) {
continue;
}
updates.push({
connectionId: input.connectionId,
table: { catalog: hostTable.catalog, db: hostTable.db, name: hostTable.name },
source: 'dbt',
...(Object.keys(tableFields).length > 0 ? { tableFields } : {}),
...(Object.keys(columnFields).length > 0 ? { columnFields } : {}),
});
}
return updates;
}

62
packages/cli/src/context/ingest/adapters/dbt-descriptions/to-relationship-updates.test.ts
View file

@ -1,62 +0,0 @@
import { describe, expect, it } from 'vitest';
import type { DbtHostTableLite } from './match-tables.js';
import type { DbtSchemaParseResult } from './parse-schema.js';
import { toRelationshipUpdates } from './to-relationship-updates.js';
const DBT_SYSTEM_EMAIL = ['system@kae', 'lio.dev'].join('');
const hostTables: DbtHostTableLite[] = [
{
id: '1',
name: 'orders',
catalog: 'warehouse',
db: 'analytics',
columns: [{ id: 'c1', name: 'customer_id' }],
},
{
id: '2',
name: 'customers',
catalog: 'warehouse',
db: 'staging',
columns: [{ id: 'c2', name: 'id' }],
},
];
const parseResult: DbtSchemaParseResult = {
projectName: null,
dbtVersion: null,
tables: [],
relationships: [
{
fromTable: 'orders',
fromColumn: 'customer_id',
toTable: 'customers',
toColumn: 'id',
fromSchema: 'analytics',
toSchema: 'analytics',
description: 'schema intentionally differs from the host customers table',
},
{ fromTable: 'orders', fromColumn: 'missing', toTable: 'customers', toColumn: 'id' },
{ fromTable: 'orders', fromColumn: 'customer_id', toTable: 'missing_table', toColumn: 'id' },
],
};
describe('dbt relationship update payloads', () => {
it('validates relationships using the current name-only matching behavior and dbt provenance', () => {
expect(toRelationshipUpdates({ connectionId: 'conn-1', parseResult, hostTables })).toEqual({
joins: [
{
connectionId: 'conn-1',
fromTable: 'orders',
fromColumns: ['customer_id'],
toTable: 'customers',
toColumns: ['id'],
relationship: 'many_to_one',
author: 'dbt',
authorEmail: DBT_SYSTEM_EMAIL,
},
],
skippedNoMatch: 2,
});
});
});

57
packages/cli/src/context/ingest/adapters/dbt-descriptions/to-relationship-updates.ts
View file

@ -1,57 +0,0 @@
import type { KtxJoinUpdate } from '../../../scan/enrichment-types.js';
import type { DbtHostTableLite } from './match-tables.js';
import type { DbtSchemaParseResult } from './parse-schema.js';
export interface DbtRelationshipUpdates {
joins: KtxJoinUpdate[];
skippedNoMatch: number;
}
const DBT_SYSTEM_EMAIL = ['system@kae', 'lio.dev'].join('');
export function toRelationshipUpdates(input: {
connectionId: string;
parseResult: DbtSchemaParseResult;
hostTables: DbtHostTableLite[];
}): DbtRelationshipUpdates {
const tablesByName = new Map<string, DbtHostTableLite>();
for (const table of input.hostTables) {
tablesByName.set(table.name.toLowerCase(), table);
}
const joins: KtxJoinUpdate[] = [];
let skippedNoMatch = 0;
for (const relationship of input.parseResult.relationships) {
const fromTable = tablesByName.get(relationship.fromTable.toLowerCase());
const toTable = tablesByName.get(relationship.toTable.toLowerCase());
if (!fromTable || !toTable) {
skippedNoMatch++;
continue;
}
const fromColumn = fromTable.columns.find(
(column) => column.name.toLowerCase() === relationship.fromColumn.toLowerCase(),
);
const toColumn = toTable.columns.find(
(column) => column.name.toLowerCase() === relationship.toColumn.toLowerCase(),
);
if (!fromColumn || !toColumn) {
skippedNoMatch++;
continue;
}
joins.push({
connectionId: input.connectionId,
fromTable: fromTable.name,
fromColumns: [fromColumn.name],
toTable: toTable.name,
toColumns: [toColumn.name],
relationship: 'many_to_one',
author: 'dbt',
authorEmail: DBT_SYSTEM_EMAIL,
});
}
return { joins, skippedNoMatch };
}

410
packages/cli/src/context/ingest/adapters/dbt-extraction-golden-parity.test.ts
View file

@ -1,410 +0,0 @@
import { describe, expect, it } from 'vitest';
import { type DbtHostTableLite, matchDbtTables } from './dbt-descriptions/match-tables.js';
import { mergeSemanticModelTables } from './dbt-descriptions/merge-semantic-model-tables.js';
import { parseDbtSchemaFiles } from './dbt-descriptions/parse-schema.js';
import { toDescriptionUpdates } from './dbt-descriptions/to-description-updates.js';
import { toRelationshipUpdates } from './dbt-descriptions/to-relationship-updates.js';
import { parseMetricflowFiles } from './metricflow/deep-parse.js';
import { mapCrossModelMetricToSource, mapSemanticModelToSource } from './metricflow/semantic-models.js';
const DBT_SYSTEM_EMAIL = ['system@kae', 'lio.dev'].join('');
const metricflowYaml = `
semantic_models:
- name: orders_semantic
description: MetricFlow order facts
model: ref('fct_orders')
defaults:
agg_time_dimension: ordered_at
entities:
- name: customer
type: foreign
expr: customer_id
description: Customer relationship
dimensions:
- name: status
type: categorical
expr: status
description: Order status
- name: ordered_at
type: time
expr: ordered_at
measures:
- name: total_revenue
agg: sum
expr: amount
description: Revenue
- name: customers_semantic
description: Customer dimension
model: ref('dim_customers')
entities:
- name: customer
type: primary
expr: id
dimensions:
- name: country
type: categorical
expr: country
description: Customer country
measures:
- name: customer_count
agg: count
expr: id
description: Customer count
metrics:
- name: total_revenue
type: simple
type_params:
measure: total_revenue
- name: customer_count
type: simple
type_params:
measure: customer_count
- name: revenue_per_customer
description: Revenue per customer
type: derived
type_params:
expr: total_revenue / NULLIF(customer_count, 0)
metrics:
- name: total_revenue
alias: total_revenue
- name: customer_count
alias: customer_count
`;
const schemaYaml = `
version: 2
sources:
- name: raw
database: warehouse
schema: landing
tables:
- name: customers
identifier: dim_customers
description: Raw customer dimension
columns:
- name: id
description: Customer primary key
- name: country
description: Country name
models:
- name: "{{ var('orders_model', 'fct_orders') }}"
schema: "{{ var('mart_schema', 'analytics') }}"
description: Modeled orders
columns:
- name: customer_id
description: Linked customer id
tests:
- relationships:
to: ref('dim_customers')
field: id
- name: status
description: Order status
- name: amount
description: Gross amount
`;
const hostTables: DbtHostTableLite[] = [
{
id: 'orders-table',
name: 'fct_orders',
catalog: 'warehouse',
db: 'analytics',
columns: [
{ id: 'orders-customer-id', name: 'customer_id' },
{ id: 'orders-status', name: 'status' },
{ id: 'orders-amount', name: 'amount' },
{ id: 'orders-ordered-at', name: 'ordered_at' },
],
},
{
id: 'customers-table',
name: 'dim_customers',
catalog: 'warehouse',
db: 'landing',
columns: [
{ id: 'customers-id', name: 'id' },
{ id: 'customers-country', name: 'country' },
],
},
];
describe('dbt extraction golden parity fixture', () => {
it('freezes the relocated MetricFlow and dbt-description contract together', () => {
const metricflow = parseMetricflowFiles([{ path: 'semantic_models/orders.yml', content: metricflowYaml }]);
expect(metricflow).toEqual({
semanticModels: [
{
name: 'orders_semantic',
description: 'MetricFlow order facts',
modelRef: 'fct_orders',
dimensions: [
{
name: 'status',
column: 'status',
type: 'string',
label: 'Status',
description: 'Order status',
},
{
name: 'ordered_at',
column: 'ordered_at',
type: 'time',
label: 'Ordered At',
description: undefined,
},
],
measures: [
{
type: 'simple',
name: 'total_revenue',
column: 'amount',
aggregation: 'sum',
label: 'Total Revenue',
description: 'Revenue',
},
],
entities: [{ name: 'customer', type: 'foreign', expr: 'customer_id', description: 'Customer relationship' }],
defaultTimeDimension: 'ordered_at',
},
{
name: 'customers_semantic',
description: 'Customer dimension',
modelRef: 'dim_customers',
dimensions: [
{
name: 'country',
column: 'country',
type: 'string',
label: 'Country',
description: 'Customer country',
},
],
measures: [
{
type: 'simple',
name: 'customer_count',
column: 'id',
aggregation: 'count',
label: 'Customer Count',
description: 'Customer count',
},
],
entities: [{ name: 'customer', type: 'primary', expr: 'id' }],
defaultTimeDimension: null,
},
],
crossModelMetrics: [
{
name: 'revenue_per_customer',
label: null,
description: 'Revenue per customer',
type: 'derived',
expr: 'total_revenue / NULLIF(customer_count, 0)',
dependsOn: [
{ metricName: 'orders_semantic', alias: 'total_revenue' },
{ metricName: 'customers_semantic', alias: 'customer_count' },
],
filter: null,
},
],
relationships: [
{
fromTable: 'fct_orders',
fromColumn: 'customer_id',
toTable: 'dim_customers',
toColumn: 'id',
description: 'Customer relationship',
},
],
warnings: [],
});
expect(mapSemanticModelToSource(metricflow.semanticModels[0], 'analytics.fct_orders')).toEqual({
name: 'fct-orders',
table: 'analytics.fct_orders',
grain: ['status', 'ordered_at'],
columns: [
{ name: 'status', type: 'string', description: 'Order status' },
{ name: 'ordered_at', type: 'time' },
],
measures: [
{
name: 'total_revenue',
expr: 'sum(amount)',
description: 'Revenue',
},
],
joins: [],
descriptions: { dbt: 'MetricFlow order facts' },
});
expect(mapCrossModelMetricToSource(metricflow.crossModelMetrics[0])).toEqual({
name: 'revenue-per-customer',
sql: 'total_revenue / NULLIF(customer_count, 0)',
descriptions: { dbt: 'Revenue per customer' },
grain: [],
columns: [],
measures: [
{
name: 'revenue_per_customer',
expr: 'total_revenue / NULLIF(customer_count, 0)',
description: 'Revenue per customer',
},
],
joins: [],
});
const schema = parseDbtSchemaFiles(
[{ path: 'models/schema.yml', content: schemaYaml }],
new Map([
['orders_model', 'fct_orders'],
['mart_schema', 'analytics'],
]),
);
const merged = mergeSemanticModelTables(schema, metricflow.semanticModels);
expect(merged).toEqual({
projectName: null,
dbtVersion: null,
tables: [
{
name: 'dim_customers',
description: 'Raw customer dimension',
database: 'warehouse',
schema: 'landing',
columns: [
{ name: 'id', description: 'Customer primary key', dataType: null },
{ name: 'country', description: 'Country name', dataType: null },
],
resourceType: 'source',
},
{
name: 'fct_orders',
description: 'Modeled orders',
database: null,
schema: 'analytics',
columns: [
{
name: 'customer_id',
description: 'Linked customer id',
dataType: null,
dataTests: [
{
name: 'relationships',
package: 'dbt',
kwargs: { to: "ref('dim_customers')", field: 'id' },
},
],
},
{ name: 'status', description: 'Order status', dataType: null },
{ name: 'amount', description: 'Gross amount', dataType: null },
],
resourceType: 'model',
},
],
relationships: [
{
fromTable: 'fct_orders',
fromColumn: 'customer_id',
toTable: 'dim_customers',
toColumn: 'id',
fromSchema: 'analytics',
},
],
});
expect(matchDbtTables(merged.tables, hostTables, 'analytics')).toEqual([
{
dbtTable: 'dim_customers',
dbtSchema: 'landing',
dbtDatabase: 'warehouse',
hostTableId: 'customers-table',
hostTableName: 'dim_customers',
matched: true,
tableDescriptionAction: 'import',
tableDescriptionFound: true,
columnsToImport: 2,
columnsMatched: 2,
columnsTotal: 2,
columnDescriptionsFound: 2,
},
{
dbtTable: 'fct_orders',
dbtSchema: 'analytics',
dbtDatabase: null,
hostTableId: 'orders-table',
hostTableName: 'fct_orders',
matched: true,
tableDescriptionAction: 'import',
tableDescriptionFound: true,
columnsToImport: 3,
columnsMatched: 3,
columnsTotal: 3,
columnDescriptionsFound: 3,
},
]);
expect(
toDescriptionUpdates({
connectionId: 'warehouse-1',
parseResult: merged,
hostTables,
targetSchema: 'analytics',
}),
).toEqual({
dbt: [
{
connectionId: 'warehouse-1',
table: { catalog: 'warehouse', db: 'landing', name: 'dim_customers' },
source: 'dbt',
tableDescription: 'Raw customer dimension',
columnDescriptions: {
id: 'Customer primary key',
country: 'Country name',
},
},
{
connectionId: 'warehouse-1',
table: { catalog: 'warehouse', db: 'analytics', name: 'fct_orders' },
source: 'dbt',
tableDescription: 'Modeled orders',
columnDescriptions: {
customer_id: 'Linked customer id',
status: 'Order status',
amount: 'Gross amount',
},
},
],
aiInvalidations: [
{
connectionId: 'warehouse-1',
table: { catalog: 'warehouse', db: 'landing', name: 'dim_customers' },
source: 'ai',
},
{
connectionId: 'warehouse-1',
table: { catalog: 'warehouse', db: 'analytics', name: 'fct_orders' },
source: 'ai',
},
],
});
expect(toRelationshipUpdates({ connectionId: 'warehouse-1', parseResult: merged, hostTables })).toEqual({
joins: [
{
connectionId: 'warehouse-1',
fromTable: 'fct_orders',
fromColumns: ['customer_id'],
toTable: 'dim_customers',
toColumns: ['id'],
relationship: 'many_to_one',
author: 'dbt',
authorEmail: DBT_SYSTEM_EMAIL,
},
],
skippedNoMatch: 0,
});
});
});

4
packages/cli/src/context/ingest/adapters/historic-sql/evidence.ts
View file

@ -9,6 +9,7 @@ function safeEvidenceSegment(value: string): string {
return segment;
}
/** @internal */
export const historicSqlTableUsageEvidenceSchema = z.object({
kind: z.literal('table_usage'),
connectionId: z.string().min(1),
@ -16,15 +17,14 @@ export const historicSqlTableUsageEvidenceSchema = z.object({
rawPath: z.string().min(1),
usage: tableUsageOutputSchema,
});
export type HistoricSqlTableUsageEvidence = z.infer<typeof historicSqlTableUsageEvidenceSchema>;
/** @internal */
export const historicSqlPatternEvidenceSchema = z.object({
kind: z.literal('pattern'),
connectionId: z.string().min(1),
rawPath: z.string().min(1),
pattern: patternOutputSchema,
});
export type HistoricSqlPatternEvidence = z.infer<typeof historicSqlPatternEvidenceSchema>;
export const historicSqlEvidenceEnvelopeSchema = z.discriminatedUnion('kind', [
historicSqlTableUsageEvidenceSchema,

2
packages/cli/src/context/ingest/adapters/historic-sql/historic-sql.adapter.test.ts
View file

@ -2,7 +2,7 @@ import { mkdtemp } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { describe, expect, it } from 'vitest';
import type { SqlAnalysisPort } from '../../../sql-analysis/index.js';
import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js';
import type { SourceAdapter } from '../../types.js';
import { HistoricSqlSourceAdapter } from './historic-sql.adapter.js';
import type { HistoricSqlReader } from './types.js';

11
packages/cli/src/context/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts
View file

@ -2,14 +2,9 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import YAML from 'yaml';
import type { AgentRunnerPort, RunLoopParams } from '../../../llm/index.js';
import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../../project/index.js';
import {
type SqlAnalysisBatchItem,
type SqlAnalysisBatchResult,
type SqlAnalysisDialect,
type SqlAnalysisPort,
} from '../../../sql-analysis/index.js';
import type { AgentRunnerPort, RunLoopParams } from '../../../../context/llm/runtime-port.js';
import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../../../../context/project/project.js';
import type { SqlAnalysisBatchItem, SqlAnalysisBatchResult, SqlAnalysisDialect, SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js';
import { searchLocalSlSources } from '../../../sl/local-sl.js';
import { searchLocalKnowledgePages } from '../../../wiki/local-knowledge.js';
import { runLocalIngest } from '../../local-ingest.js';

2
packages/cli/src/context/ingest/adapters/historic-sql/projection.ts
View file

@ -1,7 +1,7 @@
import { access, mkdir, readdir, readFile, rename, writeFile } from 'node:fs/promises';
import { dirname, join, relative } from 'node:path';
import YAML from 'yaml';
import type { MemoryAction } from '../../../memory/index.js';
import type { MemoryAction } from '../../../../context/memory/types.js';
import { rawSourcesDirForSync } from '../../raw-sources-paths.js';
import type { FinalizationOverrideReplay } from '../../types.js';
import { mergeUsagePreservingExternal } from '../live-database/manifest.js';

2
packages/cli/src/context/ingest/adapters/historic-sql/skill-schemas.ts
View file

@ -26,6 +26,6 @@ export const patternOutputSchema = z.object({
slRefs: z.array(z.string()),
constituentTemplateIds: z.array(z.string()),
});
export type PatternOutput = z.infer<typeof patternOutputSchema>;
/** @internal */
export const patternsArraySchema = z.array(patternOutputSchema);

2
packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.test.ts
View file

@ -2,7 +2,7 @@ import { mkdtemp, readFile, readdir } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { describe, expect, it, vi } from 'vitest';
import type { SqlAnalysisPort } from '../../../sql-analysis/index.js';
import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js';
import { stageHistoricSqlAggregatedSnapshot } from './stage-unified.js';
import type { AggregatedTemplate, HistoricSqlReader } from './types.js';

2
packages/cli/src/context/ingest/adapters/historic-sql/stage-unified.ts
View file

@ -1,6 +1,6 @@
import { mkdir, writeFile } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import type { SqlAnalysisPort } from '../../../sql-analysis/index.js';
import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js';
import {
bucketDistinctUsers,
bucketErrorRate,

5
packages/cli/src/context/ingest/adapters/historic-sql/types.ts
View file

@ -1,5 +1,5 @@
import { z } from 'zod';
import type { SqlAnalysisPort } from '../../../sql-analysis/index.js';
import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js';
export const HISTORIC_SQL_SOURCE_KEY = 'historic-sql' as const;
@ -115,9 +115,8 @@ export const stagedManifestSchema = z.object({
probeWarnings: z.array(z.string()),
staleArchiveAfterDays: z.number().int().positive().default(90),
});
export type StagedManifest = z.infer<typeof stagedManifestSchema>;
export interface HistoricSqlProbeResult {
interface HistoricSqlProbeResult {
warnings: string[];
info?: string[];
}

4
packages/cli/src/context/ingest/adapters/live-database/daemon-introspection.ts
View file

@ -8,9 +8,9 @@ import type { KtxSchemaColumn, KtxSchemaForeignKey, KtxSchemaSnapshot, KtxSchema
import { inferKtxDimensionType, normalizeKtxNativeType } from '../../../scan/type-normalization.js';
import type { LiveDatabaseIntrospectionPort } from './types.js';
export type KtxDaemonDatabaseIntrospectionCommand = 'database-introspect';
type KtxDaemonDatabaseIntrospectionCommand = 'database-introspect';
export type KtxDaemonDatabaseJsonRunner = (
type KtxDaemonDatabaseJsonRunner = (
subcommand: KtxDaemonDatabaseIntrospectionCommand,
payload: Record<string, unknown>,
) => Promise<Record<string, unknown>>;

136
packages/cli/src/context/ingest/adapters/live-database/extracted-schema.test.ts
View file

@ -1,136 +0,0 @@
import { describe, expect, it } from 'vitest';
import type { KtxSchemaSnapshot } from '../../../scan/types.js';
import { buildLiveDatabaseTableNaturalKey, ktxSchemaSnapshotToExtractedSchema } from './extracted-schema.js';
function snapshot(): KtxSchemaSnapshot {
return {
connectionId: 'conn-1',
driver: 'postgres',
extractedAt: '2026-04-27T00:00:00.000Z',
scope: { schemas: ['public'] },
metadata: { driver: 'postgres' },
tables: [
{
name: 'orders',
catalog: null,
db: 'public',
kind: 'table',
comment: 'Orders placed by customers',
estimatedRows: null,
columns: [
{
name: 'id',
nativeType: 'integer',
normalizedType: 'integer',
dimensionType: 'number',
nullable: false,
primaryKey: true,
comment: 'Primary key',
},
{
name: 'customer_id',
nativeType: 'integer',
normalizedType: 'integer',
dimensionType: 'number',
nullable: false,
primaryKey: false,
comment: null,
},
],
foreignKeys: [
{
fromColumn: 'customer_id',
toCatalog: null,
toDb: 'public',
toTable: 'customers',
toColumn: 'id',
constraintName: 'orders_customer_id_fkey',
},
],
},
{
name: 'customers',
catalog: null,
db: 'public',
kind: 'table',
comment: null,
estimatedRows: null,
columns: [
{
name: 'id',
nativeType: 'integer',
normalizedType: 'integer',
dimensionType: 'number',
nullable: false,
primaryKey: true,
comment: null,
},
],
foreignKeys: [],
},
],
};
}
describe('ktxSchemaSnapshotToExtractedSchema', () => {
it('preserves structural table, column, comment, and key metadata', () => {
const extracted = ktxSchemaSnapshotToExtractedSchema(snapshot());
expect(extracted.tables).toEqual([
{
name: 'orders',
catalog: null,
db: 'public',
dbComment: 'Orders placed by customers',
columns: [
{
name: 'id',
type: 'integer',
nullable: false,
primaryKey: true,
dbComment: 'Primary key',
},
{
name: 'customer_id',
type: 'integer',
nullable: false,
primaryKey: false,
dbComment: null,
},
],
foreignKeys: [
{
fromTable: 'orders',
fromColumn: 'customer_id',
toTable: 'customers',
toColumn: 'id',
constraintName: 'orders_customer_id_fkey',
},
],
},
{
name: 'customers',
catalog: null,
db: 'public',
dbComment: null,
columns: [
{
name: 'id',
type: 'integer',
nullable: false,
primaryKey: true,
dbComment: null,
},
],
foreignKeys: [],
},
]);
});
it('builds the same natural key shape used by schema sync', () => {
expect(buildLiveDatabaseTableNaturalKey({ catalog: null, db: 'public', name: 'orders' })).toBe('|public|orders');
expect(buildLiveDatabaseTableNaturalKey({ catalog: 'warehouse', db: 'analytics', name: 'events' })).toBe(
'warehouse|analytics|events',
);
});
});

61
packages/cli/src/context/ingest/adapters/live-database/extracted-schema.ts
View file

@ -1,61 +0,0 @@
import type { KtxSchemaSnapshot, KtxSchemaTable } from '../../../scan/types.js';
export interface LiveDatabaseExtractedForeignKey {
fromTable: string;
fromColumn: string;
toTable: string;
toColumn: string;
constraintName?: string;
}
export interface LiveDatabaseExtractedColumn {
name: string;
type: string;
nullable: boolean;
primaryKey: boolean;
dbComment: string | null;
}
export interface LiveDatabaseExtractedTable {
name: string;
catalog: string | null;
db: string | null;
dbComment: string | null;
columns: LiveDatabaseExtractedColumn[];
foreignKeys: LiveDatabaseExtractedForeignKey[];
}
export interface LiveDatabaseExtractedSchema {
connectionId?: string;
tables: LiveDatabaseExtractedTable[];
}
export function buildLiveDatabaseTableNaturalKey(table: Pick<KtxSchemaTable, 'catalog' | 'db' | 'name'>): string {
return `${table.catalog ?? ''}|${table.db ?? ''}|${table.name}`;
}
export function ktxSchemaSnapshotToExtractedSchema(snapshot: KtxSchemaSnapshot): LiveDatabaseExtractedSchema {
return {
connectionId: snapshot.connectionId,
tables: snapshot.tables.map((table) => ({
name: table.name,
catalog: table.catalog ?? null,
db: table.db ?? null,
dbComment: table.comment ?? null,
columns: table.columns.map((column) => ({
name: column.name,
type: column.nativeType,
nullable: column.nullable,
primaryKey: column.primaryKey,
dbComment: column.comment ?? null,
})),
foreignKeys: table.foreignKeys.map((foreignKey) => ({
fromTable: table.name,
fromColumn: foreignKey.fromColumn,
toTable: foreignKey.toTable,
toColumn: foreignKey.toColumn,
...(foreignKey.constraintName ? { constraintName: foreignKey.constraintName } : {}),
})),
})),
};
}

4
packages/cli/src/context/ingest/adapters/live-database/manifest.ts
View file

@ -22,7 +22,7 @@ const HISTORIC_SQL_MANAGED_USAGE_KEYS = new Set([
'staleSince',
]);
export interface LiveDatabaseManifestColumn {
interface LiveDatabaseManifestColumn {
name: string;
type: string;
pk?: boolean;
@ -37,7 +37,7 @@ export interface LiveDatabaseManifestJoinEntry {
source: string;
}
export interface LiveDatabaseManifestTableEntry {
interface LiveDatabaseManifestTableEntry {
table: string;
descriptions?: Record<string, string>;
usage?: TableUsageOutput;

428
packages/cli/src/context/ingest/adapters/live-database/structural-sync.test.ts
View file

@ -1,428 +0,0 @@
import { describe, expect, it } from 'vitest';
import { type LiveDatabaseSyncedSchema, planLiveDatabaseStructuralSync } from './structural-sync.js';
function idFactory(): () => string {
let next = 1;
return () => `id-${next++}`;
}
describe('planLiveDatabaseStructuralSync', () => {
it('plans table and column creates, updates, deletes, and metadata invalidation', () => {
const current: LiveDatabaseSyncedSchema = {
connectionId: 'conn-1',
tables: [
{
id: 'tbl-orders',
name: 'orders',
catalog: null,
db: 'public',
enabled: true,
descriptions: { ai: 'Old AI order text', db: 'Old DB order text' },
columns: [
{
id: 'col-order-id',
name: 'id',
type: 'number',
nullable: false,
primaryKey: true,
parentColumnId: null,
descriptions: { db: 'Order id' },
embedding: [1, 2, 3],
sampleValues: null,
cardinality: null,
},
{
id: 'col-order-total',
name: 'total',
type: 'number',
nullable: true,
primaryKey: false,
parentColumnId: null,
descriptions: { ai: 'Old AI total text', db: 'Old total text' },
embedding: [4, 5, 6],
sampleValues: ['10'],
cardinality: 12,
},
{
id: 'col-order-removed',
name: 'removed',
type: 'string',
nullable: true,
primaryKey: false,
parentColumnId: null,
descriptions: {},
embedding: null,
sampleValues: null,
cardinality: null,
},
],
},
{
id: 'tbl-removed',
name: 'removed_table',
catalog: null,
db: 'public',
enabled: true,
descriptions: {},
columns: [
{
id: 'col-removed-id',
name: 'id',
type: 'number',
nullable: false,
primaryKey: true,
parentColumnId: null,
descriptions: {},
embedding: null,
sampleValues: null,
cardinality: null,
},
],
},
],
links: [
{
id: 'inferred-total-link',
fromTableId: 'tbl-orders',
fromColumnId: 'col-order-total',
toTableId: 'tbl-orders',
toColumnId: 'col-order-id',
source: 'inferred',
confidence: 0.7,
relationshipType: 'MANY_TO_ONE',
isPrimaryKeyReference: true,
},
],
};
const plan = planLiveDatabaseStructuralSync({
connectionId: 'conn-1',
current,
extracted: {
connectionId: 'conn-1',
tables: [
{
name: 'orders',
catalog: null,
db: 'public',
dbComment: 'Fresh DB order text',
columns: [
{
name: 'id',
type: 'number',
nullable: false,
primaryKey: true,
dbComment: 'Order id',
},
{
name: 'total',
type: 'string',
nullable: false,
primaryKey: false,
dbComment: 'Fresh total text',
},
{
name: 'created_at',
type: 'time',
nullable: false,
primaryKey: false,
dbComment: 'Creation timestamp',
},
],
foreignKeys: [],
},
{
name: 'customers',
catalog: null,
db: 'public',
dbComment: 'Customer table',
columns: [
{
name: 'id',
type: 'number',
nullable: false,
primaryKey: true,
dbComment: null,
},
],
foreignKeys: [],
},
],
},
idFactory: idFactory(),
});
expect(plan.stats).toEqual({
tablesCreated: 1,
tablesDeleted: 1,
columnsCreated: 2,
columnsDeleted: 2,
columnsModified: 1,
formalLinksCreated: 0,
formalLinksDeleted: 0,
});
expect(plan.operations.deleteTableIds).toEqual(['tbl-removed']);
expect(plan.operations.deleteColumnIds).toEqual(['col-order-removed']);
expect(plan.operations.insertTables).toEqual([
{
id: 'id-2',
connectionId: 'conn-1',
name: 'customers',
catalog: null,
db: 'public',
enabled: true,
},
]);
expect(plan.operations.insertColumns).toEqual([
{
id: 'id-1',
tableId: 'tbl-orders',
name: 'created_at',
parentColumnId: null,
},
{
id: 'id-3',
tableId: 'id-2',
name: 'id',
parentColumnId: null,
},
]);
expect(plan.operations.touchColumnIds).toEqual(['col-order-total']);
expect(plan.operations.invalidateColumnEmbeddingIds).toEqual(['col-order-total']);
expect(plan.inferredLinksToValidate).toEqual(['inferred-total-link']);
expect(plan.changes).toEqual({
newTableIds: ['id-2'],
newColumnIds: ['id-1', 'id-3'],
tablesWithStructuralChanges: ['tbl-orders', 'id-2'],
columnsWithTypeChange: ['col-order-total'],
columnsWithDescriptionChange: ['col-order-total'],
tablesWithDescriptionChange: ['tbl-orders'],
});
const orders = plan.schema.tables.find((table) => table.name === 'orders');
expect(orders?.descriptions).toEqual({ db: 'Fresh DB order text' });
expect(orders?.columns.map((column) => column.name)).toEqual(['id', 'total', 'created_at']);
expect(orders?.columns.find((column) => column.name === 'total')).toMatchObject({
id: 'col-order-total',
type: 'string',
nullable: false,
primaryKey: false,
descriptions: { db: 'Fresh total text' },
embedding: null,
sampleValues: ['10'],
cardinality: 12,
});
});
it('builds formal links from extracted foreign keys and preserves valid inferred links', () => {
const current: LiveDatabaseSyncedSchema = {
connectionId: 'conn-1',
tables: [
{
id: 'tbl-orders',
name: 'orders',
catalog: null,
db: 'public',
enabled: true,
descriptions: {},
columns: [
{
id: 'col-orders-id',
name: 'id',
type: 'number',
nullable: false,
primaryKey: true,
parentColumnId: null,
descriptions: {},
embedding: null,
sampleValues: null,
cardinality: null,
},
{
id: 'col-orders-customer',
name: 'customer_id',
type: 'number',
nullable: false,
primaryKey: false,
parentColumnId: null,
descriptions: {},
embedding: null,
sampleValues: null,
cardinality: null,
},
],
},
{
id: 'tbl-customers',
name: 'customers',
catalog: null,
db: 'public',
enabled: true,
descriptions: {},
columns: [
{
id: 'col-customers-id',
name: 'id',
type: 'number',
nullable: false,
primaryKey: true,
parentColumnId: null,
descriptions: {},
embedding: null,
sampleValues: null,
cardinality: null,
},
],
},
],
links: [
{
id: 'formal-existing',
fromTableId: 'tbl-orders',
fromColumnId: 'col-orders-customer',
toTableId: 'tbl-customers',
toColumnId: 'col-customers-id',
source: 'formal',
confidence: 1,
relationshipType: 'MANY_TO_ONE',
isPrimaryKeyReference: true,
},
{
id: 'inferred-existing',
fromTableId: 'tbl-orders',
fromColumnId: 'col-orders-id',
toTableId: 'tbl-customers',
toColumnId: 'col-customers-id',
source: 'inferred',
confidence: 0.6,
relationshipType: 'MANY_TO_ONE',
isPrimaryKeyReference: true,
},
],
};
const plan = planLiveDatabaseStructuralSync({
connectionId: 'conn-1',
current,
extracted: {
connectionId: 'conn-1',
tables: [
{
name: 'orders',
catalog: null,
db: 'public',
dbComment: null,
columns: [
{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null },
{ name: 'customer_id', type: 'number', nullable: false, primaryKey: false, dbComment: null },
],
foreignKeys: [
{
fromTable: 'orders',
fromColumn: 'customer_id',
toTable: 'customers',
toColumn: 'id',
},
],
},
{
name: 'customers',
catalog: null,
db: 'public',
dbComment: null,
columns: [{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }],
foreignKeys: [],
},
],
},
idFactory: idFactory(),
});
expect(plan.stats.formalLinksCreated).toBe(0);
expect(plan.stats.formalLinksDeleted).toBe(0);
expect(plan.schema.links.map((link) => link.id)).toEqual(['formal-existing', 'inferred-existing']);
const planAfterForeignKeyRemoval = planLiveDatabaseStructuralSync({
connectionId: 'conn-1',
current,
extracted: {
connectionId: 'conn-1',
tables: [
{
name: 'orders',
catalog: null,
db: 'public',
dbComment: null,
columns: [
{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null },
{ name: 'customer_id', type: 'number', nullable: false, primaryKey: false, dbComment: null },
],
foreignKeys: [],
},
{
name: 'customers',
catalog: null,
db: 'public',
dbComment: null,
columns: [{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }],
foreignKeys: [],
},
],
},
idFactory: idFactory(),
});
expect(planAfterForeignKeyRemoval.stats.formalLinksDeleted).toBe(1);
expect(planAfterForeignKeyRemoval.schema.links.map((link) => link.id)).toEqual(['inferred-existing']);
const planAfterForeignKeyCreation = planLiveDatabaseStructuralSync({
connectionId: 'conn-1',
current: { ...current, links: [current.links[1]] },
extracted: {
connectionId: 'conn-1',
tables: [
{
name: 'orders',
catalog: null,
db: 'public',
dbComment: null,
columns: [
{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null },
{ name: 'customer_id', type: 'number', nullable: false, primaryKey: false, dbComment: null },
],
foreignKeys: [
{
fromTable: 'orders',
fromColumn: 'customer_id',
toTable: 'customers',
toColumn: 'id',
},
],
},
{
name: 'customers',
catalog: null,
db: 'public',
dbComment: null,
columns: [{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }],
foreignKeys: [],
},
],
},
idFactory: idFactory(),
});
expect(planAfterForeignKeyCreation.stats.formalLinksCreated).toBe(1);
expect(planAfterForeignKeyCreation.schema.links[0]).toMatchObject({
id: 'id-1',
fromTableId: 'tbl-orders',
fromColumnId: 'col-orders-customer',
toTableId: 'tbl-customers',
toColumnId: 'col-customers-id',
source: 'formal',
confidence: 1,
relationshipType: 'MANY_TO_ONE',
isPrimaryKeyReference: true,
});
});
});

525
packages/cli/src/context/ingest/adapters/live-database/structural-sync.ts
View file

@ -1,525 +0,0 @@
import type { LiveDatabaseExtractedSchema, LiveDatabaseExtractedTable } from './extracted-schema.js';
import { buildLiveDatabaseTableNaturalKey } from './extracted-schema.js';
export interface LiveDatabaseSyncedColumn {
id: string;
name: string;
type: string;
nullable: boolean;
primaryKey: boolean;
parentColumnId: string | null;
descriptions: Record<string, string>;
embedding: number[] | null;
sampleValues: string[] | null;
cardinality: number | null;
}
export interface LiveDatabaseSyncedTable {
id: string;
name: string;
catalog: string | null;
db: string | null;
enabled: boolean;
descriptions: Record<string, string>;
columns: LiveDatabaseSyncedColumn[];
}
export interface LiveDatabaseSyncedLink {
id: string;
fromTableId: string;
fromColumnId: string;
toTableId: string;
toColumnId: string;
source: 'formal' | 'inferred' | 'manual';
confidence: number;
relationshipType: string;
isPrimaryKeyReference: boolean;
}
export interface LiveDatabaseSyncedSchema {
connectionId: string;
tables: LiveDatabaseSyncedTable[];
links: LiveDatabaseSyncedLink[];
}
export interface LiveDatabaseStructuralChanges {
newTableIds: string[];
newColumnIds: string[];
tablesWithStructuralChanges: string[];
columnsWithTypeChange: string[];
columnsWithDescriptionChange: string[];
tablesWithDescriptionChange: string[];
}
export interface LiveDatabaseStructuralSyncStats {
tablesCreated: number;
tablesDeleted: number;
columnsCreated: number;
columnsDeleted: number;
columnsModified: number;
formalLinksCreated: number;
formalLinksDeleted: number;
}
export interface LiveDatabaseStructuralSyncOperations {
deleteTableIds: string[];
deleteColumnIds: string[];
insertTables: Array<{
id: string;
connectionId: string;
name: string;
catalog: string | null;
db: string | null;
enabled: boolean;
}>;
insertColumns: Array<{
id: string;
tableId: string;
name: string;
parentColumnId: string | null;
}>;
touchColumnIds: string[];
invalidateColumnEmbeddingIds: string[];
}
export interface LiveDatabaseStructuralSyncPlan {
schema: LiveDatabaseSyncedSchema;
inferredLinksToValidate: string[];
stats: LiveDatabaseStructuralSyncStats;
changes: LiveDatabaseStructuralChanges;
operations: LiveDatabaseStructuralSyncOperations;
}
export interface PlanLiveDatabaseStructuralSyncInput {
connectionId: string;
current: LiveDatabaseSyncedSchema | null;
extracted: LiveDatabaseExtractedSchema;
idFactory: () => string;
}
interface UpdatedTableResult {
table: LiveDatabaseSyncedTable;
columnsCreated: number;
columnsDeleted: number;
columnsModified: number;
newColumnIds: string[];
columnsWithTypeChange: string[];
columnsWithDescriptionChange: string[];
tableDescriptionChanged: boolean;
}
function updateDescription(
descriptions: Record<string, string>,
dbComment: string | null | undefined,
changed: boolean,
): Record<string, string> {
const updated = { ...descriptions };
if (dbComment) {
updated.db = dbComment;
} else {
delete updated.db;
}
if (changed) {
delete updated.ai;
}
return updated;
}
function descriptionFromDbComment(dbComment: string | null | undefined): Record<string, string> {
return dbComment ? { db: dbComment } : {};
}
function planUpdatedTable(args: {
currentTable: LiveDatabaseSyncedTable;
extractedTable: LiveDatabaseExtractedTable;
currentLinks: LiveDatabaseSyncedLink[];
inferredLinksToValidate: string[];
operations: LiveDatabaseStructuralSyncOperations;
idFactory: () => string;
}): UpdatedTableResult {
const { currentTable, extractedTable, currentLinks, inferredLinksToValidate, operations, idFactory } = args;
let columnsCreated = 0;
let columnsDeleted = 0;
let columnsModified = 0;
const newColumnIds: string[] = [];
const columnsWithTypeChange: string[] = [];
const columnsWithDescriptionChange: string[] = [];
const updatedColumns: LiveDatabaseSyncedColumn[] = [];
const tableDescriptionChanged = (currentTable.descriptions.db ?? null) !== (extractedTable.dbComment ?? null);
const currentColumnsByName = new Map(currentTable.columns.map((column) => [column.name, column]));
const extractedColumnsByName = new Map(extractedTable.columns.map((column) => [column.name, column]));
for (const [name, currentColumn] of currentColumnsByName) {
if (!extractedColumnsByName.has(name)) {
operations.deleteColumnIds.push(currentColumn.id);
columnsDeleted++;
}
}
for (const [name, extractedColumn] of extractedColumnsByName) {
const currentColumn = currentColumnsByName.get(name);
if (!currentColumn) {
const columnId = idFactory();
operations.insertColumns.push({
id: columnId,
tableId: currentTable.id,
name: extractedColumn.name,
parentColumnId: null,
});
columnsCreated++;
newColumnIds.push(columnId);
updatedColumns.push({
id: columnId,
name: extractedColumn.name,
type: extractedColumn.type,
nullable: extractedColumn.nullable,
primaryKey: extractedColumn.primaryKey,
descriptions: descriptionFromDbComment(extractedColumn.dbComment),
parentColumnId: null,
embedding: null,
sampleValues: null,
cardinality: null,
});
continue;
}
const typeChanged = currentColumn.type !== extractedColumn.type;
const nullableChanged = currentColumn.nullable !== extractedColumn.nullable;
const primaryKeyChanged = currentColumn.primaryKey !== extractedColumn.primaryKey;
const dbDescriptionChanged = (currentColumn.descriptions.db ?? null) !== (extractedColumn.dbComment ?? null);
if (typeChanged || nullableChanged || primaryKeyChanged || dbDescriptionChanged) {
operations.touchColumnIds.push(currentColumn.id);
columnsModified++;
if (typeChanged || dbDescriptionChanged) {
operations.invalidateColumnEmbeddingIds.push(currentColumn.id);
}
if (typeChanged) {
columnsWithTypeChange.push(currentColumn.id);
const affectedLinks = currentLinks.filter(
(link) =>
link.source === 'inferred' &&
(link.fromColumnId === currentColumn.id || link.toColumnId === currentColumn.id),
);
for (const link of affectedLinks) {
if (!inferredLinksToValidate.includes(link.id)) {
inferredLinksToValidate.push(link.id);
}
}
}
if (dbDescriptionChanged) {
columnsWithDescriptionChange.push(currentColumn.id);
}
}
updatedColumns.push({
...currentColumn,
type: extractedColumn.type,
nullable: extractedColumn.nullable,
primaryKey: extractedColumn.primaryKey,
descriptions: updateDescription(currentColumn.descriptions, extractedColumn.dbComment, dbDescriptionChanged),
embedding: typeChanged ? null : currentColumn.embedding,
});
}
return {
table: {
...currentTable,
descriptions: updateDescription(currentTable.descriptions, extractedTable.dbComment, tableDescriptionChanged),
columns: updatedColumns,
},
columnsCreated,
columnsDeleted,
columnsModified,
newColumnIds,
columnsWithTypeChange,
columnsWithDescriptionChange,
tableDescriptionChanged,
};
}
function planCreatedTable(args: {
connectionId: string;
extractedTable: LiveDatabaseExtractedTable;
operations: LiveDatabaseStructuralSyncOperations;
idFactory: () => string;
}): LiveDatabaseSyncedTable {
const { connectionId, extractedTable, operations, idFactory } = args;
const tableId = idFactory();
operations.insertTables.push({
id: tableId,
connectionId,
name: extractedTable.name,
catalog: extractedTable.catalog,
db: extractedTable.db,
enabled: true,
});
const columns: LiveDatabaseSyncedColumn[] = extractedTable.columns.map((extractedColumn) => {
const columnId = idFactory();
operations.insertColumns.push({
id: columnId,
tableId,
name: extractedColumn.name,
parentColumnId: null,
});
return {
id: columnId,
name: extractedColumn.name,
type: extractedColumn.type,
nullable: extractedColumn.nullable,
primaryKey: extractedColumn.primaryKey,
descriptions: descriptionFromDbComment(extractedColumn.dbComment),
parentColumnId: null,
embedding: null,
sampleValues: null,
cardinality: null,
};
});
return {
id: tableId,
name: extractedTable.name,
catalog: extractedTable.catalog,
db: extractedTable.db,
enabled: true,
descriptions: descriptionFromDbComment(extractedTable.dbComment),
columns,
};
}
function syncFormalLinks(args: {
extracted: LiveDatabaseExtractedSchema;
tables: LiveDatabaseSyncedTable[];
tableNaturalKeyToId: Map<string, string>;
currentLinks: LiveDatabaseSyncedLink[];
idFactory: () => string;
}): { links: LiveDatabaseSyncedLink[]; created: number; deleted: number } {
const { extracted, tables, tableNaturalKeyToId, currentLinks, idFactory } = args;
const columnKeyToId = new Map<string, string>();
for (const table of tables) {
const tableKey = buildLiveDatabaseTableNaturalKey(table);
for (const column of table.columns) {
columnKeyToId.set(`${tableKey}.${column.name}`, column.id);
}
}
const extractedFormalLinks: Array<{
fromTableId: string;
fromColumnId: string;
toTableId: string;
toColumnId: string;
}> = [];
for (const table of extracted.tables) {
const fromTableKey = buildLiveDatabaseTableNaturalKey(table);
const fromTableId = tableNaturalKeyToId.get(fromTableKey);
if (!fromTableId) {
continue;
}
for (const foreignKey of table.foreignKeys) {
const toTableKey = buildLiveDatabaseTableNaturalKey({
catalog: table.catalog,
db: table.db,
name: foreignKey.toTable,
});
const toTableId = tableNaturalKeyToId.get(toTableKey);
if (!toTableId) {
continue;
}
const fromColumnId = columnKeyToId.get(`${fromTableKey}.${foreignKey.fromColumn}`);
const toColumnId = columnKeyToId.get(`${toTableKey}.${foreignKey.toColumn}`);
if (!fromColumnId || !toColumnId) {
continue;
}
extractedFormalLinks.push({ fromTableId, fromColumnId, toTableId, toColumnId });
}
}
const currentFormalLinks = currentLinks.filter((link) => link.source === 'formal');
const extractedLinkKeys = new Set(extractedFormalLinks.map((link) => `${link.fromColumnId}->${link.toColumnId}`));
const linksToDelete = currentFormalLinks.filter(
(link) => !extractedLinkKeys.has(`${link.fromColumnId}->${link.toColumnId}`),
);
const currentLinkKeys = new Set(currentFormalLinks.map((link) => `${link.fromColumnId}->${link.toColumnId}`));
const linksToCreate = extractedFormalLinks.filter(
(link) => !currentLinkKeys.has(`${link.fromColumnId}->${link.toColumnId}`),
);
const newLinks = linksToCreate.map((linkData) => ({
id: idFactory(),
fromTableId: linkData.fromTableId,
fromColumnId: linkData.fromColumnId,
toTableId: linkData.toTableId,
toColumnId: linkData.toColumnId,
source: 'formal' as const,
confidence: 1,
relationshipType: 'MANY_TO_ONE',
isPrimaryKeyReference: true,
}));
const deletedLinkIds = new Set(linksToDelete.map((link) => link.id));
const preservedFormalLinks = currentFormalLinks.filter((link) => !deletedLinkIds.has(link.id));
return {
links: [...preservedFormalLinks, ...newLinks],
created: linksToCreate.length,
deleted: linksToDelete.length,
};
}
export function planLiveDatabaseStructuralSync(
input: PlanLiveDatabaseStructuralSyncInput,
): LiveDatabaseStructuralSyncPlan {
const operations: LiveDatabaseStructuralSyncOperations = {
deleteTableIds: [],
deleteColumnIds: [],
insertTables: [],
insertColumns: [],
touchColumnIds: [],
invalidateColumnEmbeddingIds: [],
};
const stats: LiveDatabaseStructuralSyncStats = {
tablesCreated: 0,
tablesDeleted: 0,
columnsCreated: 0,
columnsDeleted: 0,
columnsModified: 0,
formalLinksCreated: 0,
formalLinksDeleted: 0,
};
const changes: LiveDatabaseStructuralChanges = {
newTableIds: [],
newColumnIds: [],
tablesWithStructuralChanges: [],
columnsWithTypeChange: [],
columnsWithDescriptionChange: [],
tablesWithDescriptionChange: [],
};
const inferredLinksToValidate: string[] = [];
const currentTablesByKey = new Map<string, LiveDatabaseSyncedTable>();
const extractedTablesByKey = new Map<string, LiveDatabaseExtractedTable>();
if (input.current) {
for (const table of input.current.tables) {
currentTablesByKey.set(buildLiveDatabaseTableNaturalKey(table), table);
}
}
for (const table of input.extracted.tables) {
extractedTablesByKey.set(buildLiveDatabaseTableNaturalKey(table), table);
}
const tablesToDelete: LiveDatabaseSyncedTable[] = [];
const tablesToUpdate: Array<{
current: LiveDatabaseSyncedTable;
extracted: LiveDatabaseExtractedTable;
}> = [];
const tablesToCreate: LiveDatabaseExtractedTable[] = [];
for (const [key, table] of currentTablesByKey) {
const extractedTable = extractedTablesByKey.get(key);
if (!extractedTable) {
tablesToDelete.push(table);
} else {
tablesToUpdate.push({ current: table, extracted: extractedTable });
}
}
for (const [key, table] of extractedTablesByKey) {
if (!currentTablesByKey.has(key)) {
tablesToCreate.push(table);
}
}
for (const table of tablesToDelete) {
operations.deleteTableIds.push(table.id);
stats.tablesDeleted++;
stats.columnsDeleted += table.columns.length;
}
const updatedTables: LiveDatabaseSyncedTable[] = [];
for (const { current, extracted } of tablesToUpdate) {
const result = planUpdatedTable({
currentTable: current,
extractedTable: extracted,
currentLinks: input.current?.links ?? [],
inferredLinksToValidate,
operations,
idFactory: input.idFactory,
});
updatedTables.push(result.table);
stats.columnsCreated += result.columnsCreated;
stats.columnsDeleted += result.columnsDeleted;
stats.columnsModified += result.columnsModified;
changes.newColumnIds.push(...result.newColumnIds);
changes.columnsWithTypeChange.push(...result.columnsWithTypeChange);
changes.columnsWithDescriptionChange.push(...result.columnsWithDescriptionChange);
if (result.tableDescriptionChanged) {
changes.tablesWithDescriptionChange.push(current.id);
}
if (result.columnsCreated > 0 || result.columnsDeleted > 0 || result.columnsWithTypeChange.length > 0) {
changes.tablesWithStructuralChanges.push(current.id);
}
}
const createdTables: LiveDatabaseSyncedTable[] = [];
for (const extractedTable of tablesToCreate) {
const table = planCreatedTable({
connectionId: input.connectionId,
extractedTable,
operations,
idFactory: input.idFactory,
});
createdTables.push(table);
stats.tablesCreated++;
stats.columnsCreated += table.columns.length;
changes.newTableIds.push(table.id);
changes.newColumnIds.push(...table.columns.map((column) => column.id));
changes.tablesWithStructuralChanges.push(table.id);
}
const allTables = [...updatedTables, ...createdTables];
const tableNaturalKeyToId = new Map<string, string>();
for (const table of allTables) {
tableNaturalKeyToId.set(buildLiveDatabaseTableNaturalKey(table), table.id);
}
const formalLinkResult = syncFormalLinks({
extracted: input.extracted,
tables: allTables,
tableNaturalKeyToId,
currentLinks: input.current?.links ?? [],
idFactory: input.idFactory,
});
stats.formalLinksCreated = formalLinkResult.created;
stats.formalLinksDeleted = formalLinkResult.deleted;
const deletedTableIds = new Set(tablesToDelete.map((table) => table.id));
const preservedInferredLinks = (input.current?.links ?? []).filter(
(link) =>
link.source === 'inferred' && !deletedTableIds.has(link.fromTableId) && !deletedTableIds.has(link.toTableId),
);
return {
schema: {
connectionId: input.connectionId,
tables: allTables,
links: [...formalLinkResult.links, ...preservedInferredLinks],
},
inferredLinksToValidate,
stats,
changes,
operations,
};
}

2
packages/cli/src/context/ingest/adapters/looker/factory.ts
View file

@ -7,6 +7,7 @@ export interface LookerCredentialResolver {
resolve(lookerConnectionId: string): Promise<LookerConnectionParams>;
}
/** @internal */
export interface LookerConnectionClientFactory {
createClient(lookerConnectionId: string): Promise<LookerRuntimeClient>;
}
@ -23,6 +24,7 @@ export class DefaultLookerConnectionClientFactory implements LookerConnectionCli
}
}
/** @internal */
export class DefaultLookerClientFactory implements LookerClientFactory {
constructor(private readonly inner: LookerConnectionClientFactory) {}

2
packages/cli/src/context/ingest/adapters/looker/fetch.ts
View file

@ -31,7 +31,7 @@ import {
stagedUserFileSchema,
} from './types.js';
export interface LookerEntityRef {
interface LookerEntityRef {
id: string;
updatedAt?: string | null;
}

20
packages/cli/src/context/ingest/adapters/looker/local-looker.adapter.ts
View file

@ -1,11 +1,8 @@
import type { KtxLocalProject, KtxProjectConnectionConfig } from '../../../project/index.js';
import type { LookerClientLogger } from './client.js';
import type { KtxLocalProject } from '../../../../context/project/project.js';
import type { KtxProjectConnectionConfig } from '../../../../context/project/config.js';
import {
DefaultLookerClientFactory,
DefaultLookerConnectionClientFactory,
type LookerCredentialResolver,
} from './factory.js';
import { LookerSourceAdapter } from './looker.adapter.js';
function stringField(value: unknown): string | null {
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
@ -54,16 +51,3 @@ export function createLocalLookerCredentialResolver(
},
};
}
export function createLocalLookerSourceAdapter(
project: KtxLocalProject,
env: NodeJS.ProcessEnv = process.env,
logger?: LookerClientLogger,
): LookerSourceAdapter {
const connectionFactory = new DefaultLookerConnectionClientFactory(createLocalLookerCredentialResolver(project, env), {
...(logger ? { logger } : {}),
});
return new LookerSourceAdapter({
clientFactory: new DefaultLookerClientFactory(connectionFactory),
});
}

4
packages/cli/src/context/ingest/adapters/looker/local-runtime-store.ts
View file

@ -5,7 +5,7 @@ import type { LookerWarehouseConnectionInfo } from './client.js';
import type { LookerConnectionMapping } from './mapping.js';
import type { LookerRuntimeCursors } from './types.js';
export type LocalLookerMappingSource = 'ktx.yaml' | 'cli' | 'refresh';
type LocalLookerMappingSource = 'ktx.yaml' | 'cli' | 'refresh';
interface LocalLookerRuntimeStoreOptions {
dbPath: string;
@ -41,7 +41,7 @@ export interface ClearLocalLookerMappingsInput {
lookerConnectionName?: string;
}
export interface LookerSourceStateReader {
interface LookerSourceStateReader {
readMappings(lookerConnectionId: string): Promise<LookerConnectionMapping[]>;
readCursors(lookerConnectionId: string): Promise<LookerRuntimeCursors>;
}

19
packages/cli/src/context/ingest/adapters/looker/mapping.ts
View file

@ -2,7 +2,7 @@ import type { ParsedTargetTable } from '../../parsed-target-table.js';
import type { LookerWarehouseConnectionInfo } from './client.js';
import type { LookerPullConfig, LookerRuntimeCursors, StagedExploreFile, StagedLookmlModelsFile } from './types.js';
export const LOOKER_DIALECT_TO_CONNECTION_TYPE = {
const LOOKER_DIALECT_TO_CONNECTION_TYPE = {
bigquery: 'BIGQUERY',
bigquery_standard_sql: 'BIGQUERY',
snowflake: 'SNOWFLAKE',
@ -16,6 +16,7 @@ export const LOOKER_DIALECT_TO_CONNECTION_TYPE = {
clickhouse: 'CLICKHOUSE',
} as const;
/** @internal */
export type LookerWarehouseTargetConnectionType =
(typeof LOOKER_DIALECT_TO_CONNECTION_TYPE)[keyof typeof LOOKER_DIALECT_TO_CONNECTION_TYPE];
@ -33,6 +34,7 @@ export interface LookerTargetConnection {
connection_params?: Record<string, unknown> | null;
}
/** @internal */
export interface LookerMappingCandidateConnection extends LookerTargetConnection {}
export interface LookerMappingDrift {
@ -89,6 +91,7 @@ export async function discoverLookerConnections(
return client.listLookerConnections();
}
/** @internal */
export function lookerDialectToConnectionType(dialect: string | null): LookerWarehouseTargetConnectionType | null {
if (!dialect) {
return null;
@ -98,10 +101,12 @@ export function lookerDialectToConnectionType(dialect: string | null): LookerWar
);
}
/** @internal */
export function sqlglotDialectForConnectionType(connectionType: string): string | null {
return SQLGLOT_DIALECT_BY_CONNECTION_TYPE[connectionType as LookerWarehouseTargetConnectionType] ?? null;
}
/** @internal */
export function validateLookerWarehouseTarget(connectionType: string): { ok: true } | { ok: false; reason: string } {
return sqlglotDialectForConnectionType(connectionType)
? { ok: true }
@ -111,7 +116,7 @@ export function validateLookerWarehouseTarget(connectionType: string): { ok: tru
};
}
export function extractWarehouseHost(params: unknown, connectionType: string): string | null {
function extractWarehouseHost(params: unknown, connectionType: string): string | null {
const record = isRecord(params) ? params : {};
switch (connectionType) {
case 'POSTGRESQL':
@ -126,7 +131,7 @@ export function extractWarehouseHost(params: unknown, connectionType: string): s
}
}
export function extractWarehouseDatabase(params: unknown, connectionType: string): string | null {
function extractWarehouseDatabase(params: unknown, connectionType: string): string | null {
const record = isRecord(params) ? params : {};
switch (connectionType) {
case 'POSTGRESQL':
@ -142,14 +147,15 @@ export function extractWarehouseDatabase(params: unknown, connectionType: string
}
}
export function normalizeHost(value: string | null): string | null {
function normalizeHost(value: string | null): string | null {
return value ? value.toLowerCase().replace(/:\d+$/, '') : null;
}
export function normalizeName(value: string | null): string | null {
function normalizeName(value: string | null): string | null {
return value ? value.toLowerCase() : null;
}
/** @internal */
export function suggestKtxConnectionForLookerConnection(args: {
lookerConnection: LookerWarehouseConnectionInfo;
candidateConnections: LookerMappingCandidateConnection[];
@ -224,6 +230,7 @@ export function validateLookerMappings(args: {
return errors.length === 0 ? { ok: true } : { ok: false, errors };
}
/** @internal */
export function refreshLookerMappingPlaceholders(args: {
stored: LookerConnectionMapping[];
live: LookerWarehouseConnectionInfo[];
@ -264,6 +271,7 @@ export function refreshLookerMappingPlaceholders(args: {
return { mappings: [...byName.values()], changed };
}
/** @internal */
export function collectExploreParseItems(args: {
explore: StagedExploreFile;
connectionMappings: Record<string, string>;
@ -309,6 +317,7 @@ export function collectExploreParseItems(args: {
return { parsedTargetTables, parseItems };
}
/** @internal */
export function projectParsedIdentifier(row: LookerParsedIdentifier | undefined): ParsedTargetTable {
if (!row) {
return { ok: false, reason: 'parse_error', detail: 'Python parser response was missing this key.' };

4
packages/cli/src/context/ingest/adapters/looker/scope.ts
View file

@ -15,7 +15,7 @@ export async function describeLookerScope(stagedDir: string): Promise<ScopeDescr
};
}
export async function readLookerScope(stagedDir: string): Promise<StagedLookerScopeFile> {
async function readLookerScope(stagedDir: string): Promise<StagedLookerScopeFile> {
try {
const body = await readFile(join(stagedDir, STAGED_FILES.scope), 'utf-8');
return stagedLookerScopeFileSchema.parse(JSON.parse(body));
@ -27,6 +27,7 @@ export async function readLookerScope(stagedDir: string): Promise<StagedLookerSc
}
}
/** @internal */
export function hashLookerScope(scope: StagedLookerScopeFile): string {
const canonical = JSON.stringify({
mode: scope.mode,
@ -36,6 +37,7 @@ export function hashLookerScope(scope: StagedLookerScopeFile): string {
return createHash('sha256').update(canonical).digest('hex');
}
/** @internal */
export function isPathInLookerScope(rawPath: string, scope: StagedLookerScopeFile): boolean {
if (scope.mode === 'full') {
return true;

2
packages/cli/src/context/ingest/adapters/looker/tools/looker-query-to-sl.tool.test.ts
View file

@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest';
import type { ToolOutput } from '../../../../tools/index.js';
import type { ToolOutput } from '../../../../../context/tools/base-tool.js';
import { buildLookerSlProposal, createLookerQueryToSlTool, type LookerSlProposal } from './looker-query-to-sl.tool.js';
describe('buildLookerSlProposal', () => {

13
packages/cli/src/context/ingest/adapters/looker/tools/looker-query-to-sl.tool.ts
View file

@ -1,6 +1,6 @@
import { tool } from 'ai';
import { z } from 'zod';
import type { ToolOutput } from '../../../../tools/index.js';
import type { ToolOutput } from '../../../../../context/tools/base-tool.js';
import type { ParsedTargetTable } from '../../../parsed-target-table.js';
import { stagedLookerQuerySchema } from '../types.js';
@ -9,7 +9,7 @@ const lookerUsageInputSchema = z.object({
uniqueUsers30d: z.number().int().nonnegative().default(0),
});
export const lookerQueryToSlInputSchema = z.object({
const lookerQueryToSlInputSchema = z.object({
query: stagedLookerQuerySchema,
contentTitle: z.string().min(1).optional(),
contentType: z.enum(['look', 'dashboard_tile']).default('look'),
@ -20,17 +20,17 @@ export type LookerQueryToSlInput = z.input<typeof lookerQueryToSlInputSchema>;
type LookerTargetStatus = 'mapped' | 'unmapped' | 'unparseable' | 'missing_target_table';
export interface LookerSlFieldProposal {
interface LookerSlFieldProposal {
name: string;
lookerField: string;
}
export interface LookerSlMeasureProposal extends LookerSlFieldProposal {
interface LookerSlMeasureProposal extends LookerSlFieldProposal {
expr: string;
description: string;
}
export interface LookerSlSegmentProposal {
interface LookerSlSegmentProposal {
name: string;
filters: Record<string, unknown>;
suggestedPredicate: string;
@ -89,6 +89,7 @@ function targetNotes(status: LookerTargetStatus, targetTable: ParsedTargetTable
];
}
/** @internal */
export function buildLookerSlProposal(raw: LookerQueryToSlInput): LookerSlProposal {
const input = lookerQueryToSlInputSchema.parse(raw);
const sourceName = `looker__${toSlName(input.query.model)}__${toSlName(input.query.view)}`;
@ -182,7 +183,7 @@ export function createLookerQueryToSlTool() {
});
}
export function formatLookerSlProposal(proposal: LookerSlProposal): string {
function formatLookerSlProposal(proposal: LookerSlProposal): string {
const lines = [
'## Looker query SL proposal',
'',

5
packages/cli/src/context/ingest/adapters/looker/types.ts
View file

@ -5,9 +5,9 @@ import { parsedTargetTableSchema } from '../../parsed-target-table.js';
const lookerIdSchema = z.union([z.string(), z.number().int()]).transform(String);
const nullableLookerIdSchema = z.union([lookerIdSchema, z.null()]).default(null);
export const lookerConnectionIdSchema = z.string().min(1).regex(/^[A-Za-z0-9_-]+$/);
const lookerConnectionIdSchema = z.string().min(1).regex(/^[A-Za-z0-9_-]+$/);
export const lookerRuntimeCursorsSchema = z.object({
const lookerRuntimeCursorsSchema = z.object({
dashboardsLastSyncedAt: z.iso.datetime().nullable().default(null),
looksLastSyncedAt: z.iso.datetime().nullable().default(null),
});
@ -215,6 +215,7 @@ const stagedLookerFetchIssueKindSchema = z.enum([
'lookml_connection_mismatch',
]);
/** @internal */
export const stagedLookerFetchIssueSchema = z.object({
rawPath: z.string().min(1),
entityType: z.enum(['dashboard', 'look', 'explore', 'signals', 'lookml_models', 'looker_connection_mapping']),

Some files were not shown because too many files have changed in this diff Show more