mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-25 08:48:08 +02:00
rename klo to ktx
This commit is contained in:
parent
1a42152e6f
commit
3ce510b55b
704 changed files with 10205 additions and 10255 deletions
|
|
@ -1,7 +1,7 @@
|
|||
import { KloMessageBuilder, type KloLlmProvider, type KloModelRole } from '@klo/llm';
|
||||
import { KtxMessageBuilder, type KtxLlmProvider, type KtxModelRole } from '@ktx/llm';
|
||||
import { generateText, stepCountIs, type TelemetrySettings, type Tool } from 'ai';
|
||||
import { noopLogger, type KloLogger } from '../core/index.js';
|
||||
import { summarizeKloLlmDebugRequest, type KloLlmDebugRequestRecorder } from '../llm/index.js';
|
||||
import { noopLogger, type KtxLogger } from '../core/index.js';
|
||||
import { summarizeKtxLlmDebugRequest, type KtxLlmDebugRequestRecorder } from '../llm/index.js';
|
||||
|
||||
export type RunLoopStopReason = 'budget' | 'natural' | 'error';
|
||||
|
||||
|
|
@ -11,7 +11,7 @@ export interface RunLoopStepInfo {
|
|||
}
|
||||
|
||||
export interface RunLoopParams {
|
||||
modelRole: KloModelRole;
|
||||
modelRole: KtxModelRole;
|
||||
systemPrompt: string;
|
||||
userPrompt: string;
|
||||
toolSet: Record<string, Tool>;
|
||||
|
|
@ -30,14 +30,14 @@ export interface AgentTelemetryPort {
|
|||
}
|
||||
|
||||
export interface AgentRunnerServiceDeps {
|
||||
llmProvider: KloLlmProvider;
|
||||
llmProvider: KtxLlmProvider;
|
||||
telemetry?: AgentTelemetryPort;
|
||||
debugRequestRecorder?: KloLlmDebugRequestRecorder;
|
||||
logger?: KloLogger;
|
||||
debugRequestRecorder?: KtxLlmDebugRequestRecorder;
|
||||
logger?: KtxLogger;
|
||||
}
|
||||
|
||||
export class AgentRunnerService {
|
||||
private readonly logger: KloLogger;
|
||||
private readonly logger: KtxLogger;
|
||||
|
||||
constructor(private readonly deps: AgentRunnerServiceDeps) {
|
||||
this.logger = deps.logger ?? noopLogger;
|
||||
|
|
@ -47,7 +47,7 @@ export class AgentRunnerService {
|
|||
let stepIndex = 0;
|
||||
try {
|
||||
const model = this.deps.llmProvider.getModel(params.modelRole);
|
||||
const builder = new KloMessageBuilder(this.deps.llmProvider);
|
||||
const builder = new KtxMessageBuilder(this.deps.llmProvider);
|
||||
const built = builder.wrapSimple({
|
||||
system: params.systemPrompt,
|
||||
messages: [{ role: 'user', content: params.userPrompt }],
|
||||
|
|
@ -56,8 +56,8 @@ export class AgentRunnerService {
|
|||
});
|
||||
|
||||
await this.deps.debugRequestRecorder?.record(
|
||||
summarizeKloLlmDebugRequest({
|
||||
operationName: params.telemetryTags.operationName ?? 'klo-agent-runner',
|
||||
summarizeKtxLlmDebugRequest({
|
||||
operationName: params.telemetryTags.operationName ?? 'ktx-agent-runner',
|
||||
source: params.telemetryTags.source,
|
||||
jobId: params.telemetryTags.jobId,
|
||||
unitKey: params.telemetryTags.unitKey,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
export type {
|
||||
KloSqlQueryExecutionInput,
|
||||
KloSqlQueryExecutionResult,
|
||||
KloSqlQueryExecutorPort,
|
||||
KtxSqlQueryExecutionInput,
|
||||
KtxSqlQueryExecutionResult,
|
||||
KtxSqlQueryExecutorPort,
|
||||
} from './query-executor.js';
|
||||
export { createDefaultLocalQueryExecutor, type DefaultLocalQueryExecutorOptions } from './local-query-executor.js';
|
||||
export { normalizeQueryRows } from './query-executor.js';
|
||||
|
|
@ -17,11 +17,11 @@ export {
|
|||
type LocalWarehouseDescriptor,
|
||||
} from './local-warehouse-descriptor.js';
|
||||
export {
|
||||
KLO_NOTION_ORG_KNOWLEDGE_WARNING,
|
||||
KTX_NOTION_ORG_KNOWLEDGE_WARNING,
|
||||
notionConnectionToPullConfig,
|
||||
parseNotionConnectionConfig,
|
||||
redactNotionConnectionConfig,
|
||||
resolveNotionAuthToken,
|
||||
type KloNotionConnectionConfig,
|
||||
type RedactedKloNotionConnectionConfig,
|
||||
type KtxNotionConnectionConfig,
|
||||
type RedactedKtxNotionConnectionConfig,
|
||||
} from './notion-config.js';
|
||||
|
|
|
|||
|
|
@ -1,26 +1,26 @@
|
|||
import { createPostgresQueryExecutor } from './postgres-query-executor.js';
|
||||
import type {
|
||||
KloSqlQueryExecutionInput,
|
||||
KloSqlQueryExecutionResult,
|
||||
KloSqlQueryExecutorPort,
|
||||
KtxSqlQueryExecutionInput,
|
||||
KtxSqlQueryExecutionResult,
|
||||
KtxSqlQueryExecutorPort,
|
||||
} from './query-executor.js';
|
||||
import { createSqliteQueryExecutor } from './sqlite-query-executor.js';
|
||||
|
||||
export interface DefaultLocalQueryExecutorOptions {
|
||||
postgres?: KloSqlQueryExecutorPort;
|
||||
sqlite?: KloSqlQueryExecutorPort;
|
||||
postgres?: KtxSqlQueryExecutorPort;
|
||||
sqlite?: KtxSqlQueryExecutorPort;
|
||||
}
|
||||
|
||||
function driverFor(input: KloSqlQueryExecutionInput): string {
|
||||
function driverFor(input: KtxSqlQueryExecutionInput): string {
|
||||
return String(input.connection?.driver ?? '').toLowerCase();
|
||||
}
|
||||
|
||||
export function createDefaultLocalQueryExecutor(options: DefaultLocalQueryExecutorOptions = {}): KloSqlQueryExecutorPort {
|
||||
export function createDefaultLocalQueryExecutor(options: DefaultLocalQueryExecutorOptions = {}): KtxSqlQueryExecutorPort {
|
||||
const postgres = options.postgres ?? createPostgresQueryExecutor();
|
||||
const sqlite = options.sqlite ?? createSqliteQueryExecutor();
|
||||
|
||||
return {
|
||||
async execute(input: KloSqlQueryExecutionInput): Promise<KloSqlQueryExecutionResult> {
|
||||
async execute(input: KtxSqlQueryExecutionInput): Promise<KtxSqlQueryExecutionResult> {
|
||||
const driver = driverFor(input);
|
||||
if (driver === 'postgres' || driver === 'postgresql') {
|
||||
return postgres.execute(input);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KloProjectConnectionConfig } from '../project/config.js';
|
||||
import type { KtxProjectConnectionConfig } from '../project/config.js';
|
||||
import type { ConnectionType } from './connection-type.js';
|
||||
|
||||
export interface LocalWarehouseDescriptor {
|
||||
|
|
@ -32,7 +32,7 @@ const DRIVER_TO_CONNECTION_TYPE: Record<string, ConnectionType> = {
|
|||
|
||||
export function localConnectionToWarehouseDescriptor(
|
||||
id: string,
|
||||
connection: KloProjectConnectionConfig | undefined,
|
||||
connection: KtxProjectConnectionConfig | undefined,
|
||||
): LocalWarehouseDescriptor | null {
|
||||
if (!connection) {
|
||||
return null;
|
||||
|
|
@ -74,7 +74,7 @@ export function localConnectionToWarehouseDescriptor(
|
|||
return info;
|
||||
}
|
||||
|
||||
export function localConnectionTypeForConfig(id: string, connection: KloProjectConnectionConfig | undefined): string {
|
||||
export function localConnectionTypeForConfig(id: string, connection: KtxProjectConnectionConfig | undefined): string {
|
||||
const descriptor = localConnectionToWarehouseDescriptor(id, connection);
|
||||
if (descriptor) {
|
||||
return descriptor.connection_type;
|
||||
|
|
@ -85,7 +85,7 @@ export function localConnectionTypeForConfig(id: string, connection: KloProjectC
|
|||
|
||||
export function localConnectionInfoFromConfig(
|
||||
id: string,
|
||||
connection: KloProjectConnectionConfig | undefined,
|
||||
connection: KtxProjectConnectionConfig | undefined,
|
||||
): LocalConnectionInfo | null {
|
||||
if (!connection) {
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ describe('standalone Notion connection config', () => {
|
|||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-notion-config-'));
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-notion-config-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
|
|||
|
|
@ -2,17 +2,17 @@ import { readFile } from 'node:fs/promises';
|
|||
import { homedir } from 'node:os';
|
||||
import { resolve } from 'node:path';
|
||||
import { type NotionPullConfig, notionPullConfigSchema } from '../ingest/adapters/notion/types.js';
|
||||
import type { KloProjectConnectionConfig } from '../project/config.js';
|
||||
import type { KtxProjectConnectionConfig } from '../project/config.js';
|
||||
|
||||
export const KLO_NOTION_ORG_KNOWLEDGE_WARNING =
|
||||
export const KTX_NOTION_ORG_KNOWLEDGE_WARNING =
|
||||
'Anything accessible to this Notion integration can become organization knowledge.';
|
||||
|
||||
type KloNotionCrawlMode = 'all_accessible' | 'selected_roots';
|
||||
type KtxNotionCrawlMode = 'all_accessible' | 'selected_roots';
|
||||
|
||||
export interface KloNotionConnectionConfig extends KloProjectConnectionConfig {
|
||||
export interface KtxNotionConnectionConfig extends KtxProjectConnectionConfig {
|
||||
driver: 'notion';
|
||||
auth_token_ref: string;
|
||||
crawl_mode: KloNotionCrawlMode;
|
||||
crawl_mode: KtxNotionCrawlMode;
|
||||
root_page_ids: string[];
|
||||
root_database_ids: string[];
|
||||
root_data_source_ids: string[];
|
||||
|
|
@ -22,17 +22,17 @@ export interface KloNotionConnectionConfig extends KloProjectConnectionConfig {
|
|||
last_successful_cursor: string | null;
|
||||
}
|
||||
|
||||
export interface RedactedKloNotionConnectionConfig {
|
||||
export interface RedactedKtxNotionConnectionConfig {
|
||||
driver: 'notion';
|
||||
hasAuthToken: boolean;
|
||||
crawlMode: KloNotionCrawlMode;
|
||||
crawlMode: KtxNotionCrawlMode;
|
||||
rootPageIds: string[];
|
||||
rootDatabaseIds: string[];
|
||||
rootDataSourceIds: string[];
|
||||
maxPagesPerRun: number;
|
||||
maxKnowledgeCreatesPerRun: number;
|
||||
maxKnowledgeUpdatesPerRun: number;
|
||||
warning: typeof KLO_NOTION_ORG_KNOWLEDGE_WARNING;
|
||||
warning: typeof KTX_NOTION_ORG_KNOWLEDGE_WARNING;
|
||||
}
|
||||
|
||||
interface ResolveNotionTokenOptions {
|
||||
|
|
@ -84,7 +84,7 @@ function boundedInteger(value: unknown, fallback: number, name: string, min: num
|
|||
return parsed;
|
||||
}
|
||||
|
||||
export function parseNotionConnectionConfig(raw: unknown): KloNotionConnectionConfig {
|
||||
export function parseNotionConnectionConfig(raw: unknown): KtxNotionConnectionConfig {
|
||||
const input = record(raw);
|
||||
if (input.driver !== 'notion') {
|
||||
throw new Error('Notion connection config requires driver: notion');
|
||||
|
|
@ -135,7 +135,7 @@ export function parseNotionConnectionConfig(raw: unknown): KloNotionConnectionCo
|
|||
};
|
||||
}
|
||||
|
||||
export function redactNotionConnectionConfig(config: KloNotionConnectionConfig): RedactedKloNotionConnectionConfig {
|
||||
export function redactNotionConnectionConfig(config: KtxNotionConnectionConfig): RedactedKtxNotionConnectionConfig {
|
||||
return {
|
||||
driver: 'notion',
|
||||
hasAuthToken: Boolean(config.auth_token_ref),
|
||||
|
|
@ -146,7 +146,7 @@ export function redactNotionConnectionConfig(config: KloNotionConnectionConfig):
|
|||
maxPagesPerRun: config.max_pages_per_run,
|
||||
maxKnowledgeCreatesPerRun: config.max_knowledge_creates_per_run,
|
||||
maxKnowledgeUpdatesPerRun: config.max_knowledge_updates_per_run,
|
||||
warning: KLO_NOTION_ORG_KNOWLEDGE_WARNING,
|
||||
warning: KTX_NOTION_ORG_KNOWLEDGE_WARNING,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -179,7 +179,7 @@ export async function resolveNotionAuthToken(
|
|||
}
|
||||
|
||||
export async function notionConnectionToPullConfig(
|
||||
config: KloNotionConnectionConfig,
|
||||
config: KtxNotionConnectionConfig,
|
||||
options: ResolveNotionTokenOptions = {},
|
||||
): Promise<NotionPullConfig> {
|
||||
return notionPullConfigSchema.parse({
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ describe('createPostgresQueryExecutor', () => {
|
|||
expect(client.connect).toHaveBeenCalledTimes(1);
|
||||
expect(calls[0]).toBe('BEGIN READ ONLY');
|
||||
expect(calls[1]).toEqual({
|
||||
text: 'select * from (select status, count(*) as order_count from public.orders group by status) as klo_query_result limit 50',
|
||||
text: 'select * from (select status, count(*) as order_count from public.orders group by status) as ktx_query_result limit 50',
|
||||
rowMode: 'array',
|
||||
});
|
||||
expect(calls[2]).toBe('COMMIT');
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { Client, type ClientConfig } from 'pg';
|
||||
import type {
|
||||
KloSqlQueryExecutionInput,
|
||||
KloSqlQueryExecutionResult,
|
||||
KloSqlQueryExecutorPort,
|
||||
KtxSqlQueryExecutionInput,
|
||||
KtxSqlQueryExecutionResult,
|
||||
KtxSqlQueryExecutorPort,
|
||||
} from './query-executor.js';
|
||||
import { limitSqlForExecution } from './read-only-sql.js';
|
||||
|
||||
|
|
@ -24,7 +24,7 @@ interface PostgresQueryExecutorOptions {
|
|||
clientFactory?: (config: ClientConfig) => PgClientLike;
|
||||
}
|
||||
|
||||
function connectionDriver(input: KloSqlQueryExecutionInput): string {
|
||||
function connectionDriver(input: KtxSqlQueryExecutionInput): string {
|
||||
return String(input.connection?.driver ?? '').toLowerCase();
|
||||
}
|
||||
|
||||
|
|
@ -32,10 +32,10 @@ function createDefaultClient(config: ClientConfig): PgClientLike {
|
|||
return new Client(config);
|
||||
}
|
||||
|
||||
export function createPostgresQueryExecutor(options: PostgresQueryExecutorOptions = {}): KloSqlQueryExecutorPort {
|
||||
export function createPostgresQueryExecutor(options: PostgresQueryExecutorOptions = {}): KtxSqlQueryExecutorPort {
|
||||
const clientFactory = options.clientFactory ?? createDefaultClient;
|
||||
return {
|
||||
async execute(input: KloSqlQueryExecutionInput): Promise<KloSqlQueryExecutionResult> {
|
||||
async execute(input: KtxSqlQueryExecutionInput): Promise<KtxSqlQueryExecutionResult> {
|
||||
const driver = connectionDriver(input);
|
||||
if (driver !== 'postgres' && driver !== 'postgresql') {
|
||||
throw new Error(`Local Postgres execution cannot run driver "${input.connection?.driver ?? 'unknown'}".`);
|
||||
|
|
@ -52,7 +52,7 @@ export function createPostgresQueryExecutor(options: PostgresQueryExecutorOption
|
|||
statement_timeout: options.statementTimeoutMs ?? 30_000,
|
||||
query_timeout: options.queryTimeoutMs ?? 35_000,
|
||||
connectionTimeoutMillis: options.connectionTimeoutMs ?? 5_000,
|
||||
application_name: 'klo-local-query',
|
||||
application_name: 'ktx-local-query',
|
||||
});
|
||||
await client.connect();
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
import type { KloProjectConnectionConfig } from '../project/index.js';
|
||||
import type { KtxProjectConnectionConfig } from '../project/index.js';
|
||||
|
||||
export interface KloSqlQueryExecutionInput {
|
||||
export interface KtxSqlQueryExecutionInput {
|
||||
connectionId: string;
|
||||
projectDir?: string;
|
||||
connection: KloProjectConnectionConfig | undefined;
|
||||
connection: KtxProjectConnectionConfig | undefined;
|
||||
sql: string;
|
||||
maxRows?: number;
|
||||
}
|
||||
|
||||
export interface KloSqlQueryExecutionResult {
|
||||
export interface KtxSqlQueryExecutionResult {
|
||||
headers: string[];
|
||||
rows: unknown[][];
|
||||
totalRows: number;
|
||||
|
|
@ -16,8 +16,8 @@ export interface KloSqlQueryExecutionResult {
|
|||
rowCount: number | null;
|
||||
}
|
||||
|
||||
export interface KloSqlQueryExecutorPort {
|
||||
execute(input: KloSqlQueryExecutionInput): Promise<KloSqlQueryExecutionResult>;
|
||||
export interface KtxSqlQueryExecutorPort {
|
||||
execute(input: KtxSqlQueryExecutionInput): Promise<KtxSqlQueryExecutionResult>;
|
||||
}
|
||||
|
||||
export function normalizeQueryRows(rows: unknown[]): unknown[][] {
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ describe('assertReadOnlySql', () => {
|
|||
describe('limitSqlForExecution', () => {
|
||||
it('wraps compiled SQL and strips trailing semicolons', () => {
|
||||
expect(limitSqlForExecution('select * from public.orders; ', 25)).toBe(
|
||||
'select * from (select * from public.orders) as klo_query_result limit 25',
|
||||
'select * from (select * from public.orders) as ktx_query_result limit 25',
|
||||
);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -18,5 +18,5 @@ export function limitSqlForExecution(sql: string, maxRows: number | undefined):
|
|||
if (!Number.isInteger(maxRows) || maxRows <= 0) {
|
||||
throw new Error('maxRows must be a positive integer.');
|
||||
}
|
||||
return `select * from (${trimmed}) as klo_query_result limit ${maxRows}`;
|
||||
return `select * from (${trimmed}) as ktx_query_result limit ${maxRows}`;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ describe('createSqliteQueryExecutor', () => {
|
|||
let dbPath: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-sqlite-query-'));
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-sqlite-query-'));
|
||||
dbPath = join(tempDir, 'warehouse.db');
|
||||
const db = new Database(dbPath);
|
||||
db.exec(`
|
||||
|
|
@ -81,23 +81,23 @@ describe('createSqliteQueryExecutor', () => {
|
|||
});
|
||||
|
||||
it('resolves env references for SQLite database urls', async () => {
|
||||
const originalDatabaseUrl = process.env.KLO_SQLITE_TEST_URL;
|
||||
process.env.KLO_SQLITE_TEST_URL = `sqlite:${dbPath}`;
|
||||
const originalDatabaseUrl = process.env.KTX_SQLITE_TEST_URL;
|
||||
process.env.KTX_SQLITE_TEST_URL = `sqlite:${dbPath}`;
|
||||
|
||||
try {
|
||||
expect(
|
||||
sqliteDatabasePathFromConnection({
|
||||
connectionId: 'warehouse',
|
||||
projectDir: tempDir,
|
||||
connection: { driver: 'sqlite', url: 'env:KLO_SQLITE_TEST_URL', readonly: true },
|
||||
connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL', readonly: true },
|
||||
sql: 'select 1',
|
||||
}),
|
||||
).toBe(dbPath);
|
||||
} finally {
|
||||
if (originalDatabaseUrl === undefined) {
|
||||
delete process.env.KLO_SQLITE_TEST_URL;
|
||||
delete process.env.KTX_SQLITE_TEST_URL;
|
||||
} else {
|
||||
process.env.KLO_SQLITE_TEST_URL = originalDatabaseUrl;
|
||||
process.env.KTX_SQLITE_TEST_URL = originalDatabaseUrl;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -4,16 +4,16 @@ import Database from 'better-sqlite3';
|
|||
import { readFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import type {
|
||||
KloSqlQueryExecutionInput,
|
||||
KloSqlQueryExecutionResult,
|
||||
KloSqlQueryExecutorPort,
|
||||
KtxSqlQueryExecutionInput,
|
||||
KtxSqlQueryExecutionResult,
|
||||
KtxSqlQueryExecutorPort,
|
||||
} from './query-executor.js';
|
||||
import { normalizeQueryRows } from './query-executor.js';
|
||||
import { limitSqlForExecution } from './read-only-sql.js';
|
||||
|
||||
type SqliteConnectionConfig = Record<string, unknown> | undefined;
|
||||
|
||||
function connectionDriver(input: KloSqlQueryExecutionInput): string {
|
||||
function connectionDriver(input: KtxSqlQueryExecutionInput): string {
|
||||
return String(input.connection?.driver ?? '').toLowerCase();
|
||||
}
|
||||
|
||||
|
|
@ -49,7 +49,7 @@ function sqlitePathFromUrl(url: string): string {
|
|||
return url;
|
||||
}
|
||||
|
||||
export function sqliteDatabasePathFromConnection(input: KloSqlQueryExecutionInput): string {
|
||||
export function sqliteDatabasePathFromConnection(input: KtxSqlQueryExecutionInput): string {
|
||||
const driver = connectionDriver(input);
|
||||
if (driver !== 'sqlite' && driver !== 'sqlite3') {
|
||||
throw new Error(`Local SQLite execution cannot run driver "${input.connection?.driver ?? 'unknown'}".`);
|
||||
|
|
@ -70,9 +70,9 @@ export function sqliteDatabasePathFromConnection(input: KloSqlQueryExecutionInpu
|
|||
return isAbsolute(candidate) ? candidate : resolve(input.projectDir ?? process.cwd(), candidate);
|
||||
}
|
||||
|
||||
export function createSqliteQueryExecutor(): KloSqlQueryExecutorPort {
|
||||
export function createSqliteQueryExecutor(): KtxSqlQueryExecutorPort {
|
||||
return {
|
||||
async execute(input: KloSqlQueryExecutionInput): Promise<KloSqlQueryExecutionResult> {
|
||||
async execute(input: KtxSqlQueryExecutionInput): Promise<KtxSqlQueryExecutionResult> {
|
||||
const sql = limitSqlForExecution(input.sql, input.maxRows);
|
||||
const dbPath = sqliteDatabasePathFromConnection(input);
|
||||
const db = new Database(dbPath, { readonly: true, fileMustExist: true });
|
||||
|
|
|
|||
|
|
@ -2,33 +2,33 @@ import { mkdir, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { resolveKloConfigReference, resolveKloHomePath } from './config-reference.js';
|
||||
import { resolveKtxConfigReference, resolveKtxHomePath } from './config-reference.js';
|
||||
|
||||
describe('KLO config references', () => {
|
||||
describe('KTX config references', () => {
|
||||
it('resolves env references without returning empty values', () => {
|
||||
expect(resolveKloConfigReference('env:AI_GATEWAY_API_KEY', { AI_GATEWAY_API_KEY: ' gateway-key ' })).toBe(
|
||||
expect(resolveKtxConfigReference('env:AI_GATEWAY_API_KEY', { AI_GATEWAY_API_KEY: ' gateway-key ' })).toBe(
|
||||
'gateway-key',
|
||||
);
|
||||
expect(resolveKloConfigReference('env:AI_GATEWAY_API_KEY', { AI_GATEWAY_API_KEY: ' ' })).toBeUndefined();
|
||||
expect(resolveKloConfigReference('env:AI_GATEWAY_API_KEY', {})).toBeUndefined();
|
||||
expect(resolveKtxConfigReference('env:AI_GATEWAY_API_KEY', { AI_GATEWAY_API_KEY: ' ' })).toBeUndefined();
|
||||
expect(resolveKtxConfigReference('env:AI_GATEWAY_API_KEY', {})).toBeUndefined();
|
||||
});
|
||||
|
||||
it('resolves file references and trims file content', async () => {
|
||||
const dir = join(tmpdir(), `klo-config-reference-${process.pid}`);
|
||||
const dir = join(tmpdir(), `ktx-config-reference-${process.pid}`);
|
||||
await mkdir(dir, { recursive: true });
|
||||
const keyPath = join(dir, 'gateway-key.txt');
|
||||
await writeFile(keyPath, 'file-gateway-key\n', 'utf8');
|
||||
|
||||
expect(resolveKloConfigReference(`file:${keyPath}`, {})).toBe('file-gateway-key');
|
||||
expect(resolveKtxConfigReference(`file:${keyPath}`, {})).toBe('file-gateway-key');
|
||||
});
|
||||
|
||||
it('returns literal values unchanged after trimming blank-only values', () => {
|
||||
expect(resolveKloConfigReference('provider/model', {})).toBe('provider/model');
|
||||
expect(resolveKloConfigReference(' ', {})).toBeUndefined();
|
||||
expect(resolveKloConfigReference(undefined, {})).toBeUndefined();
|
||||
expect(resolveKtxConfigReference('provider/model', {})).toBe('provider/model');
|
||||
expect(resolveKtxConfigReference(' ', {})).toBeUndefined();
|
||||
expect(resolveKtxConfigReference(undefined, {})).toBeUndefined();
|
||||
});
|
||||
|
||||
it('resolves home-prefixed paths', () => {
|
||||
expect(resolveKloHomePath('~/klo/key.txt')).toContain('/klo/key.txt');
|
||||
expect(resolveKtxHomePath('~/ktx/key.txt')).toContain('/ktx/key.txt');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { readFileSync } from 'node:fs';
|
|||
import { homedir } from 'node:os';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
export function resolveKloHomePath(path: string): string {
|
||||
export function resolveKtxHomePath(path: string): string {
|
||||
if (path === '~') {
|
||||
return homedir();
|
||||
}
|
||||
|
|
@ -14,7 +14,7 @@ export function resolveKloHomePath(path: string): string {
|
|||
return resolve(path);
|
||||
}
|
||||
|
||||
export function resolveKloConfigReference(value: string | undefined, env: NodeJS.ProcessEnv): string | undefined {
|
||||
export function resolveKtxConfigReference(value: string | undefined, env: NodeJS.ProcessEnv): string | undefined {
|
||||
if (!value) {
|
||||
return undefined;
|
||||
}
|
||||
|
|
@ -26,7 +26,7 @@ export function resolveKloConfigReference(value: string | undefined, env: NodeJS
|
|||
}
|
||||
|
||||
if (value.startsWith('file:')) {
|
||||
const filePath = resolveKloHomePath(value.slice('file:'.length).trim());
|
||||
const filePath = resolveKtxHomePath(value.slice('file:'.length).trim());
|
||||
const fileValue = readFileSync(filePath, 'utf8').trim();
|
||||
return fileValue.length > 0 ? fileValue : undefined;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
export interface KloStorageConfig {
|
||||
export interface KtxStorageConfig {
|
||||
configDir?: string;
|
||||
homeDir?: string;
|
||||
worktreesDir?: string;
|
||||
}
|
||||
|
||||
export interface KloGitConfig {
|
||||
export interface KtxGitConfig {
|
||||
userName: string;
|
||||
userEmail: string;
|
||||
bootstrapMessage?: string;
|
||||
|
|
@ -12,31 +12,31 @@ export interface KloGitConfig {
|
|||
bootstrapAuthorEmail?: string;
|
||||
}
|
||||
|
||||
export interface KloCoreConfig {
|
||||
storage: KloStorageConfig;
|
||||
git: KloGitConfig;
|
||||
export interface KtxCoreConfig {
|
||||
storage: KtxStorageConfig;
|
||||
git: KtxGitConfig;
|
||||
}
|
||||
|
||||
export interface KloLogger {
|
||||
export interface KtxLogger {
|
||||
debug(message: string): void;
|
||||
log(message: string): void;
|
||||
warn(message: string): void;
|
||||
error(message: string, error?: unknown): void;
|
||||
}
|
||||
|
||||
export const noopLogger: KloLogger = {
|
||||
export const noopLogger: KtxLogger = {
|
||||
debug: () => undefined,
|
||||
log: () => undefined,
|
||||
warn: () => undefined,
|
||||
error: () => undefined,
|
||||
};
|
||||
|
||||
export function resolveConfigDir(config: KloCoreConfig): string {
|
||||
export function resolveConfigDir(config: KtxCoreConfig): string {
|
||||
const homeDir = config.storage.homeDir ?? '/tmp';
|
||||
return config.storage.configDir ?? `${homeDir}/klo/config`;
|
||||
return config.storage.configDir ?? `${homeDir}/ktx/config`;
|
||||
}
|
||||
|
||||
export function resolveWorktreesDir(config: KloCoreConfig): string {
|
||||
export function resolveWorktreesDir(config: KtxCoreConfig): string {
|
||||
const homeDir = config.storage.homeDir ?? '/tmp';
|
||||
return config.storage.worktreesDir ?? `${homeDir}/.worktrees`;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
export interface KloEmbeddingPort {
|
||||
export interface KtxEmbeddingPort {
|
||||
maxBatchSize: number;
|
||||
computeEmbedding(text: string): Promise<number[]>;
|
||||
computeEmbeddingsBulk(texts: string[]): Promise<number[][]>;
|
||||
|
|
|
|||
|
|
@ -1,18 +1,18 @@
|
|||
export interface KloFileWriteResult {
|
||||
export interface KtxFileWriteResult {
|
||||
commitHash?: string | null;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface KloFileReadResult {
|
||||
export interface KtxFileReadResult {
|
||||
content: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface KloFileListResult {
|
||||
export interface KtxFileListResult {
|
||||
files: string[];
|
||||
}
|
||||
|
||||
export interface KloFileHistoryEntry {
|
||||
export interface KtxFileHistoryEntry {
|
||||
sha?: string;
|
||||
message?: string;
|
||||
author?: string;
|
||||
|
|
@ -20,7 +20,7 @@ export interface KloFileHistoryEntry {
|
|||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface KloFileStorePort<TSelf = unknown> {
|
||||
export interface KtxFileStorePort<TSelf = unknown> {
|
||||
writeFile(
|
||||
path: string,
|
||||
content: string,
|
||||
|
|
@ -28,16 +28,16 @@ export interface KloFileStorePort<TSelf = unknown> {
|
|||
authorEmail: string,
|
||||
commitMessage: string,
|
||||
options?: { skipLock?: boolean },
|
||||
): Promise<KloFileWriteResult>;
|
||||
readFile(path: string): Promise<KloFileReadResult>;
|
||||
): Promise<KtxFileWriteResult>;
|
||||
readFile(path: string): Promise<KtxFileReadResult>;
|
||||
deleteFile(
|
||||
path: string,
|
||||
author: string,
|
||||
authorEmail: string,
|
||||
commitMessage: string,
|
||||
options?: { skipLock?: boolean },
|
||||
): Promise<KloFileWriteResult | null>;
|
||||
listFiles(path: string, recursive?: boolean): Promise<KloFileListResult>;
|
||||
getFileHistory(path: string): Promise<KloFileHistoryEntry[] | unknown>;
|
||||
): Promise<KtxFileWriteResult | null>;
|
||||
listFiles(path: string, recursive?: boolean): Promise<KtxFileListResult>;
|
||||
getFileHistory(path: string): Promise<KtxFileHistoryEntry[] | unknown>;
|
||||
forWorktree(workdir: string): TSelf;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { SimpleGit } from 'simple-git';
|
||||
import type { KloCoreConfig } from './config.js';
|
||||
import type { KtxCoreConfig } from './config.js';
|
||||
import { createSimpleGit } from './git-env.js';
|
||||
import { GitService } from './git.service.js';
|
||||
|
||||
|
|
@ -21,7 +21,7 @@ describe('GitService.assertWorktreeClean', () => {
|
|||
await writeFile(join(workdir, 'init'), 'init');
|
||||
await git.add('.');
|
||||
await git.commit('init');
|
||||
const coreConfig: KloCoreConfig = {
|
||||
const coreConfig: KtxCoreConfig = {
|
||||
storage: { configDir: workdir, homeDir: workdir },
|
||||
git: { userName: 'Test', userEmail: 't@test' },
|
||||
};
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { mkdir, mkdtemp, readdir, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { SimpleGit } from 'simple-git';
|
||||
import type { KloCoreConfig } from './config.js';
|
||||
import type { KtxCoreConfig } from './config.js';
|
||||
import { createSimpleGit } from './git-env.js';
|
||||
import { GitService } from './git.service.js';
|
||||
|
||||
|
|
@ -22,7 +22,7 @@ describe('GitService.deleteDirectories', () => {
|
|||
await git.add('.');
|
||||
await git.commit('init');
|
||||
|
||||
const coreConfig: KloCoreConfig = {
|
||||
const coreConfig: KtxCoreConfig = {
|
||||
storage: { configDir: workdir, homeDir: workdir },
|
||||
git: { userName: 'Test', userEmail: 't@test' },
|
||||
};
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { SimpleGit } from 'simple-git';
|
||||
import type { KloCoreConfig } from './config.js';
|
||||
import type { KtxCoreConfig } from './config.js';
|
||||
import { createSimpleGit } from './git-env.js';
|
||||
import { GitService } from './git.service.js';
|
||||
|
||||
|
|
@ -21,7 +21,7 @@ describe('GitService.resetHardTo', () => {
|
|||
await writeFile(join(workdir, 'init'), 'init');
|
||||
await git.add('.');
|
||||
await git.commit('init');
|
||||
const coreConfig: KloCoreConfig = {
|
||||
const coreConfig: KtxCoreConfig = {
|
||||
storage: { configDir: workdir, homeDir: workdir },
|
||||
git: { userName: 'Test', userEmail: 't@test' },
|
||||
};
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { mkdtemp, realpath, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import type { KloCoreConfig } from './config.js';
|
||||
import type { KtxCoreConfig } from './config.js';
|
||||
import { GitService } from './git.service.js';
|
||||
|
||||
// These tests drive a real git repo inside a temp directory — simple-git shells out to the
|
||||
|
|
@ -15,7 +15,7 @@ describe('GitService', () => {
|
|||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'git-service-spec-'));
|
||||
|
||||
const coreConfig: KloCoreConfig = {
|
||||
const coreConfig: KtxCoreConfig = {
|
||||
storage: { configDir: tempDir, homeDir: tempDir },
|
||||
git: {
|
||||
userName: 'Test User',
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { promises as fs } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import type { SimpleGit } from 'simple-git';
|
||||
import { noopLogger, resolveConfigDir, type KloCoreConfig, type KloLogger } from './config.js';
|
||||
import { noopLogger, resolveConfigDir, type KtxCoreConfig, type KtxLogger } from './config.js';
|
||||
import { createSimpleGit } from './git-env.js';
|
||||
|
||||
export interface GitCommitInfo {
|
||||
|
|
@ -32,13 +32,13 @@ export type SquashMergeResult =
|
|||
| { ok: false; conflict: true; conflictPaths: string[] };
|
||||
|
||||
export class GitService {
|
||||
private readonly logger: KloLogger;
|
||||
private readonly logger: KtxLogger;
|
||||
private git!: SimpleGit;
|
||||
private configDir: string;
|
||||
|
||||
constructor(
|
||||
private readonly config: KloCoreConfig,
|
||||
logger?: KloLogger,
|
||||
private readonly config: KtxCoreConfig,
|
||||
logger?: KtxLogger,
|
||||
) {
|
||||
this.logger = logger ?? noopLogger;
|
||||
this.configDir = resolveConfigDir(config);
|
||||
|
|
@ -73,10 +73,10 @@ export class GitService {
|
|||
// can rely on `revParseHead()` returning a SHA. Idempotent: skip if HEAD already exists.
|
||||
const head = await this.revParseHead();
|
||||
if (!head) {
|
||||
await this.git.commit(this.config.git.bootstrapMessage ?? 'Initialize klo project repository', {
|
||||
await this.git.commit(this.config.git.bootstrapMessage ?? 'Initialize ktx project repository', {
|
||||
'--allow-empty': null,
|
||||
'--author': `${this.config.git.bootstrapAuthor ?? 'klo system'} <${
|
||||
this.config.git.bootstrapAuthorEmail ?? 'system@klo.local'
|
||||
'--author': `${this.config.git.bootstrapAuthor ?? 'ktx system'} <${
|
||||
this.config.git.bootstrapAuthorEmail ?? 'system@ktx.local'
|
||||
}>`,
|
||||
});
|
||||
this.logger.log('Wrote bootstrap commit to config repo');
|
||||
|
|
@ -676,7 +676,7 @@ export class GitService {
|
|||
|
||||
/**
|
||||
* Remove the worktree entry and its on-disk directory. Uses `--force` because session
|
||||
* worktrees are klo-internal — a clean working tree is not required.
|
||||
* worktrees are ktx-internal — a clean working tree is not required.
|
||||
*/
|
||||
async removeWorktree(path: string): Promise<void> {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -1,19 +1,19 @@
|
|||
export type { KloCoreConfig, KloGitConfig, KloLogger, KloStorageConfig } from './config.js';
|
||||
export type { KtxCoreConfig, KtxGitConfig, KtxLogger, KtxStorageConfig } from './config.js';
|
||||
export { noopLogger, resolveConfigDir, resolveWorktreesDir } from './config.js';
|
||||
export { resolveKloConfigReference, resolveKloHomePath } from './config-reference.js';
|
||||
export type { KloEmbeddingPort } from './embedding.js';
|
||||
export { resolveKtxConfigReference, resolveKtxHomePath } from './config-reference.js';
|
||||
export type { KtxEmbeddingPort } from './embedding.js';
|
||||
export {
|
||||
REDACTED_KLO_CREDENTIAL_VALUE,
|
||||
redactKloSensitiveMetadata,
|
||||
redactKloSensitiveText,
|
||||
redactKloSensitiveValue,
|
||||
REDACTED_KTX_CREDENTIAL_VALUE,
|
||||
redactKtxSensitiveMetadata,
|
||||
redactKtxSensitiveText,
|
||||
redactKtxSensitiveValue,
|
||||
} from './redaction.js';
|
||||
export type {
|
||||
KloFileHistoryEntry,
|
||||
KloFileListResult,
|
||||
KloFileReadResult,
|
||||
KloFileStorePort,
|
||||
KloFileWriteResult,
|
||||
KtxFileHistoryEntry,
|
||||
KtxFileListResult,
|
||||
KtxFileReadResult,
|
||||
KtxFileStorePort,
|
||||
KtxFileWriteResult,
|
||||
} from './file-store.js';
|
||||
export type { GitCommitInfo, SquashMergeResult, WorktreeEntry } from './git.service.js';
|
||||
export { GitService } from './git.service.js';
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
export const REDACTED_KLO_CREDENTIAL_VALUE = '<redacted>';
|
||||
export const REDACTED_KTX_CREDENTIAL_VALUE = '<redacted>';
|
||||
|
||||
const SENSITIVE_FIELD_NAME = /(password|secret|token|api[_-]?key|private[_-]?key|passphrase|credential|authorization|url)/i;
|
||||
const URL_CREDENTIAL_PATTERN = /([a-z][a-z0-9+.-]*:\/\/[^:\s/@]+:)([^@\s/]+)(@)/gi;
|
||||
|
|
@ -11,37 +11,37 @@ function isSensitiveField(key: string): boolean {
|
|||
return SENSITIVE_FIELD_NAME.test(key);
|
||||
}
|
||||
|
||||
export function redactKloSensitiveValue(key: string, value: unknown): unknown {
|
||||
export function redactKtxSensitiveValue(key: string, value: unknown): unknown {
|
||||
if (isSensitiveField(key)) {
|
||||
return REDACTED_KLO_CREDENTIAL_VALUE;
|
||||
return REDACTED_KTX_CREDENTIAL_VALUE;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((item) => redactKloSensitiveValue(key, item));
|
||||
return value.map((item) => redactKtxSensitiveValue(key, item));
|
||||
}
|
||||
if (isRecord(value)) {
|
||||
return redactKloSensitiveMetadata(value);
|
||||
return redactKtxSensitiveMetadata(value);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function redactKloSensitiveMetadata(metadata: Record<string, unknown>): Record<string, unknown> {
|
||||
export function redactKtxSensitiveMetadata(metadata: Record<string, unknown>): Record<string, unknown> {
|
||||
const redacted: Record<string, unknown> = {};
|
||||
for (const [key, value] of Object.entries(metadata)) {
|
||||
if (Array.isArray(value)) {
|
||||
redacted[key] = value.map((item) =>
|
||||
isRecord(item) ? redactKloSensitiveMetadata(item) : redactKloSensitiveValue(key, item),
|
||||
isRecord(item) ? redactKtxSensitiveMetadata(item) : redactKtxSensitiveValue(key, item),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (isRecord(value)) {
|
||||
redacted[key] = redactKloSensitiveValue(key, value);
|
||||
redacted[key] = redactKtxSensitiveValue(key, value);
|
||||
continue;
|
||||
}
|
||||
redacted[key] = redactKloSensitiveValue(key, value);
|
||||
redacted[key] = redactKtxSensitiveValue(key, value);
|
||||
}
|
||||
return redacted;
|
||||
}
|
||||
|
||||
export function redactKloSensitiveText(value: string): string {
|
||||
return value.replace(URL_CREDENTIAL_PATTERN, `$1${REDACTED_KLO_CREDENTIAL_VALUE}$3`);
|
||||
export function redactKtxSensitiveText(value: string): string {
|
||||
return value.replace(URL_CREDENTIAL_PATTERN, `$1${REDACTED_KTX_CREDENTIAL_VALUE}$3`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { mkdtemp, realpath, rm, stat } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { KloCoreConfig } from './config.js';
|
||||
import type { KtxCoreConfig } from './config.js';
|
||||
import { GitService } from './git.service.js';
|
||||
import { SessionWorktreeService, type WorktreeConfigPort } from './session-worktree.service.js';
|
||||
|
||||
|
|
@ -20,7 +20,7 @@ describe('SessionWorktreeService', () => {
|
|||
homeDir = await mkdtemp(join(tmpdir(), 'sws-spec-'));
|
||||
homeDir = await realpath(homeDir);
|
||||
|
||||
const coreConfig: KloCoreConfig = {
|
||||
const coreConfig: KtxCoreConfig = {
|
||||
storage: { configDir: homeDir, homeDir },
|
||||
git: {
|
||||
userName: 'System User',
|
||||
|
|
@ -113,7 +113,7 @@ describe('SessionWorktreeService', () => {
|
|||
await expect(stat(session.workdir)).resolves.toBeTruthy();
|
||||
|
||||
const { readFile } = await import('node:fs/promises');
|
||||
const raw = await readFile(join(session.workdir, '.klo-outcome'), 'utf-8');
|
||||
const raw = await readFile(join(session.workdir, '.ktx-outcome'), 'utf-8');
|
||||
const parsed = JSON.parse(raw);
|
||||
expect(parsed.outcome).toBe('conflict');
|
||||
expect(parsed.chatId).toBe('chat-cleanup-conflict');
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { mkdir, stat, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { noopLogger, resolveWorktreesDir, type KloCoreConfig, type KloLogger } from './config.js';
|
||||
import { noopLogger, resolveWorktreesDir, type KtxCoreConfig, type KtxLogger } from './config.js';
|
||||
import { GitService } from './git.service.js';
|
||||
|
||||
export type SessionOutcome = 'success' | 'empty' | 'conflict' | 'crash';
|
||||
|
|
@ -28,14 +28,14 @@ export interface SessionWorktree<TConfig> {
|
|||
}
|
||||
|
||||
export interface SessionWorktreeServiceDeps<TConfig extends WorktreeConfigPort<TConfig>> {
|
||||
coreConfig: KloCoreConfig;
|
||||
coreConfig: KtxCoreConfig;
|
||||
gitService: GitService;
|
||||
configService: TConfig;
|
||||
logger?: KloLogger;
|
||||
logger?: KtxLogger;
|
||||
}
|
||||
|
||||
export class SessionWorktreeService<TConfig extends WorktreeConfigPort<TConfig> = WorktreeConfigPort<never>> {
|
||||
private readonly logger: KloLogger;
|
||||
private readonly logger: KtxLogger;
|
||||
private readonly worktreesRoot: string;
|
||||
|
||||
constructor(private readonly deps: SessionWorktreeServiceDeps<TConfig>) {
|
||||
|
|
@ -101,7 +101,7 @@ export class SessionWorktreeService<TConfig extends WorktreeConfigPort<TConfig>
|
|||
...(extra?.conflictPaths ? { conflictPaths: extra.conflictPaths } : {}),
|
||||
};
|
||||
try {
|
||||
await writeFile(join(session.workdir, '.klo-outcome'), JSON.stringify(payload, null, 2), 'utf-8');
|
||||
await writeFile(join(session.workdir, '.ktx-outcome'), JSON.stringify(payload, null, 2), 'utf-8');
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`cleanup(${outcome}) failed to write sentinel for ${session.chatId}: ${
|
||||
|
|
|
|||
|
|
@ -4,21 +4,21 @@ import { URL } from 'node:url';
|
|||
import { spawn } from 'node:child_process';
|
||||
import type { SemanticLayerQueryInput, SemanticLayerSource } from '../sl/index.js';
|
||||
|
||||
export interface KloSemanticLayerComputeQueryResult {
|
||||
export interface KtxSemanticLayerComputeQueryResult {
|
||||
sql: string;
|
||||
dialect: string;
|
||||
columns: Array<Record<string, unknown>>;
|
||||
plan: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface KloSemanticLayerComputeValidationResult {
|
||||
export interface KtxSemanticLayerComputeValidationResult {
|
||||
valid: boolean;
|
||||
errors: string[];
|
||||
warnings: string[];
|
||||
perSourceWarnings: Record<string, string[]>;
|
||||
}
|
||||
|
||||
export interface KloSemanticLayerSourceGenerationColumnInput {
|
||||
export interface KtxSemanticLayerSourceGenerationColumnInput {
|
||||
name: string;
|
||||
type: string;
|
||||
primaryKey?: boolean;
|
||||
|
|
@ -26,15 +26,15 @@ export interface KloSemanticLayerSourceGenerationColumnInput {
|
|||
comment?: string | null;
|
||||
}
|
||||
|
||||
export interface KloSemanticLayerSourceGenerationTableInput {
|
||||
export interface KtxSemanticLayerSourceGenerationTableInput {
|
||||
name: string;
|
||||
catalog?: string | null;
|
||||
db?: string | null;
|
||||
comment?: string | null;
|
||||
columns: KloSemanticLayerSourceGenerationColumnInput[];
|
||||
columns: KtxSemanticLayerSourceGenerationColumnInput[];
|
||||
}
|
||||
|
||||
export interface KloSemanticLayerSourceGenerationLinkInput {
|
||||
export interface KtxSemanticLayerSourceGenerationLinkInput {
|
||||
fromTable: string;
|
||||
fromColumn: string;
|
||||
toTable: string;
|
||||
|
|
@ -42,57 +42,57 @@ export interface KloSemanticLayerSourceGenerationLinkInput {
|
|||
relationshipType: string;
|
||||
}
|
||||
|
||||
export interface KloSemanticLayerSourceGenerationInput {
|
||||
tables: KloSemanticLayerSourceGenerationTableInput[];
|
||||
links: KloSemanticLayerSourceGenerationLinkInput[];
|
||||
export interface KtxSemanticLayerSourceGenerationInput {
|
||||
tables: KtxSemanticLayerSourceGenerationTableInput[];
|
||||
links: KtxSemanticLayerSourceGenerationLinkInput[];
|
||||
dialect?: string;
|
||||
}
|
||||
|
||||
export interface KloSemanticLayerSourceGenerationResult {
|
||||
export interface KtxSemanticLayerSourceGenerationResult {
|
||||
sources: Array<Record<string, unknown>>;
|
||||
sourceCount: number;
|
||||
}
|
||||
|
||||
export interface KloSemanticLayerComputePort {
|
||||
export interface KtxSemanticLayerComputePort {
|
||||
query(input: {
|
||||
sources: Array<Record<string, unknown> | SemanticLayerSource>;
|
||||
query: SemanticLayerQueryInput;
|
||||
dialect: string;
|
||||
}): Promise<KloSemanticLayerComputeQueryResult>;
|
||||
}): Promise<KtxSemanticLayerComputeQueryResult>;
|
||||
validateSources(input: {
|
||||
sources: Array<Record<string, unknown> | SemanticLayerSource>;
|
||||
dialect: string;
|
||||
recentlyTouched?: string[];
|
||||
}): Promise<KloSemanticLayerComputeValidationResult>;
|
||||
generateSources(input: KloSemanticLayerSourceGenerationInput): Promise<KloSemanticLayerSourceGenerationResult>;
|
||||
}): Promise<KtxSemanticLayerComputeValidationResult>;
|
||||
generateSources(input: KtxSemanticLayerSourceGenerationInput): Promise<KtxSemanticLayerSourceGenerationResult>;
|
||||
}
|
||||
|
||||
export type KloDaemonCommand = 'semantic-query' | 'semantic-validate' | 'semantic-generate-sources';
|
||||
export type KtxDaemonCommand = 'semantic-query' | 'semantic-validate' | 'semantic-generate-sources';
|
||||
|
||||
export type KloDaemonJsonRunner = (
|
||||
subcommand: KloDaemonCommand,
|
||||
export type KtxDaemonJsonRunner = (
|
||||
subcommand: KtxDaemonCommand,
|
||||
payload: Record<string, unknown>,
|
||||
) => Promise<Record<string, unknown>>;
|
||||
|
||||
export type KloDaemonHttpJsonRunner = (path: string, payload: Record<string, unknown>) => Promise<Record<string, unknown>>;
|
||||
export type KtxDaemonHttpJsonRunner = (path: string, payload: Record<string, unknown>) => Promise<Record<string, unknown>>;
|
||||
|
||||
export interface PythonSemanticLayerComputeOptions {
|
||||
command?: string;
|
||||
args?: string[];
|
||||
cwd?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
runJson?: KloDaemonJsonRunner;
|
||||
runJson?: KtxDaemonJsonRunner;
|
||||
}
|
||||
|
||||
export interface HttpSemanticLayerComputeOptions {
|
||||
baseUrl: string;
|
||||
requestJson?: KloDaemonHttpJsonRunner;
|
||||
requestJson?: KtxDaemonHttpJsonRunner;
|
||||
}
|
||||
|
||||
function parseJsonObject(raw: string, subcommand: string): Record<string, unknown> {
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
throw new Error(`klo-daemon ${subcommand} returned non-object JSON`);
|
||||
throw new Error(`ktx-daemon ${subcommand} returned non-object JSON`);
|
||||
}
|
||||
return parsed as Record<string, unknown>;
|
||||
}
|
||||
|
|
@ -100,8 +100,8 @@ function parseJsonObject(raw: string, subcommand: string): Record<string, unknow
|
|||
function runProcessJson(
|
||||
options: Required<Pick<PythonSemanticLayerComputeOptions, 'command' | 'args'>> &
|
||||
Pick<PythonSemanticLayerComputeOptions, 'cwd' | 'env'>,
|
||||
): KloDaemonJsonRunner {
|
||||
return async (subcommand: KloDaemonCommand, payload: Record<string, unknown>): Promise<Record<string, unknown>> =>
|
||||
): KtxDaemonJsonRunner {
|
||||
return async (subcommand: KtxDaemonCommand, payload: Record<string, unknown>): Promise<Record<string, unknown>> =>
|
||||
new Promise((resolve, reject) => {
|
||||
const child = spawn(options.command, [...options.args, subcommand], {
|
||||
cwd: options.cwd,
|
||||
|
|
@ -118,7 +118,7 @@ function runProcessJson(
|
|||
const stdoutText = Buffer.concat(stdout).toString('utf8').trim();
|
||||
const stderrText = Buffer.concat(stderr).toString('utf8').trim();
|
||||
if (code !== 0) {
|
||||
reject(new Error(`klo-daemon ${subcommand} failed: ${stderrText || `exit code ${code}`}`));
|
||||
reject(new Error(`ktx-daemon ${subcommand} failed: ${stderrText || `exit code ${code}`}`));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
|
|
@ -135,7 +135,7 @@ function normalizedBaseUrl(baseUrl: string): string {
|
|||
return baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`;
|
||||
}
|
||||
|
||||
function postJson(baseUrl: string): KloDaemonHttpJsonRunner {
|
||||
function postJson(baseUrl: string): KtxDaemonHttpJsonRunner {
|
||||
return async (path, payload) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const target = new URL(path.replace(/^\//, ''), normalizedBaseUrl(baseUrl));
|
||||
|
|
@ -158,7 +158,7 @@ function postJson(baseUrl: string): KloDaemonHttpJsonRunner {
|
|||
const text = Buffer.concat(chunks).toString('utf8');
|
||||
const statusCode = response.statusCode ?? 0;
|
||||
if (statusCode < 200 || statusCode >= 300) {
|
||||
reject(new Error(`klo-daemon HTTP ${path} failed with ${statusCode}: ${text}`));
|
||||
reject(new Error(`ktx-daemon HTTP ${path} failed with ${statusCode}: ${text}`));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
|
|
@ -190,7 +190,7 @@ function recordArray(value: unknown): Array<Record<string, unknown>> {
|
|||
: [];
|
||||
}
|
||||
|
||||
function sourceGenerationPayload(input: KloSemanticLayerSourceGenerationInput): Record<string, unknown> {
|
||||
function sourceGenerationPayload(input: KtxSemanticLayerSourceGenerationInput): Record<string, unknown> {
|
||||
return {
|
||||
tables: input.tables.map((table) => ({
|
||||
name: table.name,
|
||||
|
|
@ -216,7 +216,7 @@ function sourceGenerationPayload(input: KloSemanticLayerSourceGenerationInput):
|
|||
};
|
||||
}
|
||||
|
||||
function sourceGenerationResult(raw: Record<string, unknown>): KloSemanticLayerSourceGenerationResult {
|
||||
function sourceGenerationResult(raw: Record<string, unknown>): KtxSemanticLayerSourceGenerationResult {
|
||||
return {
|
||||
sources: recordArray(raw.sources),
|
||||
sourceCount: typeof raw.source_count === 'number' ? raw.source_count : recordArray(raw.sources).length,
|
||||
|
|
@ -225,9 +225,9 @@ function sourceGenerationResult(raw: Record<string, unknown>): KloSemanticLayerS
|
|||
|
||||
export function createPythonSemanticLayerComputePort(
|
||||
options: PythonSemanticLayerComputeOptions = {},
|
||||
): KloSemanticLayerComputePort {
|
||||
): KtxSemanticLayerComputePort {
|
||||
const command = options.command ?? 'python';
|
||||
const args = options.args ?? ['-m', 'klo_daemon'];
|
||||
const args = options.args ?? ['-m', 'ktx_daemon'];
|
||||
const runJson = options.runJson ?? runProcessJson({ command, args, cwd: options.cwd, env: options.env });
|
||||
|
||||
return {
|
||||
|
|
@ -266,7 +266,7 @@ export function createPythonSemanticLayerComputePort(
|
|||
|
||||
export function createHttpSemanticLayerComputePort(
|
||||
options: HttpSemanticLayerComputeOptions,
|
||||
): KloSemanticLayerComputePort {
|
||||
): KtxSemanticLayerComputePort {
|
||||
const requestJson = options.requestJson ?? postJson(options.baseUrl);
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { kloContextPackageInfo } from './index.js';
|
||||
import { ktxContextPackageInfo } from './index.js';
|
||||
|
||||
describe('kloContextPackageInfo', () => {
|
||||
describe('ktxContextPackageInfo', () => {
|
||||
it('identifies the context package', () => {
|
||||
expect(kloContextPackageInfo).toEqual({
|
||||
name: '@klo/context',
|
||||
expect(ktxContextPackageInfo).toEqual({
|
||||
name: '@ktx/context',
|
||||
version: '0.0.0-private',
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
export interface KloContextPackageInfo {
|
||||
name: '@klo/context';
|
||||
export interface KtxContextPackageInfo {
|
||||
name: '@ktx/context';
|
||||
version: '0.0.0-private';
|
||||
}
|
||||
|
||||
export const kloContextPackageInfo: KloContextPackageInfo = {
|
||||
name: '@klo/context',
|
||||
export const ktxContextPackageInfo: KtxContextPackageInfo = {
|
||||
name: '@ktx/context',
|
||||
version: '0.0.0-private',
|
||||
};
|
||||
|
||||
|
|
@ -36,107 +36,107 @@ export * from './prompts/index.js';
|
|||
export * from './search/index.js';
|
||||
export * from './sql-analysis/index.js';
|
||||
export type {
|
||||
KloColumnAnalysisResult,
|
||||
KloColumnDescriptionPromptInput,
|
||||
KloColumnEmbeddingForeignKeys,
|
||||
KloColumnEmbeddingTextInput,
|
||||
KloColumnSampleInput,
|
||||
KloColumnSampleResult,
|
||||
KloColumnSampleUpdate,
|
||||
KloColumnStatsInput,
|
||||
KloColumnStatsResult,
|
||||
KloConnectionDriver,
|
||||
KloConnectorCapabilities,
|
||||
KloCredentialEnvelope,
|
||||
KloCredentialEnvReference,
|
||||
KloCredentialFileReference,
|
||||
KloDataDictionaryColumnState,
|
||||
KloDataDictionarySampleDecision,
|
||||
KloDataDictionarySettings,
|
||||
KloDataDictionarySkipReason,
|
||||
KloDataSourceDescriptionPromptInput,
|
||||
KloDescriptionCachePort,
|
||||
KloDescriptionColumn,
|
||||
KloDescriptionColumnTable,
|
||||
KloDescriptionGenerationSettings,
|
||||
KloDescriptionGeneratorOptions,
|
||||
KloDescriptionSource,
|
||||
KloDescriptionTableInput,
|
||||
KloDescriptionUpdate,
|
||||
KloEmbeddingPort as KloScanEmbeddingPort,
|
||||
KloEmbeddingUpdate,
|
||||
KloEnrichedColumn,
|
||||
KloEnrichedRelationship,
|
||||
KloEnrichedSchema,
|
||||
KloEnrichedTable,
|
||||
KloEnrichmentScanPhaseResult,
|
||||
KloGenerateColumnDescriptionsInput,
|
||||
KloGenerateDataSourceDescriptionInput,
|
||||
KloGenerateTableDescriptionInput,
|
||||
KloOptionalConnectorCapabilities,
|
||||
KloProgressPort,
|
||||
KloQueryResult as KloScanQueryResult,
|
||||
KloReadOnlyQueryInput,
|
||||
KloRelationshipEndpoint,
|
||||
KloRelationshipSource,
|
||||
KloRelationshipType,
|
||||
KloRelationshipUpdate,
|
||||
KloResolvedCredentialEnvelope,
|
||||
KloScanArtifactPaths,
|
||||
KloScanConnector,
|
||||
KloScanContext,
|
||||
KloScanDiffSummary,
|
||||
KloScanEnrichmentSummary,
|
||||
KloScanInput,
|
||||
KloScanLoggerPort,
|
||||
KloScanMetadataStore,
|
||||
KloScanMode,
|
||||
KloScanOrchestratorOptions,
|
||||
KloScanOrchestratorRunInput,
|
||||
KloScanOrchestratorRunResult,
|
||||
KloScanRelationshipSummary,
|
||||
KloScanReport,
|
||||
KloScanTrigger,
|
||||
KloScanWarning,
|
||||
KloScanWarningCode,
|
||||
KloSchemaColumn,
|
||||
KloSchemaDimensionType,
|
||||
KloSchemaForeignKey,
|
||||
KloSchemaScope,
|
||||
KloSchemaSnapshot,
|
||||
KloSchemaTable,
|
||||
KloSchemaTableKind,
|
||||
KloSkippedRelationship,
|
||||
KloStructuralScanPhaseResult,
|
||||
KloStructuralSyncPlan,
|
||||
KloStructuralSyncStats,
|
||||
KloTableDescriptionPromptInput,
|
||||
KloTableRef,
|
||||
KloTableSampleInput,
|
||||
KloTableSampleResult,
|
||||
KloColumnTypeMapping,
|
||||
KtxColumnAnalysisResult,
|
||||
KtxColumnDescriptionPromptInput,
|
||||
KtxColumnEmbeddingForeignKeys,
|
||||
KtxColumnEmbeddingTextInput,
|
||||
KtxColumnSampleInput,
|
||||
KtxColumnSampleResult,
|
||||
KtxColumnSampleUpdate,
|
||||
KtxColumnStatsInput,
|
||||
KtxColumnStatsResult,
|
||||
KtxConnectionDriver,
|
||||
KtxConnectorCapabilities,
|
||||
KtxCredentialEnvelope,
|
||||
KtxCredentialEnvReference,
|
||||
KtxCredentialFileReference,
|
||||
KtxDataDictionaryColumnState,
|
||||
KtxDataDictionarySampleDecision,
|
||||
KtxDataDictionarySettings,
|
||||
KtxDataDictionarySkipReason,
|
||||
KtxDataSourceDescriptionPromptInput,
|
||||
KtxDescriptionCachePort,
|
||||
KtxDescriptionColumn,
|
||||
KtxDescriptionColumnTable,
|
||||
KtxDescriptionGenerationSettings,
|
||||
KtxDescriptionGeneratorOptions,
|
||||
KtxDescriptionSource,
|
||||
KtxDescriptionTableInput,
|
||||
KtxDescriptionUpdate,
|
||||
KtxEmbeddingPort as KtxScanEmbeddingPort,
|
||||
KtxEmbeddingUpdate,
|
||||
KtxEnrichedColumn,
|
||||
KtxEnrichedRelationship,
|
||||
KtxEnrichedSchema,
|
||||
KtxEnrichedTable,
|
||||
KtxEnrichmentScanPhaseResult,
|
||||
KtxGenerateColumnDescriptionsInput,
|
||||
KtxGenerateDataSourceDescriptionInput,
|
||||
KtxGenerateTableDescriptionInput,
|
||||
KtxOptionalConnectorCapabilities,
|
||||
KtxProgressPort,
|
||||
KtxQueryResult as KtxScanQueryResult,
|
||||
KtxReadOnlyQueryInput,
|
||||
KtxRelationshipEndpoint,
|
||||
KtxRelationshipSource,
|
||||
KtxRelationshipType,
|
||||
KtxRelationshipUpdate,
|
||||
KtxResolvedCredentialEnvelope,
|
||||
KtxScanArtifactPaths,
|
||||
KtxScanConnector,
|
||||
KtxScanContext,
|
||||
KtxScanDiffSummary,
|
||||
KtxScanEnrichmentSummary,
|
||||
KtxScanInput,
|
||||
KtxScanLoggerPort,
|
||||
KtxScanMetadataStore,
|
||||
KtxScanMode,
|
||||
KtxScanOrchestratorOptions,
|
||||
KtxScanOrchestratorRunInput,
|
||||
KtxScanOrchestratorRunResult,
|
||||
KtxScanRelationshipSummary,
|
||||
KtxScanReport,
|
||||
KtxScanTrigger,
|
||||
KtxScanWarning,
|
||||
KtxScanWarningCode,
|
||||
KtxSchemaColumn,
|
||||
KtxSchemaDimensionType,
|
||||
KtxSchemaForeignKey,
|
||||
KtxSchemaScope,
|
||||
KtxSchemaSnapshot,
|
||||
KtxSchemaTable,
|
||||
KtxSchemaTableKind,
|
||||
KtxSkippedRelationship,
|
||||
KtxStructuralScanPhaseResult,
|
||||
KtxStructuralSyncPlan,
|
||||
KtxStructuralSyncStats,
|
||||
KtxTableDescriptionPromptInput,
|
||||
KtxTableRef,
|
||||
KtxTableSampleInput,
|
||||
KtxTableSampleResult,
|
||||
KtxColumnTypeMapping,
|
||||
} from './scan/index.js';
|
||||
export {
|
||||
appendKloWordLimitInstruction,
|
||||
buildKloColumnDescriptionPrompt,
|
||||
buildKloColumnEmbeddingText,
|
||||
buildKloDataSourceDescriptionPrompt,
|
||||
buildKloTableDescriptionPrompt,
|
||||
createKloConnectorCapabilities,
|
||||
defaultKloDataDictionarySettings,
|
||||
inferKloDimensionType,
|
||||
isKloDataDictionaryCandidate,
|
||||
kloColumnTypeMappingFromNative,
|
||||
KloDescriptionGenerator,
|
||||
KloScanOrchestrator,
|
||||
normalizeKloNativeType,
|
||||
REDACTED_KLO_CREDENTIAL_VALUE,
|
||||
redactKloCredentialEnvelope,
|
||||
redactKloCredentialValue,
|
||||
redactKloScanMetadata,
|
||||
redactKloScanReport,
|
||||
redactKloScanWarning,
|
||||
shouldKloSampleColumnForDictionary,
|
||||
appendKtxWordLimitInstruction,
|
||||
buildKtxColumnDescriptionPrompt,
|
||||
buildKtxColumnEmbeddingText,
|
||||
buildKtxDataSourceDescriptionPrompt,
|
||||
buildKtxTableDescriptionPrompt,
|
||||
createKtxConnectorCapabilities,
|
||||
defaultKtxDataDictionarySettings,
|
||||
inferKtxDimensionType,
|
||||
isKtxDataDictionaryCandidate,
|
||||
ktxColumnTypeMappingFromNative,
|
||||
KtxDescriptionGenerator,
|
||||
KtxScanOrchestrator,
|
||||
normalizeKtxNativeType,
|
||||
REDACTED_KTX_CREDENTIAL_VALUE,
|
||||
redactKtxCredentialEnvelope,
|
||||
redactKtxCredentialValue,
|
||||
redactKtxScanMetadata,
|
||||
redactKtxScanReport,
|
||||
redactKtxScanWarning,
|
||||
shouldKtxSampleColumnForDictionary,
|
||||
} from './scan/index.js';
|
||||
export * from './skills/index.js';
|
||||
export * from './sl/index.js';
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type { DbtParsedTable } from './parse-schema.js';
|
||||
import { findMatchingKloTable, matchDbtTables, type DbtHostTableLite } from './match-tables.js';
|
||||
import { findMatchingKtxTable, matchDbtTables, type DbtHostTableLite } from './match-tables.js';
|
||||
|
||||
const hostTables: DbtHostTableLite[] = [
|
||||
{ id: '1', name: 'orders', catalog: 'warehouse', db: 'analytics', columns: [{ id: 'c1', name: 'id' }] },
|
||||
|
|
@ -23,20 +23,20 @@ function table(input: Partial<DbtParsedTable>): DbtParsedTable {
|
|||
describe('dbt descriptions table matching', () => {
|
||||
it('uses schema plus name first and checks catalog when dbt database is present', () => {
|
||||
expect(
|
||||
findMatchingKloTable(table({ database: 'warehouse', schema: 'analytics' }), hostTables, null)?.id,
|
||||
findMatchingKtxTable(table({ database: 'warehouse', schema: 'analytics' }), hostTables, null)?.id,
|
||||
).toBe('1');
|
||||
});
|
||||
|
||||
it('does not fall back to name-only for source tables', () => {
|
||||
expect(findMatchingKloTable(table({ resourceType: 'source' }), hostTables, null)).toBeUndefined();
|
||||
expect(findMatchingKtxTable(table({ resourceType: 'source' }), hostTables, null)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('uses targetSchema for models and name-only only when unique', () => {
|
||||
expect(findMatchingKloTable(table({ resourceType: 'model' }), hostTables, 'staging')?.id).toBe('2');
|
||||
expect(findMatchingKloTable(table({ name: 'customers', resourceType: 'model' }), hostTables, null)?.id).toBe(
|
||||
expect(findMatchingKtxTable(table({ resourceType: 'model' }), hostTables, 'staging')?.id).toBe('2');
|
||||
expect(findMatchingKtxTable(table({ name: 'customers', resourceType: 'model' }), hostTables, null)?.id).toBe(
|
||||
'3',
|
||||
);
|
||||
expect(findMatchingKloTable(table({ resourceType: 'model' }), hostTables, null)).toBeUndefined();
|
||||
expect(findMatchingKtxTable(table({ resourceType: 'model' }), hostTables, null)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('summarizes matched columns and descriptions', () => {
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ export function matchDbtTables(
|
|||
targetSchema?: string | null,
|
||||
): DbtTableMatch[] {
|
||||
return dbtTables.map((dbtTable) => {
|
||||
const hostTable = findMatchingKloTable(dbtTable, hostTables, targetSchema);
|
||||
const hostTable = findMatchingKtxTable(dbtTable, hostTables, targetSchema);
|
||||
|
||||
if (!hostTable) {
|
||||
return {
|
||||
|
|
@ -63,7 +63,7 @@ export function matchDbtTables(
|
|||
});
|
||||
}
|
||||
|
||||
export function findMatchingKloTable(
|
||||
export function findMatchingKtxTable(
|
||||
dbtTable: DbtParsedTable,
|
||||
hostTables: DbtHostTableLite[],
|
||||
targetSchema?: string | null,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { parse as parseYaml } from 'yaml';
|
||||
import { type KloLogger, noopLogger } from '../../../core/index.js';
|
||||
import { type KtxLogger, noopLogger } from '../../../core/index.js';
|
||||
import { resolveJinjaVariables } from '../../dbt-shared/project-vars.js';
|
||||
|
||||
export interface DbtParsedColumn {
|
||||
|
|
@ -65,7 +65,7 @@ interface ParseDbtSchemaOptions {
|
|||
path?: string;
|
||||
variables?: Map<string, string>;
|
||||
projectName?: string | null;
|
||||
logger?: KloLogger;
|
||||
logger?: KtxLogger;
|
||||
}
|
||||
|
||||
interface DbtSchemaYaml {
|
||||
|
|
@ -133,7 +133,7 @@ export function parseDbtSchemaFile(content: string, options: ParseDbtSchemaOptio
|
|||
export function parseDbtSchemaFiles(
|
||||
files: DbtSchemaFile[],
|
||||
variables?: Map<string, string>,
|
||||
options: { projectName?: string | null; logger?: KloLogger } = {},
|
||||
options: { projectName?: string | null; logger?: KtxLogger } = {},
|
||||
): DbtSchemaParseResult {
|
||||
return new DbtSchemaParser(options.logger ?? noopLogger).parseFiles(files, variables, options.projectName ?? null);
|
||||
}
|
||||
|
|
@ -147,7 +147,7 @@ export function computeDbtSchemaHash(files: DbtSchemaFile[]): string {
|
|||
}
|
||||
|
||||
class DbtSchemaParser {
|
||||
constructor(private readonly logger: KloLogger) {}
|
||||
constructor(private readonly logger: KtxLogger) {}
|
||||
|
||||
parseFile(yamlContent: string, options: ParseDbtSchemaOptions = {}): DbtSchemaParseResult {
|
||||
this.logger.debug(`Parsing schema file: ${options.path ?? 'unknown'}`);
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import type { KloDescriptionUpdate } from '../../../scan/enrichment-types.js';
|
||||
import { findMatchingKloTable, type DbtHostTableLite } from './match-tables.js';
|
||||
import type { KtxDescriptionUpdate } from '../../../scan/enrichment-types.js';
|
||||
import { findMatchingKtxTable, type DbtHostTableLite } from './match-tables.js';
|
||||
import type { DbtSchemaParseResult } from './parse-schema.js';
|
||||
|
||||
export interface DbtDescriptionUpdates {
|
||||
dbt: KloDescriptionUpdate[];
|
||||
aiInvalidations: KloDescriptionUpdate[];
|
||||
dbt: KtxDescriptionUpdate[];
|
||||
aiInvalidations: KtxDescriptionUpdate[];
|
||||
}
|
||||
|
||||
export function toDescriptionUpdates(input: {
|
||||
|
|
@ -13,11 +13,11 @@ export function toDescriptionUpdates(input: {
|
|||
hostTables: DbtHostTableLite[];
|
||||
targetSchema: string | null;
|
||||
}): DbtDescriptionUpdates {
|
||||
const dbt: KloDescriptionUpdate[] = [];
|
||||
const aiInvalidations: KloDescriptionUpdate[] = [];
|
||||
const dbt: KtxDescriptionUpdate[] = [];
|
||||
const aiInvalidations: KtxDescriptionUpdate[] = [];
|
||||
|
||||
for (const dbtTable of input.parseResult.tables) {
|
||||
const hostTable = findMatchingKloTable(dbtTable, input.hostTables, input.targetSchema);
|
||||
const hostTable = findMatchingKtxTable(dbtTable, input.hostTables, input.targetSchema);
|
||||
if (!hostTable) {
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import type { KloMetadataUpdate } from '../../../scan/enrichment-types.js';
|
||||
import { findMatchingKloTable, type DbtHostTableLite } from './match-tables.js';
|
||||
import type { KtxMetadataUpdate } from '../../../scan/enrichment-types.js';
|
||||
import { findMatchingKtxTable, type DbtHostTableLite } from './match-tables.js';
|
||||
import type { DbtSchemaParseResult } from './parse-schema.js';
|
||||
|
||||
export function toMetadataUpdates(input: {
|
||||
|
|
@ -7,11 +7,11 @@ export function toMetadataUpdates(input: {
|
|||
parseResult: DbtSchemaParseResult;
|
||||
hostTables: DbtHostTableLite[];
|
||||
targetSchema: string | null;
|
||||
}): KloMetadataUpdate[] {
|
||||
const updates: KloMetadataUpdate[] = [];
|
||||
}): KtxMetadataUpdate[] {
|
||||
const updates: KtxMetadataUpdate[] = [];
|
||||
|
||||
for (const dbtTable of input.parseResult.tables) {
|
||||
const hostTable = findMatchingKloTable(dbtTable, input.hostTables, input.targetSchema);
|
||||
const hostTable = findMatchingKtxTable(dbtTable, input.hostTables, input.targetSchema);
|
||||
if (!hostTable) {
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import type { KloJoinUpdate } from '../../../scan/enrichment-types.js';
|
||||
import type { KtxJoinUpdate } from '../../../scan/enrichment-types.js';
|
||||
import type { DbtHostTableLite } from './match-tables.js';
|
||||
import type { DbtSchemaParseResult } from './parse-schema.js';
|
||||
|
||||
export interface DbtRelationshipUpdates {
|
||||
joins: KloJoinUpdate[];
|
||||
joins: KtxJoinUpdate[];
|
||||
skippedNoMatch: number;
|
||||
}
|
||||
|
||||
|
|
@ -19,7 +19,7 @@ export function toRelationshipUpdates(input: {
|
|||
tablesByName.set(table.name.toLowerCase(), table);
|
||||
}
|
||||
|
||||
const joins: KloJoinUpdate[] = [];
|
||||
const joins: KtxJoinUpdate[] = [];
|
||||
let skippedNoMatch = 0;
|
||||
|
||||
for (const relationship of input.parseResult.relationships) {
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ export class DbtSourceAdapter implements SourceAdapter {
|
|||
}
|
||||
await fetchDbtRepo({
|
||||
config,
|
||||
cacheDir: join(this.options.homeDir ?? '.klo/cache', 'dbt', ctx.connectionId),
|
||||
cacheDir: join(this.options.homeDir ?? '.ktx/cache', 'dbt', ctx.connectionId),
|
||||
stagedDir,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ describe('fetchDbtRepo', () => {
|
|||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-dbt-fetch-'));
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-dbt-fetch-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import {
|
|||
HistoricSqlVersionUnsupportedError,
|
||||
} from './errors.js';
|
||||
import type {
|
||||
KloPostgresQueryClient,
|
||||
KtxPostgresQueryClient,
|
||||
PostgresPgssProbeResult,
|
||||
PostgresPgssReader,
|
||||
PostgresPgssRow,
|
||||
|
|
@ -58,19 +58,19 @@ const POSTGRES_EXTENSION_REMEDIATION = [
|
|||
|
||||
const POSTGRES_GRANTS_REMEDIATION = 'GRANT pg_read_all_stats TO <connection role>;';
|
||||
|
||||
function queryClient(client: unknown): KloPostgresQueryClient {
|
||||
function queryClient(client: unknown): KtxPostgresQueryClient {
|
||||
if (
|
||||
client &&
|
||||
typeof client === 'object' &&
|
||||
'executeQuery' in client &&
|
||||
typeof (client as { executeQuery?: unknown }).executeQuery === 'function'
|
||||
) {
|
||||
return client as KloPostgresQueryClient;
|
||||
return client as KtxPostgresQueryClient;
|
||||
}
|
||||
throw new Error('Historic SQL Postgres PGSS reader requires a query client with executeQuery(sql, params?)');
|
||||
}
|
||||
|
||||
async function execute(client: KloPostgresQueryClient, sql: string, params?: unknown[]): Promise<QueryResultLike> {
|
||||
async function execute(client: KtxPostgresQueryClient, sql: string, params?: unknown[]): Promise<QueryResultLike> {
|
||||
const result = await client.executeQuery(sql, params);
|
||||
if ('error' in result && typeof result.error === 'string' && result.error.length > 0) {
|
||||
throw new Error(result.error);
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { dirname, join, relative } from 'node:path';
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type { SqlAnalysisPort } from '../../../sql-analysis/index.js';
|
||||
import { stagePgStatStatementsTemplates, writePgssBaselineAtomic, type PgssBaseline } from './stage-pgss.js';
|
||||
import type { HistoricSqlPullConfig, KloPostgresQueryClient, PostgresPgssReader, PostgresPgssRow } from './types.js';
|
||||
import type { HistoricSqlPullConfig, KtxPostgresQueryClient, PostgresPgssReader, PostgresPgssRow } from './types.js';
|
||||
|
||||
const FIXTURE_ROOT = join(__dirname, '__fixtures__/postgres');
|
||||
|
||||
|
|
@ -45,7 +45,7 @@ async function tempDir(prefix: string): Promise<string> {
|
|||
return mkdtemp(join(tmpdir(), prefix));
|
||||
}
|
||||
|
||||
function fakePgClient(): KloPostgresQueryClient {
|
||||
function fakePgClient(): KtxPostgresQueryClient {
|
||||
return {
|
||||
async executeQuery() {
|
||||
return { headers: [], rows: [] };
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import {
|
|||
type PgssBaseline,
|
||||
} from './stage-pgss.js';
|
||||
import { historicSqlManifestSchema, historicSqlMetadataSchema, historicSqlUsageSchema } from './types.js';
|
||||
import type { KloPostgresQueryClient, PostgresPgssReader, PostgresPgssRow } from './types.js';
|
||||
import type { KtxPostgresQueryClient, PostgresPgssReader, PostgresPgssRow } from './types.js';
|
||||
|
||||
async function tempDir(prefix: string): Promise<string> {
|
||||
return mkdtemp(join(tmpdir(), prefix));
|
||||
|
|
@ -21,7 +21,7 @@ async function readJson<T>(root: string, relPath: string): Promise<T> {
|
|||
return JSON.parse(await readFile(join(root, relPath), 'utf-8')) as T;
|
||||
}
|
||||
|
||||
function fakePgClient(): KloPostgresQueryClient {
|
||||
function fakePgClient(): KtxPostgresQueryClient {
|
||||
return {
|
||||
async executeQuery() {
|
||||
return { headers: [], rows: [] };
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import {
|
|||
type HistoricSqlMetadata,
|
||||
type HistoricSqlPullConfig,
|
||||
type HistoricSqlUsage,
|
||||
type KloPostgresQueryClient,
|
||||
type KtxPostgresQueryClient,
|
||||
type PostgresPgssAggregateRow,
|
||||
type PostgresPgssReader,
|
||||
type PostgresPgssRow,
|
||||
|
|
@ -43,7 +43,7 @@ export type PgssBaseline = z.infer<typeof pgssBaselineSchema>;
|
|||
export interface StagePgStatStatementsTemplatesInput {
|
||||
stagedDir: string;
|
||||
connectionId: string;
|
||||
queryClient: KloPostgresQueryClient;
|
||||
queryClient: KtxPostgresQueryClient;
|
||||
reader: PostgresPgssReader;
|
||||
sqlAnalysis: SqlAnalysisPort;
|
||||
pullConfig: HistoricSqlPullConfig;
|
||||
|
|
@ -95,7 +95,7 @@ function pgssTemplateId(row: Pick<PostgresPgssRow, 'dbid' | 'queryid'>): string
|
|||
}
|
||||
|
||||
export function pgssBaselinePath(rootDir: string | undefined, connectionId: string): string {
|
||||
return join(rootDir ?? join(process.cwd(), '.klo/cache/historic-sql'), connectionId, 'pgss-baseline.json');
|
||||
return join(rootDir ?? join(process.cwd(), '.ktx/cache/historic-sql'), connectionId, 'pgss-baseline.json');
|
||||
}
|
||||
|
||||
export async function readPgssBaseline(path: string): Promise<PgssBaseline | null> {
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ export interface HistoricSqlQueryHistoryReader {
|
|||
): AsyncIterable<HistoricSqlRawQueryRow>;
|
||||
}
|
||||
|
||||
export interface KloPostgresQueryClient {
|
||||
export interface KtxPostgresQueryClient {
|
||||
executeQuery(sql: string, params?: unknown[]): Promise<{ headers: string[]; rows: unknown[][]; totalRows?: number }>;
|
||||
}
|
||||
|
||||
|
|
@ -61,9 +61,9 @@ export interface PostgresPgssSnapshot {
|
|||
}
|
||||
|
||||
export interface PostgresPgssReader {
|
||||
probe(client: KloPostgresQueryClient): Promise<PostgresPgssProbeResult>;
|
||||
probe(client: KtxPostgresQueryClient): Promise<PostgresPgssProbeResult>;
|
||||
readSnapshot(
|
||||
client: KloPostgresQueryClient,
|
||||
client: KtxPostgresQueryClient,
|
||||
options: { minCalls: number; maxTemplates: number },
|
||||
): Promise<PostgresPgssSnapshot>;
|
||||
}
|
||||
|
|
@ -101,7 +101,7 @@ export interface HistoricSqlSourceAdapterDeps {
|
|||
reader: HistoricSqlQueryHistoryReader;
|
||||
queryClient: unknown;
|
||||
postgresReader?: PostgresPgssReader;
|
||||
postgresQueryClient?: KloPostgresQueryClient;
|
||||
postgresQueryClient?: KtxPostgresQueryClient;
|
||||
postgresBaselineRootDir?: string;
|
||||
now?: () => Date;
|
||||
onPullSucceeded?: (ctx: {
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@ import { mkdtemp } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import type { KloSchemaSnapshot } from '../../../scan/types.js';
|
||||
import type { KtxSchemaSnapshot } from '../../../scan/types.js';
|
||||
import { chunkLiveDatabaseStagedDir } from './chunk.js';
|
||||
import { liveDatabaseTablePath, writeLiveDatabaseSnapshot } from './stage.js';
|
||||
|
||||
function snapshot(): KloSchemaSnapshot {
|
||||
function snapshot(): KtxSchemaSnapshot {
|
||||
return {
|
||||
connectionId: 'conn-1',
|
||||
driver: 'postgres',
|
||||
|
|
@ -60,7 +60,7 @@ function snapshot(): KloSchemaSnapshot {
|
|||
|
||||
describe('chunkLiveDatabaseStagedDir', () => {
|
||||
it('emits one work unit per table on the first run', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'klo-live-db-chunk-'));
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-chunk-'));
|
||||
await writeLiveDatabaseSnapshot(dir, snapshot());
|
||||
|
||||
const result = await chunkLiveDatabaseStagedDir(dir);
|
||||
|
|
@ -75,7 +75,7 @@ describe('chunkLiveDatabaseStagedDir', () => {
|
|||
});
|
||||
|
||||
it('keeps only changed tables during incremental syncs and records table evictions', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'klo-live-db-diff-'));
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-diff-'));
|
||||
await writeLiveDatabaseSnapshot(dir, snapshot());
|
||||
const ordersPath = liveDatabaseTablePath({ catalog: null, db: 'public', name: 'orders' });
|
||||
const customersPath = liveDatabaseTablePath({ catalog: null, db: 'public', name: 'customers' });
|
||||
|
|
@ -92,7 +92,7 @@ describe('chunkLiveDatabaseStagedDir', () => {
|
|||
});
|
||||
|
||||
it('fans out all table work units when the foreign-key index changes', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'klo-live-db-fk-'));
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-fk-'));
|
||||
await writeLiveDatabaseSnapshot(dir, snapshot());
|
||||
|
||||
const result = await chunkLiveDatabaseStagedDir(dir, {
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import type { ChunkResult, DiffSet, WorkUnit } from '../../types.js';
|
||||
import type { KloSchemaTable } from '../../../scan/types.js';
|
||||
import type { KtxSchemaTable } from '../../../scan/types.js';
|
||||
import { LIVE_DATABASE_FOREIGN_KEYS_FILE, LIVE_DATABASE_META_FILE, readLiveDatabaseTableFiles } from './stage.js';
|
||||
|
||||
function unitKey(table: KloSchemaTable): string {
|
||||
function unitKey(table: KtxSchemaTable): string {
|
||||
const parts = [table.catalog, table.db, table.name]
|
||||
.filter((part): part is string => typeof part === 'string' && part.length > 0)
|
||||
.map((part) =>
|
||||
|
|
@ -15,7 +15,7 @@ function unitKey(table: KloSchemaTable): string {
|
|||
return `live-database-${parts.join('-') || 'table'}`;
|
||||
}
|
||||
|
||||
function displayName(table: KloSchemaTable): string {
|
||||
function displayName(table: KtxSchemaTable): string {
|
||||
return [table.catalog, table.db, table.name].filter(Boolean).join('.');
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,25 +2,25 @@ import { spawn } from 'node:child_process';
|
|||
import { request as httpRequest } from 'node:http';
|
||||
import { request as httpsRequest } from 'node:https';
|
||||
import { URL } from 'node:url';
|
||||
import type { KloProjectConnectionConfig } from '../../../project/config.js';
|
||||
import type { KloSchemaColumn, KloSchemaForeignKey, KloSchemaSnapshot, KloSchemaTable } from '../../../scan/types.js';
|
||||
import { inferKloDimensionType, normalizeKloNativeType } from '../../../scan/type-normalization.js';
|
||||
import type { KtxProjectConnectionConfig } from '../../../project/config.js';
|
||||
import type { KtxSchemaColumn, KtxSchemaForeignKey, KtxSchemaSnapshot, KtxSchemaTable } from '../../../scan/types.js';
|
||||
import { inferKtxDimensionType, normalizeKtxNativeType } from '../../../scan/type-normalization.js';
|
||||
import type { LiveDatabaseIntrospectionPort } from './types.js';
|
||||
|
||||
export type KloDaemonDatabaseIntrospectionCommand = 'database-introspect';
|
||||
export type KtxDaemonDatabaseIntrospectionCommand = 'database-introspect';
|
||||
|
||||
export type KloDaemonDatabaseJsonRunner = (
|
||||
subcommand: KloDaemonDatabaseIntrospectionCommand,
|
||||
export type KtxDaemonDatabaseJsonRunner = (
|
||||
subcommand: KtxDaemonDatabaseIntrospectionCommand,
|
||||
payload: Record<string, unknown>,
|
||||
) => Promise<Record<string, unknown>>;
|
||||
|
||||
export type KloDaemonDatabaseHttpJsonRunner = (
|
||||
export type KtxDaemonDatabaseHttpJsonRunner = (
|
||||
path: string,
|
||||
payload: Record<string, unknown>,
|
||||
) => Promise<Record<string, unknown>>;
|
||||
|
||||
export interface DaemonLiveDatabaseIntrospectionOptions {
|
||||
connections: Record<string, KloProjectConnectionConfig>;
|
||||
connections: Record<string, KtxProjectConnectionConfig>;
|
||||
schemas?: string[];
|
||||
statementTimeoutMs?: number;
|
||||
connectionTimeoutSeconds?: number;
|
||||
|
|
@ -29,8 +29,8 @@ export interface DaemonLiveDatabaseIntrospectionOptions {
|
|||
cwd?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
baseUrl?: string;
|
||||
runJson?: KloDaemonDatabaseJsonRunner;
|
||||
requestJson?: KloDaemonDatabaseHttpJsonRunner;
|
||||
runJson?: KtxDaemonDatabaseJsonRunner;
|
||||
requestJson?: KtxDaemonDatabaseHttpJsonRunner;
|
||||
now?: () => Date;
|
||||
}
|
||||
|
||||
|
|
@ -39,7 +39,7 @@ const DEFAULT_SCHEMAS = ['public'];
|
|||
function parseJsonObject(raw: string, subcommand: string): Record<string, unknown> {
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
throw new Error(`klo-daemon ${subcommand} returned non-object JSON`);
|
||||
throw new Error(`ktx-daemon ${subcommand} returned non-object JSON`);
|
||||
}
|
||||
return parsed as Record<string, unknown>;
|
||||
}
|
||||
|
|
@ -47,7 +47,7 @@ function parseJsonObject(raw: string, subcommand: string): Record<string, unknow
|
|||
function runProcessJson(
|
||||
options: Required<Pick<DaemonLiveDatabaseIntrospectionOptions, 'command' | 'args'>> &
|
||||
Pick<DaemonLiveDatabaseIntrospectionOptions, 'cwd' | 'env'>,
|
||||
): KloDaemonDatabaseJsonRunner {
|
||||
): KtxDaemonDatabaseJsonRunner {
|
||||
return async (subcommand, payload) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const child = spawn(options.command, [...options.args, subcommand], {
|
||||
|
|
@ -65,7 +65,7 @@ function runProcessJson(
|
|||
const stdoutText = Buffer.concat(stdout).toString('utf8').trim();
|
||||
const stderrText = Buffer.concat(stderr).toString('utf8').trim();
|
||||
if (code !== 0) {
|
||||
reject(new Error(`klo-daemon ${subcommand} failed: ${stderrText || `exit code ${code}`}`));
|
||||
reject(new Error(`ktx-daemon ${subcommand} failed: ${stderrText || `exit code ${code}`}`));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
|
|
@ -82,7 +82,7 @@ function normalizedBaseUrl(baseUrl: string): string {
|
|||
return baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`;
|
||||
}
|
||||
|
||||
function postJson(baseUrl: string): KloDaemonDatabaseHttpJsonRunner {
|
||||
function postJson(baseUrl: string): KtxDaemonDatabaseHttpJsonRunner {
|
||||
return async (path, payload) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const target = new URL(path.replace(/^\//, ''), normalizedBaseUrl(baseUrl));
|
||||
|
|
@ -105,7 +105,7 @@ function postJson(baseUrl: string): KloDaemonDatabaseHttpJsonRunner {
|
|||
const text = Buffer.concat(chunks).toString('utf8');
|
||||
const statusCode = response.statusCode ?? 0;
|
||||
if (statusCode < 200 || statusCode >= 300) {
|
||||
reject(new Error(`klo-daemon HTTP ${path} failed with ${statusCode}: ${text}`));
|
||||
reject(new Error(`ktx-daemon HTTP ${path} failed with ${statusCode}: ${text}`));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
|
|
@ -135,7 +135,7 @@ function recordArray(value: unknown): Array<Record<string, unknown>> {
|
|||
|
||||
function requiredString(value: unknown, field: string): string {
|
||||
if (typeof value !== 'string' || value.length === 0) {
|
||||
throw new Error(`klo-daemon database introspection response is missing string field ${field}`);
|
||||
throw new Error(`ktx-daemon database introspection response is missing string field ${field}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
|
@ -154,9 +154,9 @@ function normalizeDriver(driver: unknown): string {
|
|||
}
|
||||
|
||||
function requirePostgresConnection(
|
||||
connections: Record<string, KloProjectConnectionConfig>,
|
||||
connections: Record<string, KtxProjectConnectionConfig>,
|
||||
connectionId: string,
|
||||
): KloProjectConnectionConfig & { url: string } {
|
||||
): KtxProjectConnectionConfig & { url: string } {
|
||||
const connection = connections[connectionId];
|
||||
const driver = normalizeDriver(connection?.driver);
|
||||
if (driver !== 'postgres') {
|
||||
|
|
@ -168,23 +168,23 @@ function requirePostgresConnection(
|
|||
if (typeof connection.url !== 'string' || connection.url.trim().length === 0) {
|
||||
throw new Error(`Local live-database ingest requires connections.${connectionId}.url.`);
|
||||
}
|
||||
return connection as KloProjectConnectionConfig & { url: string };
|
||||
return connection as KtxProjectConnectionConfig & { url: string };
|
||||
}
|
||||
|
||||
function mapColumn(raw: Record<string, unknown>): KloSchemaColumn {
|
||||
function mapColumn(raw: Record<string, unknown>): KtxSchemaColumn {
|
||||
const nativeType = requiredString(raw.type, 'tables[].columns[].type');
|
||||
return {
|
||||
name: requiredString(raw.name, 'tables[].columns[].name'),
|
||||
nativeType,
|
||||
normalizedType: normalizeKloNativeType(nativeType),
|
||||
dimensionType: inferKloDimensionType(nativeType),
|
||||
normalizedType: normalizeKtxNativeType(nativeType),
|
||||
dimensionType: inferKtxDimensionType(nativeType),
|
||||
nullable: raw.nullable !== false ? true : false,
|
||||
primaryKey: raw.primary_key === true,
|
||||
comment: nullableString(raw.comment),
|
||||
};
|
||||
}
|
||||
|
||||
function mapForeignKey(raw: Record<string, unknown>): KloSchemaForeignKey {
|
||||
function mapForeignKey(raw: Record<string, unknown>): KtxSchemaForeignKey {
|
||||
return {
|
||||
fromColumn: requiredString(raw.from_column, 'tables[].foreign_keys[].from_column'),
|
||||
toCatalog: null,
|
||||
|
|
@ -195,7 +195,7 @@ function mapForeignKey(raw: Record<string, unknown>): KloSchemaForeignKey {
|
|||
};
|
||||
}
|
||||
|
||||
function mapTable(raw: Record<string, unknown>): KloSchemaTable {
|
||||
function mapTable(raw: Record<string, unknown>): KtxSchemaTable {
|
||||
return {
|
||||
catalog: nullableString(raw.catalog),
|
||||
db: nullableString(raw.db),
|
||||
|
|
@ -211,7 +211,7 @@ function mapTable(raw: Record<string, unknown>): KloSchemaTable {
|
|||
function mapDaemonSnapshot(
|
||||
raw: Record<string, unknown>,
|
||||
input: { connectionId: string; extractedAt: string; schemas: string[] },
|
||||
): KloSchemaSnapshot {
|
||||
): KtxSchemaSnapshot {
|
||||
return {
|
||||
connectionId: requiredString(raw.connection_id, 'connection_id') || input.connectionId,
|
||||
driver: 'postgres',
|
||||
|
|
@ -227,13 +227,13 @@ export function createDaemonLiveDatabaseIntrospection(
|
|||
): LiveDatabaseIntrospectionPort {
|
||||
const schemas = options.schemas ?? DEFAULT_SCHEMAS;
|
||||
const command = options.command ?? 'python';
|
||||
const args = options.args ?? ['-m', 'klo_daemon'];
|
||||
const args = options.args ?? ['-m', 'ktx_daemon'];
|
||||
const runJson = options.runJson ?? runProcessJson({ command, args, cwd: options.cwd, env: options.env });
|
||||
const requestJson = options.requestJson ?? (options.baseUrl ? postJson(options.baseUrl) : undefined);
|
||||
const now = options.now ?? (() => new Date());
|
||||
|
||||
return {
|
||||
async extractSchema(connectionId: string): Promise<KloSchemaSnapshot> {
|
||||
async extractSchema(connectionId: string): Promise<KtxSchemaSnapshot> {
|
||||
const connection = requirePostgresConnection(options.connections, connectionId);
|
||||
const payload = {
|
||||
connection_id: connectionId,
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type { KloSchemaSnapshot } from '../../../scan/types.js';
|
||||
import { buildLiveDatabaseTableNaturalKey, kloSchemaSnapshotToExtractedSchema } from './extracted-schema.js';
|
||||
import type { KtxSchemaSnapshot } from '../../../scan/types.js';
|
||||
import { buildLiveDatabaseTableNaturalKey, ktxSchemaSnapshotToExtractedSchema } from './extracted-schema.js';
|
||||
|
||||
function snapshot(): KloSchemaSnapshot {
|
||||
function snapshot(): KtxSchemaSnapshot {
|
||||
return {
|
||||
connectionId: 'conn-1',
|
||||
driver: 'postgres',
|
||||
|
|
@ -72,9 +72,9 @@ function snapshot(): KloSchemaSnapshot {
|
|||
};
|
||||
}
|
||||
|
||||
describe('kloSchemaSnapshotToExtractedSchema', () => {
|
||||
describe('ktxSchemaSnapshotToExtractedSchema', () => {
|
||||
it('preserves structural table, column, comment, and key metadata', () => {
|
||||
const extracted = kloSchemaSnapshotToExtractedSchema(snapshot());
|
||||
const extracted = ktxSchemaSnapshotToExtractedSchema(snapshot());
|
||||
|
||||
expect(extracted.tables).toEqual([
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KloSchemaSnapshot, KloSchemaTable } from '../../../scan/types.js';
|
||||
import type { KtxSchemaSnapshot, KtxSchemaTable } from '../../../scan/types.js';
|
||||
|
||||
export interface LiveDatabaseExtractedForeignKey {
|
||||
fromTable: string;
|
||||
|
|
@ -30,11 +30,11 @@ export interface LiveDatabaseExtractedSchema {
|
|||
tables: LiveDatabaseExtractedTable[];
|
||||
}
|
||||
|
||||
export function buildLiveDatabaseTableNaturalKey(table: Pick<KloSchemaTable, 'catalog' | 'db' | 'name'>): string {
|
||||
export function buildLiveDatabaseTableNaturalKey(table: Pick<KtxSchemaTable, 'catalog' | 'db' | 'name'>): string {
|
||||
return `${table.catalog ?? ''}|${table.db ?? ''}|${table.name}`;
|
||||
}
|
||||
|
||||
export function kloSchemaSnapshotToExtractedSchema(snapshot: KloSchemaSnapshot): LiveDatabaseExtractedSchema {
|
||||
export function ktxSchemaSnapshotToExtractedSchema(snapshot: KtxSchemaSnapshot): LiveDatabaseExtractedSchema {
|
||||
return {
|
||||
connectionId: snapshot.connectionId,
|
||||
tables: snapshot.tables.map((table) => ({
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ describe('LiveDatabaseSourceAdapter', () => {
|
|||
introspection: { extractSchema },
|
||||
now: () => new Date('2026-04-27T00:00:00.000Z'),
|
||||
});
|
||||
const dir = await mkdtemp(join(tmpdir(), 'klo-live-db-adapter-'));
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-adapter-'));
|
||||
|
||||
await adapter.fetch(undefined, dir, { connectionId: 'conn-1', sourceKey: 'live-database' });
|
||||
|
||||
|
|
|
|||
|
|
@ -10,9 +10,9 @@ import {
|
|||
readLiveDatabaseTableFiles,
|
||||
writeLiveDatabaseSnapshot,
|
||||
} from './stage.js';
|
||||
import type { KloSchemaSnapshot } from '../../../scan/types.js';
|
||||
import type { KtxSchemaSnapshot } from '../../../scan/types.js';
|
||||
|
||||
function snapshot(): KloSchemaSnapshot {
|
||||
function snapshot(): KtxSchemaSnapshot {
|
||||
return {
|
||||
connectionId: 'conn-1',
|
||||
driver: 'postgres',
|
||||
|
|
@ -93,7 +93,7 @@ function snapshot(): KloSchemaSnapshot {
|
|||
|
||||
describe('live-database staged snapshot files', () => {
|
||||
it('writes deterministic metadata, table, and foreign-key files', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'klo-live-db-stage-'));
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-stage-'));
|
||||
await writeLiveDatabaseSnapshot(dir, snapshot());
|
||||
|
||||
await expect(readFile(join(dir, LIVE_DATABASE_META_FILE), 'utf8')).resolves.toContain('"connectionId": "conn-1"');
|
||||
|
|
@ -122,7 +122,7 @@ describe('live-database staged snapshot files', () => {
|
|||
});
|
||||
|
||||
it('redacts sensitive snapshot metadata before writing connection metadata', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'klo-live-db-redacted-stage-'));
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-redacted-stage-'));
|
||||
await writeLiveDatabaseSnapshot(dir, {
|
||||
...snapshot(),
|
||||
metadata: {
|
||||
|
|
@ -146,7 +146,7 @@ describe('live-database staged snapshot files', () => {
|
|||
});
|
||||
|
||||
it('returns false for a directory that is missing live database metadata', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'klo-live-db-empty-'));
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-empty-'));
|
||||
expect(await detectLiveDatabaseStagedDir(dir)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ import { Buffer } from 'node:buffer';
|
|||
import type { Dirent } from 'node:fs';
|
||||
import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { join, relative } from 'node:path';
|
||||
import { redactKloSensitiveMetadata } from '../../../core/redaction.js';
|
||||
import type { KloSchemaSnapshot, KloSchemaTable, KloTableRef } from '../../../scan/types.js';
|
||||
import { redactKtxSensitiveMetadata } from '../../../core/redaction.js';
|
||||
import type { KtxSchemaSnapshot, KtxSchemaTable, KtxTableRef } from '../../../scan/types.js';
|
||||
|
||||
export const LIVE_DATABASE_META_FILE = 'connection.json';
|
||||
export const LIVE_DATABASE_FOREIGN_KEYS_FILE = 'foreign-keys.json';
|
||||
|
|
@ -11,7 +11,7 @@ const LIVE_DATABASE_TABLES_DIR = 'tables';
|
|||
|
||||
interface LiveDatabaseTableFile {
|
||||
path: string;
|
||||
table: KloSchemaTable;
|
||||
table: KtxSchemaTable;
|
||||
}
|
||||
|
||||
interface ForeignKeyIndexEntry {
|
||||
|
|
@ -29,11 +29,11 @@ function encodePathPart(value: string | null | undefined): string {
|
|||
return Buffer.from(value ?? '_', 'utf8').toString('base64url');
|
||||
}
|
||||
|
||||
function tableSortKey(table: KloTableRef): string {
|
||||
function tableSortKey(table: KtxTableRef): string {
|
||||
return `${table.catalog ?? ''}\u0000${table.db ?? ''}\u0000${table.name}`;
|
||||
}
|
||||
|
||||
export function liveDatabaseTablePath(table: KloTableRef): string {
|
||||
export function liveDatabaseTablePath(table: KtxTableRef): string {
|
||||
return `${LIVE_DATABASE_TABLES_DIR}/${encodePathPart(table.catalog)}.${encodePathPart(table.db)}.${encodePathPart(
|
||||
table.name,
|
||||
)}.json`;
|
||||
|
|
@ -62,7 +62,7 @@ function stableJson(value: unknown): string {
|
|||
return `${JSON.stringify(value, null, 2)}\n`;
|
||||
}
|
||||
|
||||
function foreignKeyIndex(snapshot: KloSchemaSnapshot): ForeignKeyIndexEntry[] {
|
||||
function foreignKeyIndex(snapshot: KtxSchemaSnapshot): ForeignKeyIndexEntry[] {
|
||||
const entries: ForeignKeyIndexEntry[] = [];
|
||||
for (const table of snapshot.tables) {
|
||||
for (const fk of table.foreignKeys) {
|
||||
|
|
@ -88,7 +88,7 @@ function foreignKeyIndex(snapshot: KloSchemaSnapshot): ForeignKeyIndexEntry[] {
|
|||
return entries;
|
||||
}
|
||||
|
||||
export async function writeLiveDatabaseSnapshot(stagedDir: string, snapshot: KloSchemaSnapshot): Promise<void> {
|
||||
export async function writeLiveDatabaseSnapshot(stagedDir: string, snapshot: KtxSchemaSnapshot): Promise<void> {
|
||||
await mkdir(join(stagedDir, LIVE_DATABASE_TABLES_DIR), { recursive: true });
|
||||
const sortedTables = [...snapshot.tables].sort((a, b) => tableSortKey(a).localeCompare(tableSortKey(b)));
|
||||
const metadata = {
|
||||
|
|
@ -96,7 +96,7 @@ export async function writeLiveDatabaseSnapshot(stagedDir: string, snapshot: Klo
|
|||
driver: snapshot.driver,
|
||||
extractedAt: snapshot.extractedAt,
|
||||
scope: snapshot.scope,
|
||||
metadata: redactKloSensitiveMetadata(snapshot.metadata),
|
||||
metadata: redactKtxSensitiveMetadata(snapshot.metadata),
|
||||
tableCount: sortedTables.length,
|
||||
};
|
||||
await writeFile(join(stagedDir, LIVE_DATABASE_META_FILE), stableJson(metadata));
|
||||
|
|
@ -115,7 +115,7 @@ export async function readLiveDatabaseTableFiles(stagedDir: string): Promise<Liv
|
|||
for (const file of files.filter((path) => path.endsWith('.json'))) {
|
||||
const path = `${LIVE_DATABASE_TABLES_DIR}/${file}`;
|
||||
const raw = await readFile(join(stagedDir, path), 'utf8');
|
||||
const parsed = JSON.parse(raw) as KloSchemaTable;
|
||||
const parsed = JSON.parse(raw) as KtxSchemaTable;
|
||||
if (parsed && typeof parsed.name === 'string' && Array.isArray(parsed.columns)) {
|
||||
out.push({ path, table: parsed });
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import type { KloSchemaSnapshot } from '../../../scan/types.js';
|
||||
import type { KtxSchemaSnapshot } from '../../../scan/types.js';
|
||||
|
||||
export interface LiveDatabaseIntrospectionPort {
|
||||
extractSchema(connectionId: string): Promise<KloSchemaSnapshot>;
|
||||
extractSchema(connectionId: string): Promise<KtxSchemaSnapshot>;
|
||||
}
|
||||
|
||||
export interface LiveDatabaseSourceAdapterDeps {
|
||||
|
|
|
|||
|
|
@ -39,6 +39,6 @@ describe('createDaemonLookerTableIdentifierParser', () => {
|
|||
requestJson: async () => ({ results: null }),
|
||||
});
|
||||
|
||||
await expect(parser.parse([])).rejects.toThrow('klo-daemon table identifier parser returned invalid results');
|
||||
await expect(parser.parse([])).rejects.toThrow('ktx-daemon table identifier parser returned invalid results');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -7,14 +7,14 @@ import type {
|
|||
LookerTableIdentifierParser,
|
||||
} from './mapping.js';
|
||||
|
||||
export type KloDaemonTableIdentifierHttpJsonRunner = (
|
||||
export type KtxDaemonTableIdentifierHttpJsonRunner = (
|
||||
path: string,
|
||||
payload: Record<string, unknown>,
|
||||
) => Promise<Record<string, unknown>>;
|
||||
|
||||
export interface DaemonLookerTableIdentifierParserOptions {
|
||||
baseUrl: string;
|
||||
requestJson?: KloDaemonTableIdentifierHttpJsonRunner;
|
||||
requestJson?: KtxDaemonTableIdentifierHttpJsonRunner;
|
||||
}
|
||||
|
||||
export function createDaemonLookerTableIdentifierParser(
|
||||
|
|
@ -25,7 +25,7 @@ export function createDaemonLookerTableIdentifierParser(
|
|||
async parse(items: LookerTableIdentifierParseItem[]): Promise<Record<string, LookerParsedIdentifier>> {
|
||||
const raw = await requestJson('/sql/parse-table-identifier', { items });
|
||||
if (!raw.results || typeof raw.results !== 'object' || Array.isArray(raw.results)) {
|
||||
throw new Error('klo-daemon table identifier parser returned invalid results');
|
||||
throw new Error('ktx-daemon table identifier parser returned invalid results');
|
||||
}
|
||||
return raw.results as Record<string, LookerParsedIdentifier>;
|
||||
},
|
||||
|
|
@ -36,7 +36,7 @@ function normalizedBaseUrl(baseUrl: string): string {
|
|||
return baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`;
|
||||
}
|
||||
|
||||
function postJson(baseUrl: string): KloDaemonTableIdentifierHttpJsonRunner {
|
||||
function postJson(baseUrl: string): KtxDaemonTableIdentifierHttpJsonRunner {
|
||||
return async (path, payload) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const target = new URL(path.replace(/^\//, ''), normalizedBaseUrl(baseUrl));
|
||||
|
|
@ -59,13 +59,13 @@ function postJson(baseUrl: string): KloDaemonTableIdentifierHttpJsonRunner {
|
|||
const text = Buffer.concat(chunks).toString('utf8');
|
||||
const statusCode = response.statusCode ?? 0;
|
||||
if (statusCode < 200 || statusCode >= 300) {
|
||||
reject(new Error(`klo-daemon HTTP ${path} failed with ${statusCode}: ${text}`));
|
||||
reject(new Error(`ktx-daemon HTTP ${path} failed with ${statusCode}: ${text}`));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(text) as unknown;
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
reject(new Error(`klo-daemon HTTP ${path} returned non-object JSON`));
|
||||
reject(new Error(`ktx-daemon HTTP ${path} returned non-object JSON`));
|
||||
return;
|
||||
}
|
||||
resolve(parsed as Record<string, unknown>);
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ function sdk(): LookerSdkPort {
|
|||
}
|
||||
|
||||
describe('DefaultLookerConnectionClientFactory', () => {
|
||||
it('resolves credentials by Looker connection id and creates a KLO Looker client', async () => {
|
||||
it('resolves credentials by Looker connection id and creates a KTX Looker client', async () => {
|
||||
const fakeSdk = sdk();
|
||||
const resolver: LookerCredentialResolver = {
|
||||
resolve: vi.fn().mockResolvedValue({
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { KloLocalProject, KloProjectConnectionConfig } from '../../../project/index.js';
|
||||
import type { KtxLocalProject, KtxProjectConnectionConfig } from '../../../project/index.js';
|
||||
import {
|
||||
DefaultLookerClientFactory,
|
||||
DefaultLookerConnectionClientFactory,
|
||||
|
|
@ -19,7 +19,7 @@ function resolveEnvReference(ref: string, env: NodeJS.ProcessEnv): string | null
|
|||
|
||||
export function lookerCredentialsFromLocalConnection(
|
||||
connectionId: string,
|
||||
connection: KloProjectConnectionConfig | undefined,
|
||||
connection: KtxProjectConnectionConfig | undefined,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
) {
|
||||
if (!connection || String(connection.driver).toLowerCase() !== 'looker') {
|
||||
|
|
@ -46,7 +46,7 @@ export function lookerCredentialsFromLocalConnection(
|
|||
}
|
||||
|
||||
export function createLocalLookerCredentialResolver(
|
||||
project: KloLocalProject,
|
||||
project: KtxLocalProject,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): LookerCredentialResolver {
|
||||
return {
|
||||
|
|
@ -57,7 +57,7 @@ export function createLocalLookerCredentialResolver(
|
|||
}
|
||||
|
||||
export function createLocalLookerSourceAdapter(
|
||||
project: KloLocalProject,
|
||||
project: KtxLocalProject,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): LookerSourceAdapter {
|
||||
const connectionFactory = new DefaultLookerConnectionClientFactory(createLocalLookerCredentialResolver(project, env));
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import { LocalLookerRuntimeStore } from './local-runtime-store.js';
|
|||
|
||||
describe('LocalLookerRuntimeStore', () => {
|
||||
async function store() {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'klo-looker-store-'));
|
||||
const dir = await mkdtemp(join(tmpdir(), 'ktx-looker-store-'));
|
||||
return new LocalLookerRuntimeStore({
|
||||
dbPath: join(dir, 'db.sqlite'),
|
||||
now: () => new Date('2026-05-05T12:00:00.000Z'),
|
||||
|
|
@ -23,7 +23,7 @@ describe('LocalLookerRuntimeStore', () => {
|
|||
await local.upsertConnectionMapping({
|
||||
lookerConnectionId: 'prod-looker',
|
||||
lookerConnectionName: 'bq_reporting',
|
||||
kloConnectionId: 'prod-warehouse',
|
||||
ktxConnectionId: 'prod-warehouse',
|
||||
source: 'cli',
|
||||
});
|
||||
|
||||
|
|
@ -34,7 +34,7 @@ describe('LocalLookerRuntimeStore', () => {
|
|||
await expect(local.readMappings('prod-looker')).resolves.toEqual([
|
||||
{
|
||||
lookerConnectionName: 'bq_reporting',
|
||||
kloConnectionId: 'prod-warehouse',
|
||||
ktxConnectionId: 'prod-warehouse',
|
||||
lookerHost: null,
|
||||
lookerDatabase: null,
|
||||
lookerDialect: null,
|
||||
|
|
@ -47,7 +47,7 @@ describe('LocalLookerRuntimeStore', () => {
|
|||
await local.upsertConnectionMapping({
|
||||
lookerConnectionId: 'prod-looker',
|
||||
lookerConnectionName: 'bq_reporting',
|
||||
kloConnectionId: 'prod-warehouse',
|
||||
ktxConnectionId: 'prod-warehouse',
|
||||
source: 'cli',
|
||||
});
|
||||
|
||||
|
|
@ -67,7 +67,7 @@ describe('LocalLookerRuntimeStore', () => {
|
|||
await expect(local.listConnectionMappings('prod-looker')).resolves.toEqual([
|
||||
{
|
||||
lookerConnectionName: 'bq_reporting',
|
||||
kloConnectionId: 'prod-warehouse',
|
||||
ktxConnectionId: 'prod-warehouse',
|
||||
lookerHost: 'bigquery.googleapis.com',
|
||||
lookerDatabase: 'analytics',
|
||||
lookerDialect: 'bigquery_standard_sql',
|
||||
|
|
@ -85,30 +85,30 @@ describe('LocalLookerRuntimeStore', () => {
|
|||
await local.upsertConnectionMapping({
|
||||
lookerConnectionId: 'prod-looker',
|
||||
lookerConnectionName: 'manual',
|
||||
kloConnectionId: 'cli-warehouse',
|
||||
ktxConnectionId: 'cli-warehouse',
|
||||
source: 'cli',
|
||||
});
|
||||
|
||||
await local.applyYamlBootstrap({
|
||||
lookerConnectionId: 'prod-looker',
|
||||
mappings: [
|
||||
{ lookerConnectionName: 'analytics', kloConnectionId: 'yaml-warehouse' },
|
||||
{ lookerConnectionName: 'manual', kloConnectionId: 'yaml-warehouse' },
|
||||
{ lookerConnectionName: 'analytics', ktxConnectionId: 'yaml-warehouse' },
|
||||
{ lookerConnectionName: 'manual', ktxConnectionId: 'yaml-warehouse' },
|
||||
],
|
||||
});
|
||||
|
||||
await expect(local.listConnectionMappings('prod-looker')).resolves.toMatchObject([
|
||||
{
|
||||
lookerConnectionName: 'analytics',
|
||||
kloConnectionId: 'yaml-warehouse',
|
||||
ktxConnectionId: 'yaml-warehouse',
|
||||
lookerHost: 'looker-db.test',
|
||||
lookerDatabase: 'warehouse',
|
||||
lookerDialect: 'postgres',
|
||||
source: 'klo.yaml',
|
||||
source: 'ktx.yaml',
|
||||
},
|
||||
{
|
||||
lookerConnectionName: 'manual',
|
||||
kloConnectionId: 'cli-warehouse',
|
||||
ktxConnectionId: 'cli-warehouse',
|
||||
source: 'cli',
|
||||
},
|
||||
]);
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import type { LookerWarehouseConnectionInfo } from './client.js';
|
|||
import type { LookerConnectionMapping } from './mapping.js';
|
||||
import type { LookerRuntimeCursors } from './types.js';
|
||||
|
||||
export type LocalLookerMappingSource = 'klo.yaml' | 'cli' | 'refresh';
|
||||
export type LocalLookerMappingSource = 'ktx.yaml' | 'cli' | 'refresh';
|
||||
|
||||
interface LocalLookerRuntimeStoreOptions {
|
||||
dbPath: string;
|
||||
|
|
@ -19,7 +19,7 @@ export interface LocalLookerConnectionMappingListRow extends LookerConnectionMap
|
|||
export interface UpsertLocalLookerConnectionMappingInput {
|
||||
lookerConnectionId: string;
|
||||
lookerConnectionName: string;
|
||||
kloConnectionId: string | null;
|
||||
ktxConnectionId: string | null;
|
||||
source: LocalLookerMappingSource;
|
||||
}
|
||||
|
||||
|
|
@ -27,7 +27,7 @@ interface ApplyLocalLookerYamlBootstrapInput {
|
|||
lookerConnectionId: string;
|
||||
mappings: Array<{
|
||||
lookerConnectionName: string;
|
||||
kloConnectionId: string | null;
|
||||
ktxConnectionId: string | null;
|
||||
}>;
|
||||
}
|
||||
|
||||
|
|
@ -67,7 +67,7 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
CREATE TABLE IF NOT EXISTS local_looker_connection_mappings (
|
||||
looker_connection_id TEXT NOT NULL,
|
||||
looker_connection_name TEXT NOT NULL,
|
||||
klo_connection_id TEXT,
|
||||
ktx_connection_id TEXT,
|
||||
looker_host TEXT,
|
||||
looker_database TEXT,
|
||||
looker_dialect TEXT,
|
||||
|
|
@ -82,7 +82,7 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
const timestamp = this.now().toISOString();
|
||||
const apply = this.db.transaction(() => {
|
||||
const existing = this.db.prepare(`
|
||||
SELECT klo_connection_id, source
|
||||
SELECT ktx_connection_id, source
|
||||
FROM local_looker_connection_mappings
|
||||
WHERE looker_connection_id = ? AND looker_connection_name = ?
|
||||
`);
|
||||
|
|
@ -90,36 +90,36 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
INSERT INTO local_looker_connection_mappings (
|
||||
looker_connection_id,
|
||||
looker_connection_name,
|
||||
klo_connection_id,
|
||||
ktx_connection_id,
|
||||
looker_host,
|
||||
looker_database,
|
||||
looker_dialect,
|
||||
source,
|
||||
updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, NULL, NULL, NULL, 'klo.yaml', ?)
|
||||
VALUES (?, ?, ?, NULL, NULL, NULL, 'ktx.yaml', ?)
|
||||
`);
|
||||
const updateRefreshRow = this.db.prepare(`
|
||||
UPDATE local_looker_connection_mappings
|
||||
SET klo_connection_id = ?,
|
||||
source = 'klo.yaml',
|
||||
SET ktx_connection_id = ?,
|
||||
source = 'ktx.yaml',
|
||||
updated_at = ?
|
||||
WHERE looker_connection_id = ?
|
||||
AND looker_connection_name = ?
|
||||
AND source = 'refresh'
|
||||
AND klo_connection_id IS NULL
|
||||
AND ktx_connection_id IS NULL
|
||||
`);
|
||||
|
||||
for (const mapping of input.mappings) {
|
||||
const row = existing.get(input.lookerConnectionId, mapping.lookerConnectionName) as
|
||||
| { klo_connection_id: string | null; source: LocalLookerMappingSource }
|
||||
| { ktx_connection_id: string | null; source: LocalLookerMappingSource }
|
||||
| undefined;
|
||||
if (!row) {
|
||||
insert.run(input.lookerConnectionId, mapping.lookerConnectionName, mapping.kloConnectionId, timestamp);
|
||||
insert.run(input.lookerConnectionId, mapping.lookerConnectionName, mapping.ktxConnectionId, timestamp);
|
||||
continue;
|
||||
}
|
||||
if (row.source === 'refresh' && row.klo_connection_id === null) {
|
||||
updateRefreshRow.run(mapping.kloConnectionId, timestamp, input.lookerConnectionId, mapping.lookerConnectionName);
|
||||
if (row.source === 'refresh' && row.ktx_connection_id === null) {
|
||||
updateRefreshRow.run(mapping.ktxConnectionId, timestamp, input.lookerConnectionId, mapping.lookerConnectionName);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
@ -174,7 +174,7 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
`
|
||||
SELECT
|
||||
looker_connection_name,
|
||||
klo_connection_id,
|
||||
ktx_connection_id,
|
||||
looker_host,
|
||||
looker_database,
|
||||
looker_dialect,
|
||||
|
|
@ -186,7 +186,7 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
)
|
||||
.all(lookerConnectionId) as Array<{
|
||||
looker_connection_name: string;
|
||||
klo_connection_id: string | null;
|
||||
ktx_connection_id: string | null;
|
||||
looker_host: string | null;
|
||||
looker_database: string | null;
|
||||
looker_dialect: string | null;
|
||||
|
|
@ -195,7 +195,7 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
|
||||
return rows.map((row) => ({
|
||||
lookerConnectionName: row.looker_connection_name,
|
||||
kloConnectionId: row.klo_connection_id,
|
||||
ktxConnectionId: row.ktx_connection_id,
|
||||
lookerHost: row.looker_host,
|
||||
lookerDatabase: row.looker_database,
|
||||
lookerDialect: row.looker_dialect,
|
||||
|
|
@ -210,7 +210,7 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
INSERT INTO local_looker_connection_mappings (
|
||||
looker_connection_id,
|
||||
looker_connection_name,
|
||||
klo_connection_id,
|
||||
ktx_connection_id,
|
||||
looker_host,
|
||||
looker_database,
|
||||
looker_dialect,
|
||||
|
|
@ -219,12 +219,12 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
)
|
||||
VALUES (?, ?, ?, NULL, NULL, NULL, ?, ?)
|
||||
ON CONFLICT(looker_connection_id, looker_connection_name) DO UPDATE SET
|
||||
klo_connection_id = excluded.klo_connection_id,
|
||||
ktx_connection_id = excluded.ktx_connection_id,
|
||||
source = excluded.source,
|
||||
updated_at = excluded.updated_at
|
||||
`,
|
||||
)
|
||||
.run(input.lookerConnectionId, input.lookerConnectionName, input.kloConnectionId, input.source, this.now().toISOString());
|
||||
.run(input.lookerConnectionId, input.lookerConnectionName, input.ktxConnectionId, input.source, this.now().toISOString());
|
||||
}
|
||||
|
||||
async refreshDiscoveredConnections(input: RefreshLocalLookerDiscoveredConnectionsInput): Promise<void> {
|
||||
|
|
@ -234,7 +234,7 @@ export class LocalLookerRuntimeStore implements LookerSourceStateReader {
|
|||
INSERT INTO local_looker_connection_mappings (
|
||||
looker_connection_id,
|
||||
looker_connection_name,
|
||||
klo_connection_id,
|
||||
ktx_connection_id,
|
||||
looker_host,
|
||||
looker_database,
|
||||
looker_dialect,
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import {
|
|||
projectParsedIdentifier,
|
||||
refreshLookerMappingPlaceholders,
|
||||
sqlglotDialectForConnectionType,
|
||||
suggestKloConnectionForLookerConnection,
|
||||
suggestKtxConnectionForLookerConnection,
|
||||
validateLookerMappings,
|
||||
validateLookerWarehouseTarget,
|
||||
} from './mapping.js';
|
||||
|
|
@ -69,7 +69,7 @@ describe('discoverLookerConnections', () => {
|
|||
});
|
||||
|
||||
describe('looker dialect and target validation helpers', () => {
|
||||
it('maps Looker dialect names to KLO connection types', () => {
|
||||
it('maps Looker dialect names to KTX connection types', () => {
|
||||
expect(lookerDialectToConnectionType('bigquery_standard_sql')).toBe('BIGQUERY');
|
||||
expect(lookerDialectToConnectionType('postgres')).toBe('POSTGRESQL');
|
||||
expect(lookerDialectToConnectionType('mssql')).toBe('SQLSERVER');
|
||||
|
|
@ -90,10 +90,10 @@ describe('looker dialect and target validation helpers', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('suggestKloConnectionForLookerConnection', () => {
|
||||
describe('suggestKtxConnectionForLookerConnection', () => {
|
||||
it('returns the single deterministic target with matching type, host, and database', () => {
|
||||
expect(
|
||||
suggestKloConnectionForLookerConnection({
|
||||
suggestKtxConnectionForLookerConnection({
|
||||
lookerConnection: liveConnections[1],
|
||||
candidateConnections: [
|
||||
{
|
||||
|
|
@ -113,7 +113,7 @@ describe('suggestKloConnectionForLookerConnection', () => {
|
|||
|
||||
it('returns null when more than one target matches', () => {
|
||||
expect(
|
||||
suggestKloConnectionForLookerConnection({
|
||||
suggestKtxConnectionForLookerConnection({
|
||||
lookerConnection: liveConnections[1],
|
||||
candidateConnections: [
|
||||
{
|
||||
|
|
@ -139,7 +139,7 @@ describe('refreshLookerMappingPlaceholders', () => {
|
|||
stored: [
|
||||
{
|
||||
lookerConnectionName: 'b2b_sandbox_bq',
|
||||
kloConnectionId: 'warehouse',
|
||||
ktxConnectionId: 'warehouse',
|
||||
lookerHost: null,
|
||||
lookerDatabase: null,
|
||||
lookerDialect: null,
|
||||
|
|
@ -152,14 +152,14 @@ describe('refreshLookerMappingPlaceholders', () => {
|
|||
mappings: [
|
||||
{
|
||||
lookerConnectionName: 'b2b_sandbox_bq',
|
||||
kloConnectionId: 'warehouse',
|
||||
ktxConnectionId: 'warehouse',
|
||||
lookerHost: 'warehouse.example.com',
|
||||
lookerDatabase: 'analytics',
|
||||
lookerDialect: 'bigquery_standard_sql',
|
||||
},
|
||||
{
|
||||
lookerConnectionName: 'pg_runtime',
|
||||
kloConnectionId: null,
|
||||
ktxConnectionId: null,
|
||||
lookerHost: 'pg.internal:5432',
|
||||
lookerDatabase: 'app',
|
||||
lookerDialect: 'postgres',
|
||||
|
|
@ -176,14 +176,14 @@ describe('computeLookerMappingDrift and validateLookerMappings', () => {
|
|||
storedMappings: [
|
||||
{
|
||||
lookerConnectionName: 'b2b_sandbox_bq',
|
||||
kloConnectionId: 'warehouse',
|
||||
ktxConnectionId: 'warehouse',
|
||||
lookerHost: null,
|
||||
lookerDatabase: null,
|
||||
lookerDialect: null,
|
||||
},
|
||||
{
|
||||
lookerConnectionName: 'stale_runtime',
|
||||
kloConnectionId: 'warehouse',
|
||||
ktxConnectionId: 'warehouse',
|
||||
lookerHost: null,
|
||||
lookerDatabase: null,
|
||||
lookerDialect: null,
|
||||
|
|
@ -194,7 +194,7 @@ describe('computeLookerMappingDrift and validateLookerMappings', () => {
|
|||
).toEqual({
|
||||
unmappedDiscovered: [liveConnections[1]],
|
||||
staleMappings: [{ lookerConnectionName: 'stale_runtime', reason: 'looker_connection_not_found' }],
|
||||
inSync: [{ lookerConnectionName: 'b2b_sandbox_bq', kloConnectionId: 'warehouse' }],
|
||||
inSync: [{ lookerConnectionName: 'b2b_sandbox_bq', ktxConnectionId: 'warehouse' }],
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -204,26 +204,26 @@ describe('computeLookerMappingDrift and validateLookerMappings', () => {
|
|||
mappings: [
|
||||
{
|
||||
lookerConnectionName: 'b2b_sandbox_bq',
|
||||
kloConnectionId: 'missing',
|
||||
ktxConnectionId: 'missing',
|
||||
lookerHost: null,
|
||||
lookerDatabase: null,
|
||||
lookerDialect: null,
|
||||
},
|
||||
{
|
||||
lookerConnectionName: 'pg_runtime',
|
||||
kloConnectionId: 'looker-target',
|
||||
ktxConnectionId: 'looker-target',
|
||||
lookerHost: null,
|
||||
lookerDatabase: null,
|
||||
lookerDialect: null,
|
||||
},
|
||||
],
|
||||
knownKloConnectionIds: new Set(['looker-target']),
|
||||
knownKtxConnectionIds: new Set(['looker-target']),
|
||||
knownConnectionTypes: new Map([['looker-target', 'LOOKER']]),
|
||||
}),
|
||||
).toEqual({
|
||||
ok: false,
|
||||
errors: [
|
||||
{ key: 'b2b_sandbox_bq', reason: 'KLO connection missing does not exist' },
|
||||
{ key: 'b2b_sandbox_bq', reason: 'KTX connection missing does not exist' },
|
||||
{
|
||||
key: 'pg_runtime',
|
||||
reason: 'Connection type LOOKER cannot be used as a Looker warehouse mapping target',
|
||||
|
|
@ -258,7 +258,7 @@ describe('collectExploreParseItems and projectParsedIdentifier', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('projects successful and failed parser rows into KLO parsed target tables', () => {
|
||||
it('projects successful and failed parser rows into KTX parsed target tables', () => {
|
||||
expect(
|
||||
projectParsedIdentifier({
|
||||
ok: true,
|
||||
|
|
@ -317,7 +317,7 @@ describe('buildLookerPullConfigFromInputs', () => {
|
|||
refreshedMappings: [
|
||||
{
|
||||
lookerConnectionName: 'b2b_sandbox_bq',
|
||||
kloConnectionId: 'warehouse',
|
||||
ktxConnectionId: 'warehouse',
|
||||
lookerHost: 'warehouse.example.com',
|
||||
lookerDatabase: 'analytics',
|
||||
lookerDialect: 'bigquery_standard_sql',
|
||||
|
|
@ -365,7 +365,7 @@ describe('buildLookerPullConfigFromInputs', () => {
|
|||
refreshedMappings: [
|
||||
{
|
||||
lookerConnectionName: 'b2b_sandbox_bq',
|
||||
kloConnectionId: 'warehouse',
|
||||
ktxConnectionId: 'warehouse',
|
||||
lookerHost: null,
|
||||
lookerDatabase: null,
|
||||
lookerDialect: null,
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ export type LookerWarehouseTargetConnectionType =
|
|||
|
||||
export interface LookerConnectionMapping {
|
||||
lookerConnectionName: string;
|
||||
kloConnectionId: string | null;
|
||||
ktxConnectionId: string | null;
|
||||
lookerHost: string | null;
|
||||
lookerDatabase: string | null;
|
||||
lookerDialect: string | null;
|
||||
|
|
@ -43,7 +43,7 @@ export interface LookerMappingCandidateConnection extends LookerTargetConnection
|
|||
export interface LookerMappingDrift {
|
||||
unmappedDiscovered: LookerWarehouseConnectionInfo[];
|
||||
staleMappings: Array<{ lookerConnectionName: string; reason: 'looker_connection_not_found' }>;
|
||||
inSync: Array<{ lookerConnectionName: string; kloConnectionId: string }>;
|
||||
inSync: Array<{ lookerConnectionName: string; ktxConnectionId: string }>;
|
||||
}
|
||||
|
||||
export type LookerMappingValidationResult =
|
||||
|
|
@ -155,7 +155,7 @@ export function normalizeName(value: string | null): string | null {
|
|||
return value ? value.toLowerCase() : null;
|
||||
}
|
||||
|
||||
export function suggestKloConnectionForLookerConnection(args: {
|
||||
export function suggestKtxConnectionForLookerConnection(args: {
|
||||
lookerConnection: LookerWarehouseConnectionInfo;
|
||||
candidateConnections: LookerMappingCandidateConnection[];
|
||||
}): string | null {
|
||||
|
|
@ -187,7 +187,7 @@ export function computeLookerMappingDrift(args: {
|
|||
const storedByName = new Map(args.storedMappings.map((mapping) => [mapping.lookerConnectionName, mapping]));
|
||||
|
||||
return {
|
||||
unmappedDiscovered: args.discovered.filter((connection) => !storedByName.get(connection.name)?.kloConnectionId),
|
||||
unmappedDiscovered: args.discovered.filter((connection) => !storedByName.get(connection.name)?.ktxConnectionId),
|
||||
staleMappings: args.storedMappings
|
||||
.filter((mapping) => !discoveredByName.has(mapping.lookerConnectionName))
|
||||
.map((mapping) => ({
|
||||
|
|
@ -195,32 +195,32 @@ export function computeLookerMappingDrift(args: {
|
|||
reason: 'looker_connection_not_found' as const,
|
||||
})),
|
||||
inSync: args.storedMappings
|
||||
.filter((mapping) => discoveredByName.has(mapping.lookerConnectionName) && mapping.kloConnectionId)
|
||||
.filter((mapping) => discoveredByName.has(mapping.lookerConnectionName) && mapping.ktxConnectionId)
|
||||
.map((mapping) => ({
|
||||
lookerConnectionName: mapping.lookerConnectionName,
|
||||
kloConnectionId: mapping.kloConnectionId as string,
|
||||
ktxConnectionId: mapping.ktxConnectionId as string,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
export function validateLookerMappings(args: {
|
||||
mappings: LookerConnectionMapping[];
|
||||
knownKloConnectionIds: Set<string>;
|
||||
knownKtxConnectionIds: Set<string>;
|
||||
knownConnectionTypes: ReadonlyMap<string, string>;
|
||||
}): LookerMappingValidationResult {
|
||||
const errors: Array<{ key: string; reason: string }> = [];
|
||||
for (const mapping of args.mappings) {
|
||||
if (!mapping.kloConnectionId) {
|
||||
if (!mapping.ktxConnectionId) {
|
||||
continue;
|
||||
}
|
||||
if (!args.knownKloConnectionIds.has(mapping.kloConnectionId)) {
|
||||
if (!args.knownKtxConnectionIds.has(mapping.ktxConnectionId)) {
|
||||
errors.push({
|
||||
key: mapping.lookerConnectionName,
|
||||
reason: `KLO connection ${mapping.kloConnectionId} does not exist`,
|
||||
reason: `KTX connection ${mapping.ktxConnectionId} does not exist`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const connectionType = args.knownConnectionTypes.get(mapping.kloConnectionId);
|
||||
const connectionType = args.knownConnectionTypes.get(mapping.ktxConnectionId);
|
||||
const validation = validateLookerWarehouseTarget(connectionType ?? 'unknown');
|
||||
if (!validation.ok) {
|
||||
errors.push({ key: mapping.lookerConnectionName, reason: validation.reason });
|
||||
|
|
@ -241,7 +241,7 @@ export function refreshLookerMappingPlaceholders(args: {
|
|||
if (!existing) {
|
||||
byName.set(live.name, {
|
||||
lookerConnectionName: live.name,
|
||||
kloConnectionId: null,
|
||||
ktxConnectionId: null,
|
||||
lookerHost: live.host,
|
||||
lookerDatabase: live.database,
|
||||
lookerDialect: live.dialect,
|
||||
|
|
@ -346,14 +346,14 @@ export async function buildLookerPullConfigFromInputs(args: {
|
|||
const connectionTypes: Record<string, LookerWarehouseTargetConnectionType> = {};
|
||||
|
||||
for (const mapping of args.refreshedMappings) {
|
||||
if (!mapping.kloConnectionId) {
|
||||
if (!mapping.ktxConnectionId) {
|
||||
continue;
|
||||
}
|
||||
const target = args.targetConnections.get(mapping.kloConnectionId);
|
||||
const target = args.targetConnections.get(mapping.ktxConnectionId);
|
||||
if (!target || !validateLookerWarehouseTarget(target.connection_type).ok) {
|
||||
continue;
|
||||
}
|
||||
connectionMappings[mapping.lookerConnectionName] = mapping.kloConnectionId;
|
||||
connectionMappings[mapping.lookerConnectionName] = mapping.ktxConnectionId;
|
||||
connectionTypes[mapping.lookerConnectionName] = target.connection_type as LookerWarehouseTargetConnectionType;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -255,7 +255,7 @@ describe('Looker staged runtime schemas', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('accepts slug-shaped connection ids inside KLO Looker runtime schemas', () => {
|
||||
it('accepts slug-shaped connection ids inside KTX Looker runtime schemas', () => {
|
||||
const parsedTargetTable = {
|
||||
ok: true as const,
|
||||
catalog: 'proj',
|
||||
|
|
@ -313,7 +313,7 @@ describe('Looker staged runtime schemas', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('rejects unsafe KLO Looker connection ids', () => {
|
||||
it('rejects unsafe KTX Looker connection ids', () => {
|
||||
expect(() =>
|
||||
parseLookerPullConfig({
|
||||
lookerConnectionId: '../prod-looker',
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ async function readMetabaseFile(name: string): Promise<string> {
|
|||
return readFile(join(metabaseDir, name), 'utf-8');
|
||||
}
|
||||
|
||||
describe('KLO Metabase client boundary', () => {
|
||||
it('keeps NestJS, server data-source base classes, and server-relative imports out of the KLO client', async () => {
|
||||
describe('KTX Metabase client boundary', () => {
|
||||
it('keeps NestJS, server data-source base classes, and server-relative imports out of the KTX client', async () => {
|
||||
const client = await readMetabaseFile('client.ts');
|
||||
expect(client).not.toContain(`@${'nestjs'}`);
|
||||
expect(client).not.toContain(`DataSource${'Client'}`);
|
||||
|
|
@ -19,7 +19,7 @@ describe('KLO Metabase client boundary', () => {
|
|||
expect(client).not.toContain('../../types/brand');
|
||||
});
|
||||
|
||||
it('keeps proxy implementation code out of the KLO v1 client', async () => {
|
||||
it('keeps proxy implementation code out of the KTX v1 client', async () => {
|
||||
const client = await readMetabaseFile('client.ts');
|
||||
expect(client).not.toContain(`network-${'proxy'}`);
|
||||
expect(client).not.toContain(`ssh${'2'}`);
|
||||
|
|
|
|||
|
|
@ -199,7 +199,7 @@ describe('MetabaseClient admin auth helpers', () => {
|
|||
);
|
||||
|
||||
await expect(client.getPermissionGroups()).resolves.toEqual([{ id: 2, name: 'Administrators' }]);
|
||||
await expect(client.createApiKey({ name: 'KLO CLI test', groupId: 2 })).resolves.toBe(mintedMetabaseCredential);
|
||||
await expect(client.createApiKey({ name: 'KTX CLI test', groupId: 2 })).resolves.toBe(mintedMetabaseCredential);
|
||||
|
||||
expect(fetchMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
|
|
@ -214,7 +214,7 @@ describe('MetabaseClient admin auth helpers', () => {
|
|||
'https://metabase.example.test/api/api-key',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ name: 'KLO CLI test', group_id: 2 }),
|
||||
body: JSON.stringify({ name: 'KTX CLI test', group_id: 2 }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
|
@ -343,7 +343,7 @@ describe('MetabaseClient.getResolvedSql', () => {
|
|||
expect(result?.resolutionStatus).toBe('resolved');
|
||||
const sql = result?.resolvedSql ?? '';
|
||||
expect(sql.startsWith('--')).toBe(true);
|
||||
expect(sql).toMatch(/KLO_PLACEHOLDER_WARNING/);
|
||||
expect(sql).toMatch(/KTX_PLACEHOLDER_WARNING/);
|
||||
expect(sql).toMatch(/\bid\b/);
|
||||
expect(sql).toMatch(/\bn\b/);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ class MetabaseApiError extends Error {
|
|||
* Strip Metabase `[[ ... {{ var }} ... ]]` optional-clause blocks from native SQL.
|
||||
*
|
||||
* The bracketed blocks are emitted only when the embedded `{{ var }}` is supplied at
|
||||
* Metabase query time. For KLO semantic-layer ingest there's no such runtime
|
||||
* Metabase query time. For KTX semantic-layer ingest there's no such runtime
|
||||
* parameter — chat-time filters are composed by the SL query planner — so the optional
|
||||
* block must be removed before the SQL becomes a permanent SL source. Substituting a
|
||||
* dummy value (the alternative) bakes a placeholder filter into the source and silently
|
||||
|
|
@ -425,7 +425,7 @@ export class MetabaseClient implements MetabaseRuntimeClient {
|
|||
|
||||
private buildPlaceholderWarningComment(tags: MetabaseTemplateTag[]): string {
|
||||
const lines = [
|
||||
'-- KLO_PLACEHOLDER_WARNING: this SQL was extracted from a Metabase card with',
|
||||
'-- KTX_PLACEHOLDER_WARNING: this SQL was extracted from a Metabase card with',
|
||||
'-- unbound template parameters. The placeholders below were substituted with DUMMY',
|
||||
"-- values to satisfy Metabase's parser — they DO NOT represent intended filters.",
|
||||
'-- Drop the corresponding clauses (or expose them as runtime SL filters) before',
|
||||
|
|
|
|||
|
|
@ -240,7 +240,7 @@ describe('fetchMetabaseBundle', () => {
|
|||
clientFactory,
|
||||
sourceStateReader,
|
||||
}),
|
||||
).rejects.toThrow(/unhydrated.*klo connection mapping refresh/);
|
||||
).rejects.toThrow(/unhydrated.*ktx connection mapping refresh/);
|
||||
});
|
||||
|
||||
it('skips cards whose getResolvedSql returns null and records them in unresolved-cards.json', async () => {
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ export async function fetchMetabaseBundle(params: FetchMetabaseBundleParams): Pr
|
|||
}
|
||||
if (mapping.metabaseDatabaseName === null) {
|
||||
throw new IngestInputError(
|
||||
`mapping for database ${pullConfig.metabaseDatabaseId} on Metabase connection ${pullConfig.metabaseConnectionId} is unhydrated; run \`klo connection mapping refresh ${pullConfig.metabaseConnectionId}\` to populate metabaseDatabaseName before ingest.`,
|
||||
`mapping for database ${pullConfig.metabaseDatabaseId} on Metabase connection ${pullConfig.metabaseConnectionId} is unhydrated; run \`ktx connection mapping refresh ${pullConfig.metabaseConnectionId}\` to populate metabaseDatabaseName before ingest.`,
|
||||
);
|
||||
}
|
||||
const mappingDatabaseName: string = mapping.metabaseDatabaseName;
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type { KloProjectConnectionConfig } from '../../../project/index.js';
|
||||
import type { KtxProjectConnectionConfig } from '../../../project/index.js';
|
||||
import { metabaseRuntimeConfigFromLocalConnection } from './local-metabase.adapter.js';
|
||||
|
||||
describe('metabaseRuntimeConfigFromLocalConnection', () => {
|
||||
it('resolves api_url and env-backed api_key_ref from a flat klo.yaml connection', () => {
|
||||
const connection: KloProjectConnectionConfig = {
|
||||
it('resolves api_url and env-backed api_key_ref from a flat ktx.yaml connection', () => {
|
||||
const connection: KtxProjectConnectionConfig = {
|
||||
driver: 'metabase',
|
||||
api_url: 'https://metabase.example.com',
|
||||
api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret
|
||||
|
|
@ -21,7 +21,7 @@ describe('metabaseRuntimeConfigFromLocalConnection', () => {
|
|||
});
|
||||
|
||||
it('accepts url as the local api URL alias', () => {
|
||||
const connection: KloProjectConnectionConfig = {
|
||||
const connection: KtxProjectConnectionConfig = {
|
||||
driver: 'metabase',
|
||||
url: 'https://metabase.example.com',
|
||||
api_key: 'literal-test-key', // pragma: allowlist secret
|
||||
|
|
@ -34,7 +34,7 @@ describe('metabaseRuntimeConfigFromLocalConnection', () => {
|
|||
});
|
||||
|
||||
it('rejects proxy-bearing local Metabase connections', () => {
|
||||
const connection: KloProjectConnectionConfig = {
|
||||
const connection: KtxProjectConnectionConfig = {
|
||||
driver: 'metabase',
|
||||
api_url: 'https://metabase.example.com',
|
||||
api_key: 'literal-test-key', // pragma: allowlist secret
|
||||
|
|
@ -42,12 +42,12 @@ describe('metabaseRuntimeConfigFromLocalConnection', () => {
|
|||
};
|
||||
|
||||
expect(() => metabaseRuntimeConfigFromLocalConnection('prod-metabase', connection)).toThrow(
|
||||
'Standalone KLO does not support proxy-bearing Metabase connections yet',
|
||||
'Standalone KTX does not support proxy-bearing Metabase connections yet',
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects non-Metabase source connections', () => {
|
||||
const connection: KloProjectConnectionConfig = {
|
||||
const connection: KtxProjectConnectionConfig = {
|
||||
driver: 'postgres',
|
||||
url: 'postgres://localhost/db',
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import type { KloLocalProject, KloProjectConnectionConfig } from '../../../project/index.js';
|
||||
import { kloLocalStateDbPath } from '../../../project/index.js';
|
||||
import type { KtxLocalProject, KtxProjectConnectionConfig } from '../../../project/index.js';
|
||||
import { ktxLocalStateDbPath } from '../../../project/index.js';
|
||||
import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory } from './client.js';
|
||||
import {
|
||||
IngestMetabaseClientFactory,
|
||||
|
|
@ -21,13 +21,13 @@ function resolveEnvReference(ref: string, env: NodeJS.ProcessEnv): string | null
|
|||
return stringField(env[name]);
|
||||
}
|
||||
|
||||
function hasNetworkProxy(connection: KloProjectConnectionConfig): boolean {
|
||||
function hasNetworkProxy(connection: KtxProjectConnectionConfig): boolean {
|
||||
return connection.networkProxy != null || connection.network_proxy != null;
|
||||
}
|
||||
|
||||
export function metabaseRuntimeConfigFromLocalConnection(
|
||||
connectionId: string,
|
||||
connection: KloProjectConnectionConfig | undefined,
|
||||
connection: KtxProjectConnectionConfig | undefined,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): MetabaseClientRuntimeConfig {
|
||||
if (!connection || String(connection.driver).toLowerCase() !== 'metabase') {
|
||||
|
|
@ -35,7 +35,7 @@ export function metabaseRuntimeConfigFromLocalConnection(
|
|||
}
|
||||
if (hasNetworkProxy(connection)) {
|
||||
throw new Error(
|
||||
`Standalone KLO does not support proxy-bearing Metabase connections yet. Use hosted Metabase ingest for "${connectionId}" until the KLO Metabase proxy support spec lands.`,
|
||||
`Standalone KTX does not support proxy-bearing Metabase connections yet. Use hosted Metabase ingest for "${connectionId}" until the KTX Metabase proxy support spec lands.`,
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -60,10 +60,10 @@ interface CreateLocalMetabaseSourceAdapterOptions {
|
|||
}
|
||||
|
||||
export function createLocalMetabaseSourceAdapter(
|
||||
project: KloLocalProject,
|
||||
project: KtxLocalProject,
|
||||
options: CreateLocalMetabaseSourceAdapterOptions = {},
|
||||
): MetabaseSourceAdapter {
|
||||
const sourceStateReader = new LocalMetabaseSourceStateReader({ dbPath: kloLocalStateDbPath(project) });
|
||||
const sourceStateReader = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) });
|
||||
const connectionFactory = new DefaultMetabaseConnectionClientFactory(
|
||||
(metabaseConnectionId) =>
|
||||
metabaseRuntimeConfigFromLocalConnection(
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ describe('LocalMetabaseSourceStateReader', () => {
|
|||
let store: LocalMetabaseSourceStateReader;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-metabase-local-state-'));
|
||||
store = new LocalMetabaseSourceStateReader({ dbPath: join(tempDir, '.klo', 'db.sqlite') });
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-metabase-local-state-'));
|
||||
store = new LocalMetabaseSourceStateReader({ dbPath: join(tempDir, '.ktx', 'db.sqlite') });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
@ -74,7 +74,7 @@ describe('LocalMetabaseSourceStateReader', () => {
|
|||
metabaseDbName: null,
|
||||
targetConnectionId: 'warehouse',
|
||||
syncEnabled: true,
|
||||
source: 'klo.yaml',
|
||||
source: 'ktx.yaml',
|
||||
},
|
||||
{
|
||||
metabaseDatabaseId: 2,
|
||||
|
|
@ -247,7 +247,7 @@ describe('LocalMetabaseSourceStateReader', () => {
|
|||
await store.applyYamlBootstrap({
|
||||
connectionId: 'prod-metabase',
|
||||
syncMode: 'ALL',
|
||||
defaultTagNames: ['klo'],
|
||||
defaultTagNames: ['ktx'],
|
||||
selections: [{ selectionType: 'collection', metabaseObjectId: 12 }],
|
||||
mappings: [{ metabaseDatabaseId: 1, targetConnectionId: 'prod-warehouse', syncEnabled: true }],
|
||||
});
|
||||
|
|
@ -255,7 +255,7 @@ describe('LocalMetabaseSourceStateReader', () => {
|
|||
await expect(store.getUnhydratedSyncEnabledMappingIds('prod-metabase')).resolves.toEqual([1]);
|
||||
await expect(store.getSourceState('prod-metabase')).resolves.toMatchObject({
|
||||
syncMode: 'ALL',
|
||||
defaultTagNames: ['klo'],
|
||||
defaultTagNames: ['ktx'],
|
||||
selections: [{ selectionType: 'collection', metabaseObjectId: 12 }],
|
||||
mappings: [],
|
||||
});
|
||||
|
|
@ -265,7 +265,7 @@ describe('LocalMetabaseSourceStateReader', () => {
|
|||
metabaseDatabaseName: null,
|
||||
targetConnectionId: 'prod-warehouse',
|
||||
syncEnabled: true,
|
||||
source: 'klo.yaml',
|
||||
source: 'ktx.yaml',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
|
@ -301,7 +301,7 @@ describe('LocalMetabaseSourceStateReader', () => {
|
|||
metabaseEngine: 'postgres',
|
||||
targetConnectionId: 'yaml-warehouse',
|
||||
syncEnabled: true,
|
||||
source: 'klo.yaml',
|
||||
source: 'ktx.yaml',
|
||||
},
|
||||
{
|
||||
metabaseDatabaseId: 2,
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import Database from 'better-sqlite3';
|
|||
import type { MetabaseSourceState, MetabaseSourceStateReader, MetabaseSourceStateSelection } from './source-state-port.js';
|
||||
import type { MetabaseSyncMode } from './types.js';
|
||||
|
||||
export type LocalMetabaseMappingSource = 'klo.yaml' | 'cli' | 'refresh';
|
||||
export type LocalMetabaseMappingSource = 'ktx.yaml' | 'cli' | 'refresh';
|
||||
|
||||
interface LocalMetabaseSourceStateStoreOptions {
|
||||
dbPath: string;
|
||||
|
|
@ -197,13 +197,13 @@ export class LocalMetabaseSourceStateReader implements MetabaseSourceStateReader
|
|||
source,
|
||||
updated_at
|
||||
)
|
||||
VALUES (?, ?, NULL, NULL, NULL, NULL, ?, ?, 'klo.yaml', ?)
|
||||
VALUES (?, ?, NULL, NULL, NULL, NULL, ?, ?, 'ktx.yaml', ?)
|
||||
`);
|
||||
const updateRefreshRow = this.db.prepare(`
|
||||
UPDATE local_metabase_database_mappings
|
||||
SET target_connection_id = ?,
|
||||
sync_enabled = ?,
|
||||
source = 'klo.yaml',
|
||||
source = 'ktx.yaml',
|
||||
updated_at = ?
|
||||
WHERE metabase_connection_id = ?
|
||||
AND metabase_database_id = ?
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ describe('computeMetabaseMappingDrift', () => {
|
|||
{ id: 3, name: 'Warehouse', engine: 'mysql', host: 'mysql.internal', dbName: 'warehouse' },
|
||||
],
|
||||
staleMappings: [{ id: '9', reason: 'database_not_found' }],
|
||||
inSync: [{ id: 2, kloConnectionId: 'target-postgres' }],
|
||||
inSync: [{ id: 2, ktxConnectionId: 'target-postgres' }],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -74,7 +74,7 @@ describe('validateMetabaseMappings', () => {
|
|||
expect(
|
||||
validateMetabaseMappings({
|
||||
mappings: { '2': 'target-postgres' },
|
||||
knownKloConnectionIds: new Set(['target-postgres']),
|
||||
knownKtxConnectionIds: new Set(['target-postgres']),
|
||||
}),
|
||||
).toEqual({ ok: true });
|
||||
});
|
||||
|
|
@ -83,11 +83,11 @@ describe('validateMetabaseMappings', () => {
|
|||
expect(
|
||||
validateMetabaseMappings({
|
||||
mappings: { '2': 'missing-target', '3': 'target-mysql' },
|
||||
knownKloConnectionIds: new Set(['target-mysql']),
|
||||
knownKtxConnectionIds: new Set(['target-mysql']),
|
||||
}),
|
||||
).toEqual({
|
||||
ok: false,
|
||||
errors: [{ key: '2', reason: 'KLO connection missing-target does not exist' }],
|
||||
errors: [{ key: '2', reason: 'KTX connection missing-target does not exist' }],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -149,7 +149,7 @@ describe('validateMappingPhysicalMatch', () => {
|
|||
).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for unknown engines because KLO cannot validate them', () => {
|
||||
it('returns null for unknown engines because KTX cannot validate them', () => {
|
||||
expect(
|
||||
validateMappingPhysicalMatch(
|
||||
{ metabaseEngine: 'unknown-engine', metabaseDbName: 'X', metabaseHost: 'host' },
|
||||
|
|
@ -177,7 +177,7 @@ describe('computeMetabaseMappingPhysicalMismatches', () => {
|
|||
).toEqual([
|
||||
{
|
||||
mappingId: 'mapping-bad',
|
||||
reason: "Metabase database 'app' does not match KLO connection database 'other_app'",
|
||||
reason: "Metabase database 'app' does not match KTX connection database 'other_app'",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
|
@ -201,7 +201,7 @@ describe('refreshMetabaseMapping', () => {
|
|||
refreshMetabaseMapping({
|
||||
client,
|
||||
currentMappings: { '2': 'target-postgres' },
|
||||
resolveKloConnectionPhysicalInfo: vi.fn().mockResolvedValue({
|
||||
resolveKtxConnectionPhysicalInfo: vi.fn().mockResolvedValue({
|
||||
connection_type: 'POSTGRESQL',
|
||||
host: 'pg.internal',
|
||||
database: 'wrong_database',
|
||||
|
|
@ -211,12 +211,12 @@ describe('refreshMetabaseMapping', () => {
|
|||
drift: {
|
||||
unmappedDiscovered: [],
|
||||
staleMappings: [],
|
||||
inSync: [{ id: 2, kloConnectionId: 'target-postgres' }],
|
||||
inSync: [{ id: 2, ktxConnectionId: 'target-postgres' }],
|
||||
},
|
||||
physicalMismatches: [
|
||||
{
|
||||
mappingId: '2',
|
||||
reason: "Metabase database 'analytics' does not match KLO connection database 'wrong_database'",
|
||||
reason: "Metabase database 'analytics' does not match KTX connection database 'wrong_database'",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
|
@ -282,7 +282,7 @@ describe('findBestMatch', () => {
|
|||
});
|
||||
|
||||
describe('METABASE_ENGINE_TO_CONNECTION_TYPE', () => {
|
||||
it('keeps the server-supported Metabase engine table in KLO', () => {
|
||||
it('keeps the server-supported Metabase engine table in KTX', () => {
|
||||
expect(METABASE_ENGINE_TO_CONNECTION_TYPE).toMatchObject({
|
||||
postgres: 'POSTGRESQL',
|
||||
bigquery: 'BIGQUERY',
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ export interface DiscoveredMetabaseDatabase {
|
|||
export interface MetabaseMappingDrift {
|
||||
unmappedDiscovered: DiscoveredMetabaseDatabase[];
|
||||
staleMappings: Array<{ id: string; reason: 'database_not_found' }>;
|
||||
inSync: Array<{ id: number; kloConnectionId: string }>;
|
||||
inSync: Array<{ id: number; ktxConnectionId: string }>;
|
||||
}
|
||||
|
||||
export interface MappingPhysicalInfo {
|
||||
|
|
@ -32,7 +32,7 @@ export interface MappingPhysicalInfo {
|
|||
metabaseHost: string | null;
|
||||
}
|
||||
|
||||
export interface KloConnectionPhysicalInfo {
|
||||
export interface KtxConnectionPhysicalInfo {
|
||||
connection_type: string;
|
||||
database?: unknown;
|
||||
host?: unknown;
|
||||
|
|
@ -45,7 +45,7 @@ export interface KloConnectionPhysicalInfo {
|
|||
export interface PhysicalMismatchInput {
|
||||
mappingId: string;
|
||||
metabase: MappingPhysicalInfo;
|
||||
target: KloConnectionPhysicalInfo;
|
||||
target: KtxConnectionPhysicalInfo;
|
||||
}
|
||||
|
||||
export interface PhysicalMismatch {
|
||||
|
|
@ -102,7 +102,7 @@ function displayValue(value: unknown): string {
|
|||
return typeof value === 'string' && value.length > 0 ? value : 'unknown';
|
||||
}
|
||||
|
||||
function getTargetDatabase(target: KloConnectionPhysicalInfo): unknown {
|
||||
function getTargetDatabase(target: KtxConnectionPhysicalInfo): unknown {
|
||||
if (target.connection_type === 'BIGQUERY') {
|
||||
return target.dataset_id ?? target.project_id ?? target.database;
|
||||
}
|
||||
|
|
@ -164,23 +164,23 @@ export function computeMetabaseMappingDrift(args: {
|
|||
.filter((id) => !discoveredById.has(id))
|
||||
.map((id) => ({ id, reason: 'database_not_found' as const }));
|
||||
const inSync = Object.entries(args.currentMappings)
|
||||
.filter(([id, kloConnectionId]) => discoveredById.has(id) && typeof kloConnectionId === 'string')
|
||||
.map(([id, kloConnectionId]) => ({ id: Number(id), kloConnectionId: kloConnectionId as string }));
|
||||
.filter(([id, ktxConnectionId]) => discoveredById.has(id) && typeof ktxConnectionId === 'string')
|
||||
.map(([id, ktxConnectionId]) => ({ id: Number(id), ktxConnectionId: ktxConnectionId as string }));
|
||||
|
||||
return { unmappedDiscovered, staleMappings, inSync };
|
||||
}
|
||||
|
||||
export function validateMetabaseMappings(args: {
|
||||
mappings: Record<string, string | null | undefined>;
|
||||
knownKloConnectionIds: Set<string>;
|
||||
knownKtxConnectionIds: Set<string>;
|
||||
}): MetabaseMappingValidationResult {
|
||||
const errors: Array<{ key: string; reason: string }> = [];
|
||||
for (const [key, connectionId] of Object.entries(args.mappings)) {
|
||||
if (!connectionId) {
|
||||
continue;
|
||||
}
|
||||
if (!args.knownKloConnectionIds.has(connectionId)) {
|
||||
errors.push({ key, reason: `KLO connection ${connectionId} does not exist` });
|
||||
if (!args.knownKtxConnectionIds.has(connectionId)) {
|
||||
errors.push({ key, reason: `KTX connection ${connectionId} does not exist` });
|
||||
}
|
||||
}
|
||||
return errors.length === 0 ? { ok: true } : { ok: false, errors };
|
||||
|
|
@ -188,7 +188,7 @@ export function validateMetabaseMappings(args: {
|
|||
|
||||
export function validateMappingPhysicalMatch(
|
||||
mapping: MappingPhysicalInfo,
|
||||
target: KloConnectionPhysicalInfo,
|
||||
target: KtxConnectionPhysicalInfo,
|
||||
): string | null {
|
||||
const engine = mapping.metabaseEngine?.toLowerCase();
|
||||
if (!engine) {
|
||||
|
|
@ -201,7 +201,7 @@ export function validateMappingPhysicalMatch(
|
|||
}
|
||||
|
||||
if (target.connection_type !== expectedType) {
|
||||
return `Metabase database engine '${engine}' does not match KLO connection type '${target.connection_type}'`;
|
||||
return `Metabase database engine '${engine}' does not match KTX connection type '${target.connection_type}'`;
|
||||
}
|
||||
|
||||
const metabaseDb = normalizeName(mapping.metabaseDbName);
|
||||
|
|
@ -209,7 +209,7 @@ export function validateMappingPhysicalMatch(
|
|||
|
||||
if (engine === 'snowflake' || engine === 'bigquery' || engine === 'bigquery-cloud-sdk') {
|
||||
if (metabaseDb && targetDb && metabaseDb !== targetDb) {
|
||||
return `Metabase database '${mapping.metabaseDbName}' does not match KLO connection database '${displayValue(
|
||||
return `Metabase database '${mapping.metabaseDbName}' does not match KTX connection database '${displayValue(
|
||||
getTargetDatabase(target),
|
||||
)}'`;
|
||||
}
|
||||
|
|
@ -221,12 +221,12 @@ export function validateMappingPhysicalMatch(
|
|||
const targetHost = normalizeHost(target.host);
|
||||
|
||||
if (metabaseHost && targetHost && metabaseHost !== targetHost) {
|
||||
return `Metabase host '${mapping.metabaseHost}' does not match KLO connection host '${displayValue(
|
||||
return `Metabase host '${mapping.metabaseHost}' does not match KTX connection host '${displayValue(
|
||||
target.host,
|
||||
)}'`;
|
||||
}
|
||||
if (metabaseDb && targetDb && metabaseDb !== targetDb) {
|
||||
return `Metabase database '${mapping.metabaseDbName}' does not match KLO connection database '${displayValue(
|
||||
return `Metabase database '${mapping.metabaseDbName}' does not match KTX connection database '${displayValue(
|
||||
getTargetDatabase(target),
|
||||
)}'`;
|
||||
}
|
||||
|
|
@ -250,7 +250,7 @@ export function computeMetabaseMappingPhysicalMismatches(inputs: PhysicalMismatc
|
|||
export async function refreshMetabaseMapping(args: {
|
||||
client: Pick<MetabaseRuntimeClient, 'getDatabases'>;
|
||||
currentMappings: Record<string, string | null | undefined>;
|
||||
resolveKloConnectionPhysicalInfo: (kloConnectionId: string) => Promise<KloConnectionPhysicalInfo | null>;
|
||||
resolveKtxConnectionPhysicalInfo: (ktxConnectionId: string) => Promise<KtxConnectionPhysicalInfo | null>;
|
||||
}): Promise<MappingRefreshReport> {
|
||||
const discovered = await discoverMetabaseDatabases(args.client);
|
||||
const drift = computeMetabaseMappingDrift({ currentMappings: args.currentMappings, discovered });
|
||||
|
|
@ -262,11 +262,11 @@ export async function refreshMetabaseMapping(args: {
|
|||
if (!discoveredDatabase) {
|
||||
continue;
|
||||
}
|
||||
const target = await args.resolveKloConnectionPhysicalInfo(mapping.kloConnectionId);
|
||||
const target = await args.resolveKtxConnectionPhysicalInfo(mapping.ktxConnectionId);
|
||||
if (!target) {
|
||||
physicalMismatches.push({
|
||||
mappingId: String(mapping.id),
|
||||
reason: `KLO connection ${mapping.kloConnectionId} does not exist`,
|
||||
reason: `KTX connection ${mapping.ktxConnectionId} does not exist`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { parse as parseYaml } from 'yaml';
|
||||
import { noopLogger, type KloLogger } from '../../../core/index.js';
|
||||
import { noopLogger, type KtxLogger } from '../../../core/index.js';
|
||||
|
||||
export interface DimensionDefinition {
|
||||
name: string;
|
||||
|
|
@ -42,7 +42,7 @@ export interface ParsedMetricflowRelationship {
|
|||
}
|
||||
|
||||
export interface MetricflowParseOptions {
|
||||
logger?: KloLogger;
|
||||
logger?: KtxLogger;
|
||||
}
|
||||
|
||||
// ============ MetricFlow YAML Interfaces ============
|
||||
|
|
@ -191,7 +191,7 @@ export function translateMetricflowJinjaFilter(filter: string): string {
|
|||
}
|
||||
|
||||
class MetricflowDeepParser {
|
||||
constructor(private readonly logger: KloLogger) {}
|
||||
constructor(private readonly logger: KtxLogger) {}
|
||||
|
||||
parseFiles(files: Array<{ content: string; path: string }>): MetricFlowParseResult {
|
||||
this.logger.log(`Parsing ${files.length} files for MetricFlow definitions`);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { mkdir, mkdtemp, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { describe, expect, test } from 'vitest';
|
||||
import type { KloEmbeddingPort } from '../../../core/embedding.js';
|
||||
import type { KtxEmbeddingPort } from '../../../core/embedding.js';
|
||||
import type { WorkUnit } from '../../types.js';
|
||||
import { clusterNotionWorkUnits, MIN_PAGES_TO_CLUSTER } from './cluster.js';
|
||||
|
||||
|
|
@ -14,7 +14,7 @@ function fakeEmbedding(text: string): number[] {
|
|||
return v;
|
||||
}
|
||||
|
||||
const mockEmbed: KloEmbeddingPort = {
|
||||
const mockEmbed: KtxEmbeddingPort = {
|
||||
maxBatchSize: 100,
|
||||
computeEmbedding: async (t: string) => fakeEmbedding(t),
|
||||
computeEmbeddingsBulk: async (texts: string[]) => texts.map(fakeEmbedding),
|
||||
|
|
@ -104,7 +104,7 @@ describe('clusterNotionWorkUnits', () => {
|
|||
}));
|
||||
const stagedDir = await makeStaged(pages);
|
||||
const wus = makeWorkUnits(pages);
|
||||
const failingEmbed: KloEmbeddingPort = {
|
||||
const failingEmbed: KtxEmbeddingPort = {
|
||||
maxBatchSize: 100,
|
||||
computeEmbedding: async () => {
|
||||
throw new Error('embedding down');
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { readFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import type { KloEmbeddingPort } from '../../../core/embedding.js';
|
||||
import type { KtxEmbeddingPort } from '../../../core/embedding.js';
|
||||
import { kmeans, pickK } from '../../clustering/kmeans.js';
|
||||
import type { WorkUnit } from '../../types.js';
|
||||
import { notionMetadataSchema } from './types.js';
|
||||
|
|
@ -12,7 +12,7 @@ const CLUSTER_SEED = 42;
|
|||
interface ClusterNotionWorkUnitsArgs {
|
||||
workUnits: WorkUnit[];
|
||||
stagedDir: string;
|
||||
embedding: KloEmbeddingPort;
|
||||
embedding: KtxEmbeddingPort;
|
||||
}
|
||||
|
||||
async function buildClusterText(wu: WorkUnit, stagedDir: string): Promise<string> {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { type KloLogger, noopLogger } from '../../core/index.js';
|
||||
import { type KtxLogger, noopLogger } from '../../core/index.js';
|
||||
import type { CandidateDedupResult, ContextCandidateForDedup, JsonValue } from '../ports.js';
|
||||
import { buildContextCandidateEmbeddingText } from './embedding-text.js';
|
||||
import type { ContextCandidateStorePort } from './store.js';
|
||||
|
|
@ -17,11 +17,11 @@ export interface CandidateDedupServiceDeps {
|
|||
store: ContextCandidateStorePort;
|
||||
embeddings: ContextCandidateEmbeddingPort;
|
||||
settings: CandidateDedupSettings;
|
||||
logger?: KloLogger;
|
||||
logger?: KtxLogger;
|
||||
}
|
||||
|
||||
export class CandidateDedupService {
|
||||
private readonly logger: KloLogger;
|
||||
private readonly logger: KtxLogger;
|
||||
|
||||
constructor(private readonly deps: CandidateDedupServiceDeps) {
|
||||
this.logger = deps.logger ?? noopLogger;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { type KloLogger, noopLogger } from '../../core/index.js';
|
||||
import { type KtxLogger, noopLogger } from '../../core/index.js';
|
||||
import type { JsonValue } from '../ports.js';
|
||||
import type { ContextCandidateStorePort } from './store.js';
|
||||
import type {
|
||||
|
|
@ -26,11 +26,11 @@ export interface ContextCandidateCarryforwardResult {
|
|||
export interface ContextCandidateCarryforwardServiceDeps {
|
||||
store: ContextCandidateStorePort;
|
||||
settings: ContextCandidateCarryforwardSettings;
|
||||
logger?: KloLogger;
|
||||
logger?: KtxLogger;
|
||||
}
|
||||
|
||||
export class ContextCandidateCarryforwardService {
|
||||
private readonly logger: KloLogger;
|
||||
private readonly logger: KtxLogger;
|
||||
|
||||
constructor(private readonly deps: ContextCandidateCarryforwardServiceDeps) {
|
||||
this.logger = deps.logger ?? noopLogger;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import type { KloModelRole } from '@klo/llm';
|
||||
import type { KtxModelRole } from '@ktx/llm';
|
||||
import type { ToolSet } from 'ai';
|
||||
import type { AgentRunnerService } from '../../agent/index.js';
|
||||
import { type KloLogger, noopLogger } from '../../core/index.js';
|
||||
import { type KtxLogger, noopLogger } from '../../core/index.js';
|
||||
import type { MemoryAction } from '../../memory/index.js';
|
||||
import type { ContextCandidateForDedup, CuratorPaginationPort, CuratorPaginationReport } from '../ports.js';
|
||||
import type {
|
||||
|
|
@ -35,7 +35,7 @@ export interface CuratorPaginationInput {
|
|||
evictionUnit: EvictionUnit | undefined;
|
||||
representatives: ContextCandidateForDedup[];
|
||||
initialBudget: CuratorPaginationBudget;
|
||||
modelRole: KloModelRole;
|
||||
modelRole: KtxModelRole;
|
||||
buildSystemPrompt: () => string;
|
||||
buildUserPrompt: (input: CuratorPaginationPromptInput) => string;
|
||||
buildToolSet: (passNumber: number) => ToolSet;
|
||||
|
|
@ -52,11 +52,11 @@ export interface CuratorPaginationServiceDeps {
|
|||
store: ContextCandidateStorePort;
|
||||
agentRunner: AgentRunnerService;
|
||||
settings: CuratorPaginationSettings;
|
||||
logger?: KloLogger;
|
||||
logger?: KtxLogger;
|
||||
}
|
||||
|
||||
export class CuratorPaginationService implements CuratorPaginationPort {
|
||||
private readonly logger: KloLogger;
|
||||
private readonly logger: KtxLogger;
|
||||
|
||||
constructor(private readonly deps: CuratorPaginationServiceDeps) {
|
||||
this.logger = deps.logger ?? noopLogger;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { readdir, readFile } from 'node:fs/promises';
|
||||
import { basename, dirname, join, relative } from 'node:path';
|
||||
import { noopLogger, type KloLogger } from '../../core/index.js';
|
||||
import { noopLogger, type KtxLogger } from '../../core/index.js';
|
||||
import type { JsonValue } from '../ports.js';
|
||||
import type { DiffSet } from '../types.js';
|
||||
import type { ContextEvidenceIndexStorePort } from './store.js';
|
||||
|
|
@ -32,7 +32,7 @@ interface PublishSyncArgs {
|
|||
interface ContextEvidenceIndexServiceDeps {
|
||||
store: ContextEvidenceIndexStorePort;
|
||||
embeddings: ContextEvidenceEmbeddingPort;
|
||||
logger?: Pick<KloLogger, 'warn'>;
|
||||
logger?: Pick<KtxLogger, 'warn'>;
|
||||
}
|
||||
|
||||
type JsonObject = { [key: string]: JsonValue | undefined };
|
||||
|
|
@ -66,7 +66,7 @@ interface MarkdownChunk {
|
|||
export class ContextEvidenceIndexService {
|
||||
private readonly store: ContextEvidenceIndexStorePort;
|
||||
private readonly embeddings: ContextEvidenceEmbeddingPort;
|
||||
private readonly logger: Pick<KloLogger, 'warn'>;
|
||||
private readonly logger: Pick<KtxLogger, 'warn'>;
|
||||
|
||||
constructor(deps: ContextEvidenceIndexServiceDeps) {
|
||||
this.store = deps.store;
|
||||
|
|
|
|||
|
|
@ -11,8 +11,8 @@ describe('SqliteContextEvidenceStore', () => {
|
|||
let dbPath: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-context-evidence-sqlite-'));
|
||||
dbPath = join(tempDir, '.klo', 'db.sqlite');
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-context-evidence-sqlite-'));
|
||||
dbPath = join(tempDir, '.ktx', 'db.sqlite');
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@ export { DbtSourceAdapter } from './adapters/dbt/dbt.adapter.js';
|
|||
export { FakeSourceAdapter } from './adapters/fake/fake.adapter.js';
|
||||
export type {
|
||||
DaemonLiveDatabaseIntrospectionOptions,
|
||||
KloDaemonDatabaseHttpJsonRunner,
|
||||
KloDaemonDatabaseIntrospectionCommand,
|
||||
KloDaemonDatabaseJsonRunner,
|
||||
KtxDaemonDatabaseHttpJsonRunner,
|
||||
KtxDaemonDatabaseIntrospectionCommand,
|
||||
KtxDaemonDatabaseJsonRunner,
|
||||
} from './adapters/live-database/daemon-introspection.js';
|
||||
export { createDaemonLiveDatabaseIntrospection } from './adapters/live-database/daemon-introspection.js';
|
||||
export type {
|
||||
|
|
@ -15,7 +15,7 @@ export type {
|
|||
} from './adapters/live-database/extracted-schema.js';
|
||||
export {
|
||||
buildLiveDatabaseTableNaturalKey,
|
||||
kloSchemaSnapshotToExtractedSchema,
|
||||
ktxSchemaSnapshotToExtractedSchema,
|
||||
} from './adapters/live-database/extracted-schema.js';
|
||||
export { LiveDatabaseSourceAdapter } from './adapters/live-database/live-database.adapter.js';
|
||||
export type {
|
||||
|
|
@ -68,7 +68,7 @@ export {
|
|||
export {
|
||||
createDaemonLookerTableIdentifierParser,
|
||||
type DaemonLookerTableIdentifierParserOptions,
|
||||
type KloDaemonTableIdentifierHttpJsonRunner,
|
||||
type KtxDaemonTableIdentifierHttpJsonRunner,
|
||||
} from './adapters/looker/daemon-table-identifier-parser.js';
|
||||
export type {
|
||||
LookerConnectionClientFactory,
|
||||
|
|
@ -102,12 +102,12 @@ export {
|
|||
projectParsedIdentifier,
|
||||
refreshLookerMappingPlaceholders,
|
||||
sqlglotDialectForConnectionType,
|
||||
suggestKloConnectionForLookerConnection,
|
||||
suggestKtxConnectionForLookerConnection,
|
||||
validateLookerMappings,
|
||||
validateLookerWarehouseTarget,
|
||||
} from './adapters/looker/mapping.js';
|
||||
export type {
|
||||
LookerConnectionMapping as KloLookerConnectionMapping,
|
||||
LookerConnectionMapping as KtxLookerConnectionMapping,
|
||||
LookerMappingCandidateConnection,
|
||||
LookerMappingClient,
|
||||
LookerMappingDrift,
|
||||
|
|
@ -220,7 +220,7 @@ export type {
|
|||
AutoMatchCandidate,
|
||||
AutoMatchResult as MetabaseAutoMatchResult,
|
||||
DiscoveredMetabaseDatabase,
|
||||
KloConnectionPhysicalInfo,
|
||||
KtxConnectionPhysicalInfo,
|
||||
MappingPhysicalInfo,
|
||||
MappingRefreshReport,
|
||||
MetabaseMappedConnectionType,
|
||||
|
|
@ -347,7 +347,7 @@ export type {
|
|||
HistoricSqlSourceAdapterDeps,
|
||||
HistoricSqlTimeWindow,
|
||||
HistoricSqlUsage,
|
||||
KloPostgresQueryClient,
|
||||
KtxPostgresQueryClient,
|
||||
PostgresPgssAggregateRow,
|
||||
PostgresPgssProbeResult,
|
||||
PostgresPgssReader,
|
||||
|
|
@ -424,7 +424,7 @@ export type {
|
|||
RunLocalMetabaseIngestOptions,
|
||||
} from './local-ingest.js';
|
||||
export { getLatestLocalIngestStatus, getLocalIngestStatus, runLocalIngest, runLocalMetabaseIngest } from './local-ingest.js';
|
||||
export { seedLocalMappingStateFromKloYaml } from './local-mapping-reconcile.js';
|
||||
export { seedLocalMappingStateFromKtxYaml } from './local-mapping-reconcile.js';
|
||||
export type {
|
||||
CreateLocalBundleIngestRuntimeOptions,
|
||||
LocalBundleIngestRuntime,
|
||||
|
|
@ -475,7 +475,7 @@ export type {
|
|||
DbtSchemaFile,
|
||||
DbtSchemaParseResult,
|
||||
} from './adapters/dbt-descriptions/parse-schema.js';
|
||||
export { findMatchingKloTable, matchDbtTables } from './adapters/dbt-descriptions/match-tables.js';
|
||||
export { findMatchingKtxTable, matchDbtTables } from './adapters/dbt-descriptions/match-tables.js';
|
||||
export type { DbtHostTableLite, DbtTableMatch } from './adapters/dbt-descriptions/match-tables.js';
|
||||
export { toDescriptionUpdates } from './adapters/dbt-descriptions/to-description-updates.js';
|
||||
export type { DbtDescriptionUpdates } from './adapters/dbt-descriptions/to-description-updates.js';
|
||||
|
|
@ -483,7 +483,7 @@ export { toRelationshipUpdates } from './adapters/dbt-descriptions/to-relationsh
|
|||
export type { DbtRelationshipUpdates } from './adapters/dbt-descriptions/to-relationship-updates.js';
|
||||
export { toMetadataUpdates } from './adapters/dbt-descriptions/to-metadata-updates.js';
|
||||
export { mergeSemanticModelTables } from './adapters/dbt-descriptions/merge-semantic-model-tables.js';
|
||||
export type { KloJoinUpdate, KloMetadataUpdate } from '../scan/enrichment-types.js';
|
||||
export type { KtxJoinUpdate, KtxMetadataUpdate } from '../scan/enrichment-types.js';
|
||||
export {
|
||||
createInitialMemoryFlowInteractionState,
|
||||
findMemoryFlowSearchMatches,
|
||||
|
|
|
|||
|
|
@ -243,11 +243,11 @@ const buildRunner = (deps: ReturnType<typeof makeDeps> = makeDeps(), overrides:
|
|||
gitService: deps.gitService as any,
|
||||
lockingService: deps.lockingService as any,
|
||||
storage: {
|
||||
homeDir: '/tmp/klo-test',
|
||||
systemGitAuthor: { name: 'KLO Test', email: 'system@klo.local' },
|
||||
resolveUploadDir: (uploadId) => `/tmp/klo-test/ingest-uploads/${uploadId}`,
|
||||
resolvePullDir: (jobId) => `/tmp/klo-test/ingest-pulls/${jobId}`,
|
||||
resolveTranscriptDir: (jobId) => `/tmp/klo-test/run/wu-transcripts/${jobId}`,
|
||||
homeDir: '/tmp/ktx-test',
|
||||
systemGitAuthor: { name: 'KTX Test', email: 'system@ktx.local' },
|
||||
resolveUploadDir: (uploadId) => `/tmp/ktx-test/ingest-uploads/${uploadId}`,
|
||||
resolvePullDir: (jobId) => `/tmp/ktx-test/ingest-pulls/${jobId}`,
|
||||
resolveTranscriptDir: (jobId) => `/tmp/ktx-test/run/wu-transcripts/${jobId}`,
|
||||
},
|
||||
settings: { probeRowCount: 1, memoryIngestionModel: 'test-model' },
|
||||
skillsRegistry: deps.skillsRegistry as any,
|
||||
|
|
@ -845,7 +845,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|||
toolTranscripts: [
|
||||
{
|
||||
unitKey: 'u1',
|
||||
path: '/tmp/klo-test/run/wu-transcripts/j1/u1.jsonl',
|
||||
path: '/tmp/ktx-test/run/wu-transcripts/j1/u1.jsonl',
|
||||
toolCallCount: 2,
|
||||
errorCount: 0,
|
||||
toolNames: ['read_raw_span', 'wiki_write'],
|
||||
|
|
@ -1065,7 +1065,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|||
});
|
||||
|
||||
it('runs manual override reconciliation from the prior report snapshot and marks the prior report superseded', async () => {
|
||||
const tempRoot = await mkdtemp(join(tmpdir(), 'klo-override-'));
|
||||
const tempRoot = await mkdtemp(join(tmpdir(), 'ktx-override-'));
|
||||
const deps = makeDeps();
|
||||
deps.reportsRepo.findByJobId.mockResolvedValue({
|
||||
id: 'report-old',
|
||||
|
|
@ -1134,7 +1134,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|||
...(buildRunner(deps) as any).deps,
|
||||
storage: {
|
||||
homeDir: tempRoot,
|
||||
systemGitAuthor: { name: 'KLO Test', email: 'system@klo.local' },
|
||||
systemGitAuthor: { name: 'KTX Test', email: 'system@ktx.local' },
|
||||
resolveUploadDir: (uploadId: string) => join(tempRoot, 'ingest-uploads', uploadId),
|
||||
resolvePullDir: (jobId: string) => join(tempRoot, 'ingest-pulls', jobId),
|
||||
resolveTranscriptDir: (jobId: string) => join(tempRoot, 'run', 'wu-transcripts', jobId),
|
||||
|
|
@ -1770,8 +1770,8 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|||
await currentToolSession.gitService.commitFiles(
|
||||
['semantic-layer/c1/good.yaml'],
|
||||
'test: add good source',
|
||||
'KLO Test',
|
||||
'system@klo.local',
|
||||
'KTX Test',
|
||||
'system@ktx.local',
|
||||
);
|
||||
}
|
||||
if (unitKey === 'wu-bad') {
|
||||
|
|
@ -1782,8 +1782,8 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|||
await currentToolSession.gitService.commitFiles(
|
||||
['semantic-layer/c1/bad.yaml'],
|
||||
'test: add bad source',
|
||||
'KLO Test',
|
||||
'system@klo.local',
|
||||
'KTX Test',
|
||||
'system@ktx.local',
|
||||
);
|
||||
}
|
||||
return { stopReason: 'natural' };
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { dirname, join } from 'node:path';
|
|||
import { type Tool, tool } from 'ai';
|
||||
import pLimit from 'p-limit';
|
||||
import { z } from 'zod';
|
||||
import { type KloLogger, noopLogger } from '../core/index.js';
|
||||
import { type KtxLogger, noopLogger } from '../core/index.js';
|
||||
import type { CaptureSession, MemoryAction } from '../memory/index.js';
|
||||
import type { SlValidationDeps } from '../sl/index.js';
|
||||
import { createTouchedSlSources, type ToolContext, type ToolSession } from '../tools/index.js';
|
||||
|
|
@ -88,7 +88,7 @@ function reportIdFromCreateResult(result: unknown): string | undefined {
|
|||
}
|
||||
|
||||
export class IngestBundleRunner {
|
||||
private readonly logger: KloLogger;
|
||||
private readonly logger: KtxLogger;
|
||||
private readonly chainByConnection = new Map<string, Promise<unknown>>();
|
||||
|
||||
constructor(private readonly deps: IngestBundleRunnerDeps) {
|
||||
|
|
|
|||
|
|
@ -18,14 +18,14 @@ describe('ingest prompt assets', () => {
|
|||
expect(prompt).toContain('Do not create a duplicate contested artifact');
|
||||
});
|
||||
|
||||
it('uses product-neutral KLO runtime wording', async () => {
|
||||
it('uses product-neutral KTX runtime wording', async () => {
|
||||
const prompt = await readFile(
|
||||
new URL('../../prompts/memory_agent_bundle_ingest_work_unit.md', import.meta.url),
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
expect(prompt).toContain('KLO semantic-layer sources and/or knowledge wiki pages');
|
||||
expect(prompt).toContain('maps cleanly to KLO');
|
||||
expect(prompt).toContain('KTX semantic-layer sources and/or knowledge wiki pages');
|
||||
expect(prompt).toContain('maps cleanly to KTX');
|
||||
expect(prompt).not.toMatch(forbiddenProductPattern());
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ function forbiddenProductPattern() {
|
|||
}
|
||||
|
||||
describe('ingest runtime assets', () => {
|
||||
it('resolves every reusable ingest skill from packaged KLO assets without server fallback', async () => {
|
||||
it('resolves every reusable ingest skill from packaged KTX assets without server fallback', async () => {
|
||||
const registry = new SkillsRegistryService({ skillsDir });
|
||||
const expected = [...new Set([...adapterSkillNames, ...adapterReconcileSkillNames])].sort();
|
||||
|
||||
|
|
@ -48,7 +48,7 @@ describe('ingest runtime assets', () => {
|
|||
}
|
||||
});
|
||||
|
||||
it('loads page-triage and light-extraction prompts from packaged KLO prompt assets', async () => {
|
||||
it('loads page-triage and light-extraction prompts from packaged KTX prompt assets', async () => {
|
||||
const prompts = new PromptService({ promptsDir, partials: [] });
|
||||
|
||||
for (const promptName of pageTriagePromptNames) {
|
||||
|
|
@ -67,7 +67,7 @@ describe('ingest runtime assets', () => {
|
|||
await expect(prompts.loadPrompt('skills/light_extraction')).resolves.toContain('# Light Context Extraction');
|
||||
});
|
||||
|
||||
it('packages historic-SQL WorkUnit skill guidance from KLO assets', async () => {
|
||||
it('packages historic-SQL WorkUnit skill guidance from KTX assets', async () => {
|
||||
const registry = new SkillsRegistryService({ skillsDir });
|
||||
const skills = await registry.listSkills(['historic_sql_ingest'], 'memory_agent');
|
||||
|
||||
|
|
@ -100,7 +100,7 @@ describe('ingest runtime assets', () => {
|
|||
expect(body).not.toMatch(forbiddenProductPattern());
|
||||
});
|
||||
|
||||
it('packages historic-SQL curator reconcile guidance from KLO assets', async () => {
|
||||
it('packages historic-SQL curator reconcile guidance from KTX assets', async () => {
|
||||
const registry = new SkillsRegistryService({ skillsDir });
|
||||
const skills = await registry.listSkills(['historic_sql_curator'], 'memory_agent');
|
||||
|
||||
|
|
|
|||
|
|
@ -2,27 +2,27 @@ import { mkdtemp, rm } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { initKloProject, type KloLocalProject, loadKloProject } from '../project/index.js';
|
||||
import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js';
|
||||
import type { SqlAnalysisPort } from '../sql-analysis/index.js';
|
||||
import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js';
|
||||
import { createDefaultLocalIngestAdapters, localPullConfigForAdapter } from './local-adapters.js';
|
||||
|
||||
describe('local ingest adapters', () => {
|
||||
let tempDir: string;
|
||||
let project: KloLocalProject;
|
||||
let project: KtxLocalProject;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-local-adapters-'));
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-adapters-'));
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
project = await loadKloProject({ projectDir });
|
||||
await initKtxProject({ projectDir, projectName: 'warehouse' });
|
||||
project = await loadKtxProject({ projectDir });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function projectWithConnections(connections: KloLocalProject['config']['connections']): KloLocalProject {
|
||||
function projectWithConnections(connections: KtxLocalProject['config']['connections']): KtxLocalProject {
|
||||
return {
|
||||
...project,
|
||||
config: {
|
||||
|
|
@ -101,7 +101,7 @@ describe('local ingest adapters', () => {
|
|||
return { headers: [], rows: [] };
|
||||
},
|
||||
},
|
||||
postgresBaselineRootDir: join(project.projectDir, '.klo/cache/historic-sql'),
|
||||
postgresBaselineRootDir: join(project.projectDir, '.ktx/cache/historic-sql'),
|
||||
},
|
||||
});
|
||||
|
||||
|
|
@ -187,7 +187,7 @@ describe('local ingest adapters', () => {
|
|||
});
|
||||
|
||||
it('builds Looker pull config from local mapping state', async () => {
|
||||
const projectDir = await mkdtemp(join(tmpdir(), 'klo-local-looker-'));
|
||||
const projectDir = await mkdtemp(join(tmpdir(), 'ktx-local-looker-'));
|
||||
const lookerProject = {
|
||||
projectDir,
|
||||
config: {
|
||||
|
|
@ -204,12 +204,12 @@ describe('local ingest adapters', () => {
|
|||
},
|
||||
},
|
||||
} as never;
|
||||
const store = new LocalLookerRuntimeStore({ dbPath: join(projectDir, '.klo/db.sqlite') });
|
||||
const store = new LocalLookerRuntimeStore({ dbPath: join(projectDir, '.ktx/db.sqlite') });
|
||||
await store.setCursors('prod-looker', { dashboardsLastSyncedAt: null, looksLastSyncedAt: null });
|
||||
await store.upsertConnectionMapping({
|
||||
lookerConnectionId: 'prod-looker',
|
||||
lookerConnectionName: 'analytics',
|
||||
kloConnectionId: 'prod-warehouse',
|
||||
ktxConnectionId: 'prod-warehouse',
|
||||
source: 'cli',
|
||||
});
|
||||
const lookerDeps = {
|
||||
|
|
@ -263,7 +263,7 @@ describe('local ingest adapters', () => {
|
|||
});
|
||||
|
||||
it('builds Looker pull config from yaml mapping bootstrap when SQLite is empty', async () => {
|
||||
const projectDir = await mkdtemp(join(tmpdir(), 'klo-local-looker-yaml-'));
|
||||
const projectDir = await mkdtemp(join(tmpdir(), 'ktx-local-looker-yaml-'));
|
||||
const lookerProject = {
|
||||
projectDir,
|
||||
config: {
|
||||
|
|
@ -327,7 +327,7 @@ describe('local ingest adapters', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('builds LookML pull config from flat klo.yaml connection fields', async () => {
|
||||
it('builds LookML pull config from flat ktx.yaml connection fields', async () => {
|
||||
const lookmlProject = {
|
||||
projectDir: tempDir,
|
||||
config: {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { join } from 'node:path';
|
||||
import { localConnectionToWarehouseDescriptor, notionConnectionToPullConfig, parseNotionConnectionConfig } from '../connections/index.js';
|
||||
import { resolveKloConfigReference } from '../core/config-reference.js';
|
||||
import type { KloLocalProject } from '../project/index.js';
|
||||
import { resolveKtxConfigReference } from '../core/config-reference.js';
|
||||
import type { KtxLocalProject } from '../project/index.js';
|
||||
import type { SqlAnalysisPort } from '../sql-analysis/index.js';
|
||||
import { DbtSourceAdapter } from './adapters/dbt/dbt.adapter.js';
|
||||
import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js';
|
||||
|
|
@ -11,7 +11,7 @@ import { SnowflakeHistoricSqlQueryHistoryReader } from './adapters/historic-sql/
|
|||
import {
|
||||
HISTORIC_SQL_SOURCE_KEY,
|
||||
historicSqlPullConfigSchema,
|
||||
type KloPostgresQueryClient,
|
||||
type KtxPostgresQueryClient,
|
||||
} from './adapters/historic-sql/types.js';
|
||||
import {
|
||||
createDaemonLiveDatabaseIntrospection,
|
||||
|
|
@ -35,7 +35,7 @@ import { createLocalMetabaseSourceAdapter } from './adapters/metabase/local-meta
|
|||
import { MetricflowSourceAdapter } from './adapters/metricflow/metricflow.adapter.js';
|
||||
import { pullConfigFromMetricflowIntegration } from './adapters/metricflow/pull-config.js';
|
||||
import { NotionSourceAdapter } from './adapters/notion/notion.adapter.js';
|
||||
import { seedLocalMappingStateFromKloYaml } from './local-mapping-reconcile.js';
|
||||
import { seedLocalMappingStateFromKtxYaml } from './local-mapping-reconcile.js';
|
||||
import type { SourceAdapter } from './types.js';
|
||||
|
||||
export interface DefaultLocalIngestAdaptersOptions {
|
||||
|
|
@ -43,7 +43,7 @@ export interface DefaultLocalIngestAdaptersOptions {
|
|||
databaseIntrospection?: Omit<DaemonLiveDatabaseIntrospectionOptions, 'connections' | 'baseUrl'>;
|
||||
historicSql?: {
|
||||
sqlAnalysis: SqlAnalysisPort;
|
||||
postgresQueryClient: KloPostgresQueryClient;
|
||||
postgresQueryClient: KtxPostgresQueryClient;
|
||||
postgresBaselineRootDir?: string;
|
||||
now?: () => Date;
|
||||
};
|
||||
|
|
@ -57,7 +57,7 @@ export interface DefaultLocalIngestAdaptersOptions {
|
|||
}
|
||||
|
||||
export function createDefaultLocalIngestAdapters(
|
||||
project: KloLocalProject,
|
||||
project: KtxLocalProject,
|
||||
options: DefaultLocalIngestAdaptersOptions = {},
|
||||
): SourceAdapter[] {
|
||||
const lookerConnectionFactory = new DefaultLookerConnectionClientFactory(
|
||||
|
|
@ -73,8 +73,8 @@ export function createDefaultLocalIngestAdapters(
|
|||
...(options.databaseIntrospectionUrl ? { baseUrl: options.databaseIntrospectionUrl } : {}),
|
||||
}),
|
||||
}),
|
||||
new LookmlSourceAdapter({ homeDir: join(project.projectDir, '.klo/cache') }),
|
||||
new DbtSourceAdapter({ homeDir: join(project.projectDir, '.klo/cache') }),
|
||||
new LookmlSourceAdapter({ homeDir: join(project.projectDir, '.ktx/cache') }),
|
||||
new DbtSourceAdapter({ homeDir: join(project.projectDir, '.ktx/cache') }),
|
||||
createLocalMetabaseSourceAdapter(project),
|
||||
new LookerSourceAdapter({
|
||||
clientFactory: {
|
||||
|
|
@ -86,7 +86,7 @@ export function createDefaultLocalIngestAdapters(
|
|||
},
|
||||
},
|
||||
}),
|
||||
new MetricflowSourceAdapter({ homeDir: join(project.projectDir, '.klo/cache') }),
|
||||
new MetricflowSourceAdapter({ homeDir: join(project.projectDir, '.ktx/cache') }),
|
||||
new NotionSourceAdapter(),
|
||||
];
|
||||
|
||||
|
|
@ -128,7 +128,7 @@ function localLookmlPullConfigFromConnection(connection: Record<string, unknown>
|
|||
repoUrl: stringField(connection?.repoUrl) ?? stringField(connection?.repo_url) ?? null,
|
||||
branch: stringField(connection?.branch),
|
||||
path: stringField(connection?.path),
|
||||
authToken: literalAuthToken ?? resolveKloConfigReference(authTokenRef ?? undefined, env) ?? null,
|
||||
authToken: literalAuthToken ?? resolveKtxConfigReference(authTokenRef ?? undefined, env) ?? null,
|
||||
expectedLookerConnectionName: stringField(mappings.expectedLookerConnectionName),
|
||||
});
|
||||
}
|
||||
|
|
@ -151,7 +151,7 @@ function localDbtPullConfigFromConnection(connection: Record<string, unknown> |
|
|||
}
|
||||
const authToken =
|
||||
stringField(connection?.authToken) ??
|
||||
resolveKloConfigReference(
|
||||
resolveKtxConfigReference(
|
||||
stringField(connection?.auth_token_ref) ?? stringField(connection?.authTokenRef) ?? undefined,
|
||||
env,
|
||||
);
|
||||
|
|
@ -164,14 +164,14 @@ function localDbtPullConfigFromConnection(connection: Record<string, unknown> |
|
|||
}
|
||||
|
||||
export async function localPullConfigForAdapter(
|
||||
project: KloLocalProject,
|
||||
project: KtxLocalProject,
|
||||
adapter: SourceAdapter,
|
||||
connectionId: string,
|
||||
options: DefaultLocalIngestAdaptersOptions = {},
|
||||
): Promise<unknown> {
|
||||
if (adapter.source === 'metabase') {
|
||||
throw new Error(
|
||||
'Metabase scheduled pulls fan out by mapping. Call runLocalMetabaseIngest() or use `klo ingest run --adapter metabase --connection-id <metabase-source-id>` from the CLI.',
|
||||
'Metabase scheduled pulls fan out by mapping. Call runLocalMetabaseIngest() or use `ktx ingest run --adapter metabase --connection-id <metabase-source-id>` from the CLI.',
|
||||
);
|
||||
}
|
||||
const connection = project.config.connections[connectionId];
|
||||
|
|
@ -186,8 +186,8 @@ export async function localPullConfigForAdapter(
|
|||
});
|
||||
}
|
||||
if (adapter.source === 'looker') {
|
||||
await seedLocalMappingStateFromKloYaml(project, connectionId);
|
||||
const store = new LocalLookerRuntimeStore({ dbPath: join(project.projectDir, '.klo', 'db.sqlite') });
|
||||
await seedLocalMappingStateFromKtxYaml(project, connectionId);
|
||||
const store = new LocalLookerRuntimeStore({ dbPath: join(project.projectDir, '.ktx', 'db.sqlite') });
|
||||
const targetConnections = new Map(
|
||||
Object.entries(project.config.connections).flatMap(([id, config]) => {
|
||||
const descriptor = localConnectionToWarehouseDescriptor(id, config);
|
||||
|
|
@ -197,7 +197,7 @@ export async function localPullConfigForAdapter(
|
|||
const parser =
|
||||
options.looker?.parser ??
|
||||
createDaemonLookerTableIdentifierParser({
|
||||
baseUrl: options.looker?.daemonBaseUrl ?? process.env.KLO_DAEMON_URL ?? 'http://127.0.0.1:8765',
|
||||
baseUrl: options.looker?.daemonBaseUrl ?? process.env.KTX_DAEMON_URL ?? 'http://127.0.0.1:8765',
|
||||
});
|
||||
let cleanupClient: Pick<LookerRuntimeClient, 'cleanup'> | null = null;
|
||||
let client: Pick<LookerMappingClient, 'listLookmlModels' | 'getExplore'>;
|
||||
|
|
@ -241,7 +241,7 @@ export async function localPullConfigForAdapter(
|
|||
const authToken =
|
||||
typeof metricflowConfig?.authToken === 'string'
|
||||
? metricflowConfig.authToken
|
||||
: resolveKloConfigReference(
|
||||
: resolveKtxConfigReference(
|
||||
typeof metricflowConfig?.auth_token_ref === 'string' ? metricflowConfig.auth_token_ref : undefined,
|
||||
options.looker?.env ?? process.env,
|
||||
);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { AgentRunnerService } from '../agent/index.js';
|
||||
import { initKloProject, type KloLocalProject, loadKloProject } from '../project/index.js';
|
||||
import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js';
|
||||
import { makeLocalGitRepo } from '../test/make-local-git-repo.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js';
|
||||
|
|
@ -149,14 +149,14 @@ function makeLookerParser() {
|
|||
|
||||
describe('canonical local ingest', () => {
|
||||
let tempDir: string;
|
||||
let project: KloLocalProject;
|
||||
let project: KtxLocalProject;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-local-full-ingest-'));
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-full-ingest-'));
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await initKtxProject({ projectDir, projectName: 'warehouse' });
|
||||
await writeFile(
|
||||
join(projectDir, 'klo.yaml'),
|
||||
join(projectDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
|
|
@ -171,7 +171,7 @@ describe('canonical local ingest', () => {
|
|||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
project = await loadKloProject({ projectDir });
|
||||
project = await loadKtxProject({ projectDir });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
@ -254,9 +254,9 @@ describe('canonical local ingest', () => {
|
|||
|
||||
it('rejects direct Metabase scheduled pulls before requiring a local ingest LLM provider', async () => {
|
||||
const projectDir = join(tempDir, 'metabase-project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await initKtxProject({ projectDir, projectName: 'warehouse' });
|
||||
await writeFile(
|
||||
join(projectDir, 'klo.yaml'),
|
||||
join(projectDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
|
|
@ -271,7 +271,7 @@ describe('canonical local ingest', () => {
|
|||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
const metabaseProject = await loadKloProject({ projectDir });
|
||||
const metabaseProject = await loadKtxProject({ projectDir });
|
||||
|
||||
await expect(
|
||||
runLocalIngest({
|
||||
|
|
@ -286,7 +286,7 @@ describe('canonical local ingest', () => {
|
|||
|
||||
it('runs full MetricFlow local ingest from a dbt repo fixture through the canonical runner', async () => {
|
||||
const projectDir = join(tempDir, 'metricflow-run-project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await initKtxProject({ projectDir, projectName: 'warehouse' });
|
||||
|
||||
const fixtureDir = join(tempDir, 'metricflow-fixture');
|
||||
await mkdir(join(fixtureDir, 'models'), { recursive: true });
|
||||
|
|
@ -332,7 +332,7 @@ describe('canonical local ingest', () => {
|
|||
const repo = await makeLocalGitRepo(fixtureDir, join(tempDir, 'metricflow-origin'));
|
||||
|
||||
await writeFile(
|
||||
join(projectDir, 'klo.yaml'),
|
||||
join(projectDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
|
|
@ -351,13 +351,13 @@ describe('canonical local ingest', () => {
|
|||
' search: sqlite-fts5',
|
||||
' git:',
|
||||
' auto_commit: false',
|
||||
' author: KLO Test <system@klo.local>',
|
||||
' author: KTX Test <system@ktx.local>',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
const metricflowProject = await loadKloProject({ projectDir });
|
||||
const metricflowProject = await loadKtxProject({ projectDir });
|
||||
const agentRunner = new TestAgentRunner();
|
||||
const result = await runLocalIngest({
|
||||
project: metricflowProject,
|
||||
|
|
@ -403,7 +403,7 @@ describe('canonical local ingest', () => {
|
|||
});
|
||||
|
||||
it('local metricflow ingest can fetch from connection metricflow config without sourceDir', async () => {
|
||||
const projectDir = await mkdtemp(join(tmpdir(), 'klo-local-mf-fetch-'));
|
||||
const projectDir = await mkdtemp(join(tmpdir(), 'ktx-local-mf-fetch-'));
|
||||
const fixtureDir = join(projectDir, 'fixture-src');
|
||||
await mkdir(join(fixtureDir, 'models'), { recursive: true });
|
||||
await writeFile(join(fixtureDir, 'dbt_project.yml'), 'name: analytics\n', 'utf-8');
|
||||
|
|
@ -414,7 +414,7 @@ describe('canonical local ingest', () => {
|
|||
);
|
||||
const repo = await makeLocalGitRepo(fixtureDir, join(projectDir, 'origin'));
|
||||
await writeFile(
|
||||
join(projectDir, 'klo.yaml'),
|
||||
join(projectDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: local-mf',
|
||||
'connections:',
|
||||
|
|
@ -428,13 +428,13 @@ describe('canonical local ingest', () => {
|
|||
' search: sqlite-fts5',
|
||||
' git:',
|
||||
' auto_commit: false',
|
||||
' author: KLO Test <system@klo.local>',
|
||||
' author: KTX Test <system@ktx.local>',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
const metricflowProject = await loadKloProject({ projectDir });
|
||||
const metricflowProject = await loadKtxProject({ projectDir });
|
||||
const adapters = createDefaultLocalIngestAdapters(metricflowProject);
|
||||
const metricflow = adapters.find((adapter) => adapter.source === 'metricflow');
|
||||
|
||||
|
|
@ -450,9 +450,9 @@ describe('canonical local ingest', () => {
|
|||
|
||||
it('runs scheduled Looker ingest through the canonical local runner and records SL target evidence', async () => {
|
||||
const projectDir = join(tempDir, 'looker-project');
|
||||
await initKloProject({ projectDir, projectName: 'looker-runtime' });
|
||||
await initKtxProject({ projectDir, projectName: 'looker-runtime' });
|
||||
await writeFile(
|
||||
join(projectDir, 'klo.yaml'),
|
||||
join(projectDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: looker-runtime',
|
||||
'connections:',
|
||||
|
|
@ -473,14 +473,14 @@ describe('canonical local ingest', () => {
|
|||
' search: sqlite-fts5',
|
||||
' git:',
|
||||
' auto_commit: false',
|
||||
' author: KLO Test <system@klo.local>',
|
||||
' author: KTX Test <system@ktx.local>',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
const lookerProject = await loadKloProject({ projectDir });
|
||||
const localStore = new LocalLookerRuntimeStore({ dbPath: join(lookerProject.projectDir, '.klo', 'db.sqlite') });
|
||||
const lookerProject = await loadKtxProject({ projectDir });
|
||||
const localStore = new LocalLookerRuntimeStore({ dbPath: join(lookerProject.projectDir, '.ktx', 'db.sqlite') });
|
||||
await localStore.setCursors('prod-looker', {
|
||||
dashboardsLastSyncedAt: null,
|
||||
looksLastSyncedAt: null,
|
||||
|
|
@ -488,7 +488,7 @@ describe('canonical local ingest', () => {
|
|||
await localStore.upsertConnectionMapping({
|
||||
lookerConnectionId: 'prod-looker',
|
||||
lookerConnectionName: 'analytics',
|
||||
kloConnectionId: 'prod-warehouse',
|
||||
ktxConnectionId: 'prod-warehouse',
|
||||
source: 'cli',
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { AgentRunnerService } from '../agent/index.js';
|
||||
import { initKloProject, type KloLocalProject, loadKloProject } from '../project/index.js';
|
||||
import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js';
|
||||
import { createLocalBundleIngestRuntime } from './local-bundle-runtime.js';
|
||||
|
|
@ -18,14 +18,14 @@ type RuntimeWithConnectionDeps = {
|
|||
|
||||
describe('createLocalBundleIngestRuntime', () => {
|
||||
let tempDir: string;
|
||||
let project: KloLocalProject;
|
||||
let project: KtxLocalProject;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-local-bundle-runtime-'));
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-bundle-runtime-'));
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await initKtxProject({ projectDir, projectName: 'warehouse' });
|
||||
await writeFile(
|
||||
join(projectDir, 'klo.yaml'),
|
||||
join(projectDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
|
|
@ -40,7 +40,7 @@ describe('createLocalBundleIngestRuntime', () => {
|
|||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
project = await loadKloProject({ projectDir });
|
||||
project = await loadKtxProject({ projectDir });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
@ -53,7 +53,7 @@ describe('createLocalBundleIngestRuntime', () => {
|
|||
project,
|
||||
adapters: [new FakeSourceAdapter()],
|
||||
}),
|
||||
).toThrow('klo dev ingest run requires llm.provider.backend: anthropic, vertex, or gateway, or an injected agentRunner');
|
||||
).toThrow('ktx dev ingest run requires llm.provider.backend: anthropic, vertex, or gateway, or an injected agentRunner');
|
||||
});
|
||||
|
||||
it('builds runner deps with local SQLite stores and context tools enabled', async () => {
|
||||
|
|
@ -67,12 +67,12 @@ describe('createLocalBundleIngestRuntime', () => {
|
|||
});
|
||||
|
||||
expect(runtime.nextJobId()).toBe('job-1');
|
||||
expect(runtime.storage.resolvePullDir('job-1')).toBe(join(project.projectDir, '.klo/cache/local-ingest/job-1/pull'));
|
||||
expect(runtime.storage.resolvePullDir('job-1')).toBe(join(project.projectDir, '.ktx/cache/local-ingest/job-1/pull'));
|
||||
expect(runtime.storage.resolveUploadDir('job-1')).toBe(
|
||||
join(project.projectDir, '.klo/cache/local-ingest/job-1/upload'),
|
||||
join(project.projectDir, '.ktx/cache/local-ingest/job-1/upload'),
|
||||
);
|
||||
expect(runtime.storage.resolveTranscriptDir('job-1')).toBe(
|
||||
join(project.projectDir, '.klo/ingest-transcripts/job-1'),
|
||||
join(project.projectDir, '.ktx/ingest-transcripts/job-1'),
|
||||
);
|
||||
|
||||
await mkdir(runtime.storage.resolveUploadDir('job-1'), { recursive: true });
|
||||
|
|
@ -109,7 +109,7 @@ describe('createLocalBundleIngestRuntime', () => {
|
|||
|
||||
it('accepts a debug LLM request file when constructing the default agent runner', async () => {
|
||||
await writeFile(
|
||||
join(project.projectDir, 'klo.yaml'),
|
||||
join(project.projectDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
|
|
@ -131,14 +131,14 @@ describe('createLocalBundleIngestRuntime', () => {
|
|||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
project = await loadKloProject({ projectDir: project.projectDir });
|
||||
project = await loadKtxProject({ projectDir: project.projectDir });
|
||||
|
||||
const runtime = createLocalBundleIngestRuntime({
|
||||
project,
|
||||
adapters: [new FakeSourceAdapter()],
|
||||
llmDebugRequestFile: join(project.projectDir, '.klo', 'llm-debug.jsonl'),
|
||||
llmDebugRequestFile: join(project.projectDir, '.ktx', 'llm-debug.jsonl'),
|
||||
});
|
||||
|
||||
expect(runtime.storage.resolvePullDir('job-1')).toBe(join(project.projectDir, '.klo/cache/local-ingest/job-1/pull'));
|
||||
expect(runtime.storage.resolvePullDir('job-1')).toBe(join(project.projectDir, '.ktx/cache/local-ingest/job-1/pull'));
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,27 +1,27 @@
|
|||
import { mkdirSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import type { KloLlmProvider } from '@klo/llm';
|
||||
import type { KtxLlmProvider } from '@ktx/llm';
|
||||
import YAML from 'yaml';
|
||||
import type { AgentRunnerService } from '../agent/index.js';
|
||||
import { AgentRunnerService as DefaultAgentRunnerService } from '../agent/index.js';
|
||||
import { localConnectionInfoFromConfig } from '../connections/index.js';
|
||||
import type { KloEmbeddingPort, KloLogger } from '../core/index.js';
|
||||
import type { KtxEmbeddingPort, KtxLogger } from '../core/index.js';
|
||||
import { noopLogger, SessionWorktreeService } from '../core/index.js';
|
||||
import type { KloSemanticLayerComputePort } from '../daemon/index.js';
|
||||
import type { KtxSemanticLayerComputePort } from '../daemon/index.js';
|
||||
import {
|
||||
createJsonlKloLlmDebugRequestRecorder,
|
||||
createLocalKloEmbeddingProviderFromConfig,
|
||||
createLocalKloLlmProviderFromConfig,
|
||||
KloIngestEmbeddingPortAdapter,
|
||||
createJsonlKtxLlmDebugRequestRecorder,
|
||||
createLocalKtxEmbeddingProviderFromConfig,
|
||||
createLocalKtxLlmProviderFromConfig,
|
||||
KtxIngestEmbeddingPortAdapter,
|
||||
} from '../llm/index.js';
|
||||
import type { KloLocalProject } from '../project/index.js';
|
||||
import { kloLocalStateDbPath } from '../project/index.js';
|
||||
import type { KtxLocalProject } from '../project/index.js';
|
||||
import { ktxLocalStateDbPath } from '../project/index.js';
|
||||
import { PromptService } from '../prompts/index.js';
|
||||
import { SkillsRegistryService } from '../skills/index.js';
|
||||
import {
|
||||
type KloConnectionInfo,
|
||||
type KloQueryResult,
|
||||
type KtxConnectionInfo,
|
||||
type KtxQueryResult,
|
||||
SemanticLayerService,
|
||||
type SemanticLayerSource,
|
||||
type SlConnectionCatalogPort,
|
||||
|
|
@ -86,20 +86,20 @@ import type { SourceAdapter } from './types.js';
|
|||
|
||||
const promptsDir = fileURLToPath(new URL('../../prompts', import.meta.url));
|
||||
const skillsDir = fileURLToPath(new URL('../../skills', import.meta.url));
|
||||
const LOCAL_AUTHOR = { name: 'KLO Local', email: 'local@klo.local' };
|
||||
const LOCAL_AUTHOR = { name: 'KTX Local', email: 'local@ktx.local' };
|
||||
const LOCAL_SHAPE_WARNING = 'Local ingest validates semantic-layer YAML shape only.';
|
||||
|
||||
export interface CreateLocalBundleIngestRuntimeOptions {
|
||||
project: KloLocalProject;
|
||||
project: KtxLocalProject;
|
||||
adapters: SourceAdapter[];
|
||||
agentRunner?: AgentRunnerService;
|
||||
llmProvider?: KloLlmProvider;
|
||||
llmProvider?: KtxLlmProvider;
|
||||
llmDebugRequestFile?: string;
|
||||
memoryModel?: string;
|
||||
semanticLayerCompute?: KloSemanticLayerComputePort;
|
||||
queryExecutor?: { execute(input: { connectionId: string; sql: string; maxRows?: number }): Promise<KloQueryResult> };
|
||||
semanticLayerCompute?: KtxSemanticLayerComputePort;
|
||||
queryExecutor?: { execute(input: { connectionId: string; sql: string; maxRows?: number }): Promise<KtxQueryResult> };
|
||||
jobIdFactory?: () => string;
|
||||
logger?: KloLogger;
|
||||
logger?: KtxLogger;
|
||||
}
|
||||
|
||||
export interface LocalBundleIngestRuntime {
|
||||
|
|
@ -111,7 +111,7 @@ export interface LocalBundleIngestRuntime {
|
|||
nextJobId(): string;
|
||||
}
|
||||
|
||||
class NoopEmbeddingPort implements KloEmbeddingPort {
|
||||
class NoopEmbeddingPort implements KtxEmbeddingPort {
|
||||
readonly maxBatchSize = 64;
|
||||
|
||||
async computeEmbedding(): Promise<number[]> {
|
||||
|
|
@ -127,20 +127,20 @@ class LocalIngestStorage implements IngestStoragePort {
|
|||
readonly homeDir: string;
|
||||
readonly systemGitAuthor = LOCAL_AUTHOR;
|
||||
|
||||
constructor(private readonly project: KloLocalProject) {
|
||||
this.homeDir = join(project.projectDir, '.klo');
|
||||
constructor(private readonly project: KtxLocalProject) {
|
||||
this.homeDir = join(project.projectDir, '.ktx');
|
||||
}
|
||||
|
||||
resolveUploadDir(uploadId: string): string {
|
||||
return join(this.project.projectDir, '.klo/cache/local-ingest', uploadId, 'upload');
|
||||
return join(this.project.projectDir, '.ktx/cache/local-ingest', uploadId, 'upload');
|
||||
}
|
||||
|
||||
resolvePullDir(jobId: string): string {
|
||||
return join(this.project.projectDir, '.klo/cache/local-ingest', jobId, 'pull');
|
||||
return join(this.project.projectDir, '.ktx/cache/local-ingest', jobId, 'pull');
|
||||
}
|
||||
|
||||
resolveTranscriptDir(jobId: string): string {
|
||||
return join(this.project.projectDir, '.klo/ingest-transcripts', jobId);
|
||||
return join(this.project.projectDir, '.ktx/ingest-transcripts', jobId);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -162,19 +162,19 @@ class LocalAuthorResolver implements GitAuthorResolverPort {
|
|||
|
||||
class LocalConnectionCatalog implements SlConnectionCatalogPort {
|
||||
constructor(
|
||||
private readonly project: KloLocalProject,
|
||||
private readonly project: KtxLocalProject,
|
||||
private readonly queryExecutor?: {
|
||||
execute(input: { connectionId: string; sql: string; maxRows?: number }): Promise<KloQueryResult>;
|
||||
execute(input: { connectionId: string; sql: string; maxRows?: number }): Promise<KtxQueryResult>;
|
||||
},
|
||||
) {}
|
||||
|
||||
async listEnabledConnections(ids: string[]): Promise<KloConnectionInfo[]> {
|
||||
async listEnabledConnections(ids: string[]): Promise<KtxConnectionInfo[]> {
|
||||
return ids
|
||||
.map((id) => localConnectionInfoFromConfig(id, this.project.config.connections[id]))
|
||||
.filter((connection): connection is KloConnectionInfo => connection !== null);
|
||||
.filter((connection): connection is KtxConnectionInfo => connection !== null);
|
||||
}
|
||||
|
||||
async getConnectionById(connectionId: string): Promise<KloConnectionInfo> {
|
||||
async getConnectionById(connectionId: string): Promise<KtxConnectionInfo> {
|
||||
const connection = localConnectionInfoFromConfig(connectionId, this.project.config.connections[connectionId]);
|
||||
if (!connection) {
|
||||
throw new Error(`Connection not found: ${connectionId}`);
|
||||
|
|
@ -182,7 +182,7 @@ class LocalConnectionCatalog implements SlConnectionCatalogPort {
|
|||
return connection;
|
||||
}
|
||||
|
||||
async executeQuery(connectionId: string, sql: string): Promise<KloQueryResult> {
|
||||
async executeQuery(connectionId: string, sql: string): Promise<KtxQueryResult> {
|
||||
if (!this.queryExecutor) {
|
||||
throw new Error('Local ingest has no query executor configured');
|
||||
}
|
||||
|
|
@ -191,7 +191,7 @@ class LocalConnectionCatalog implements SlConnectionCatalogPort {
|
|||
}
|
||||
|
||||
class LocalSlPythonPort implements SlPythonPort {
|
||||
constructor(private readonly compute?: KloSemanticLayerComputePort) {}
|
||||
constructor(private readonly compute?: KtxSemanticLayerComputePort) {}
|
||||
|
||||
async validateSources(input: Parameters<SlPythonPort['validateSources']>[0]) {
|
||||
if (!this.compute) {
|
||||
|
|
@ -271,7 +271,7 @@ function scoreText(text: string, query: string): number {
|
|||
}
|
||||
|
||||
class LocalKnowledgeIndex implements KnowledgeIndexPort {
|
||||
constructor(private readonly project: KloLocalProject) {}
|
||||
constructor(private readonly project: KtxLocalProject) {}
|
||||
|
||||
async upsertPage(): Promise<void> {}
|
||||
|
||||
|
|
@ -363,7 +363,7 @@ class LocalIngestToolsetFactory implements IngestToolsetFactoryPort {
|
|||
private readonly contextTools: BaseTool[];
|
||||
|
||||
constructor(deps: {
|
||||
project: KloLocalProject;
|
||||
project: KtxLocalProject;
|
||||
wikiService: KnowledgeWikiService;
|
||||
knowledgeIndex: KnowledgeIndexPort;
|
||||
knowledgeEvents: KnowledgeEventPort;
|
||||
|
|
@ -373,7 +373,7 @@ class LocalIngestToolsetFactory implements IngestToolsetFactoryPort {
|
|||
slSourcesRepository: SlSourcesIndexPort;
|
||||
connections: SlConnectionCatalogPort;
|
||||
contextStore: SqliteContextEvidenceStore;
|
||||
embedding: KloEmbeddingPort;
|
||||
embedding: KtxEmbeddingPort;
|
||||
}) {
|
||||
const slDeps = {
|
||||
semanticLayerService: deps.semanticLayerService,
|
||||
|
|
@ -443,10 +443,10 @@ function nextLocalJobId(): string {
|
|||
|
||||
function resolveAgentRunner(options: CreateLocalBundleIngestRuntimeOptions): {
|
||||
agentRunner: AgentRunnerService;
|
||||
llmProvider?: KloLlmProvider;
|
||||
llmProvider?: KtxLlmProvider;
|
||||
} {
|
||||
const llmProvider =
|
||||
options.llmProvider ?? createLocalKloLlmProviderFromConfig(options.project.config.llm) ?? undefined;
|
||||
options.llmProvider ?? createLocalKtxLlmProviderFromConfig(options.project.config.llm) ?? undefined;
|
||||
|
||||
if (options.agentRunner) {
|
||||
return { agentRunner: options.agentRunner, ...(llmProvider ? { llmProvider } : {}) };
|
||||
|
|
@ -454,7 +454,7 @@ function resolveAgentRunner(options: CreateLocalBundleIngestRuntimeOptions): {
|
|||
|
||||
if (!llmProvider) {
|
||||
throw new Error(
|
||||
'klo dev ingest run requires llm.provider.backend: anthropic, vertex, or gateway, or an injected agentRunner',
|
||||
'ktx dev ingest run requires llm.provider.backend: anthropic, vertex, or gateway, or an injected agentRunner',
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -463,7 +463,7 @@ function resolveAgentRunner(options: CreateLocalBundleIngestRuntimeOptions): {
|
|||
llmProvider,
|
||||
logger: options.logger ?? noopLogger,
|
||||
...(options.llmDebugRequestFile
|
||||
? { debugRequestRecorder: createJsonlKloLlmDebugRequestRecorder(options.llmDebugRequestFile) }
|
||||
? { debugRequestRecorder: createJsonlKtxLlmDebugRequestRecorder(options.llmDebugRequestFile) }
|
||||
: {}),
|
||||
}),
|
||||
llmProvider,
|
||||
|
|
@ -474,12 +474,12 @@ export function createLocalBundleIngestRuntime(
|
|||
options: CreateLocalBundleIngestRuntimeOptions,
|
||||
): LocalBundleIngestRuntime {
|
||||
const logger = options.logger ?? noopLogger;
|
||||
const dbPath = kloLocalStateDbPath(options.project);
|
||||
mkdirSync(join(options.project.projectDir, '.klo/cache/local-ingest'), { recursive: true });
|
||||
const dbPath = ktxLocalStateDbPath(options.project);
|
||||
mkdirSync(join(options.project.projectDir, '.ktx/cache/local-ingest'), { recursive: true });
|
||||
const store = new SqliteBundleIngestStore({ dbPath });
|
||||
const contextStore = new SqliteContextEvidenceStore({ dbPath });
|
||||
const embeddingProvider = createLocalKloEmbeddingProviderFromConfig(options.project.config.ingest.embeddings);
|
||||
const embedding = embeddingProvider ? new KloIngestEmbeddingPortAdapter(embeddingProvider) : new NoopEmbeddingPort();
|
||||
const embeddingProvider = createLocalKtxEmbeddingProviderFromConfig(options.project.config.ingest.embeddings);
|
||||
const embedding = embeddingProvider ? new KtxIngestEmbeddingPortAdapter(embeddingProvider) : new NoopEmbeddingPort();
|
||||
const connections = new LocalConnectionCatalog(options.project, options.queryExecutor);
|
||||
const rootFileStore = options.project.fileStore;
|
||||
const semanticLayerService = new SemanticLayerService(
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { createLocalKloEmbeddingProviderFromConfig, KloIngestEmbeddingPortAdapter } from '../llm/index.js';
|
||||
import { createLocalKtxEmbeddingProviderFromConfig, KtxIngestEmbeddingPortAdapter } from '../llm/index.js';
|
||||
import { CandidateDedupService } from './context-candidates/candidate-dedup.service.js';
|
||||
import { ContextEvidenceIndexService } from './context-evidence/context-evidence-index.service.js';
|
||||
import { SqliteContextEvidenceStore } from './context-evidence/sqlite-context-evidence-store.js';
|
||||
|
|
@ -14,8 +14,8 @@ describe('local ingest embedding providers with SQLite ingest stores', () => {
|
|||
let stagedDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-local-ingest-embedding-'));
|
||||
dbPath = join(tempDir, '.klo', 'db.sqlite');
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-ingest-embedding-'));
|
||||
dbPath = join(tempDir, '.ktx', 'db.sqlite');
|
||||
stagedDir = join(tempDir, 'staged');
|
||||
await mkdir(join(stagedDir, 'pages', 'revenue'), { recursive: true });
|
||||
await writeFile(
|
||||
|
|
@ -44,7 +44,7 @@ describe('local ingest embedding providers with SQLite ingest stores', () => {
|
|||
});
|
||||
|
||||
function embeddings() {
|
||||
const provider = createLocalKloEmbeddingProviderFromConfig({
|
||||
const provider = createLocalKtxEmbeddingProviderFromConfig({
|
||||
backend: 'deterministic',
|
||||
dimensions: 8,
|
||||
batchSize: 4,
|
||||
|
|
@ -52,7 +52,7 @@ describe('local ingest embedding providers with SQLite ingest stores', () => {
|
|||
if (!provider) {
|
||||
throw new Error('deterministic local embedding provider was not created');
|
||||
}
|
||||
return new KloIngestEmbeddingPortAdapter(provider);
|
||||
return new KtxIngestEmbeddingPortAdapter(provider);
|
||||
}
|
||||
|
||||
it('indexes and searches context evidence using a package-owned local embedding provider', async () => {
|
||||
|
|
|
|||
|
|
@ -1,18 +1,18 @@
|
|||
import { randomUUID } from 'node:crypto';
|
||||
import { cp, mkdir, rm } from 'node:fs/promises';
|
||||
import { isAbsolute, resolve } from 'node:path';
|
||||
import type { KloLlmProvider } from '@klo/llm';
|
||||
import type { KtxLlmProvider } from '@ktx/llm';
|
||||
import type { AgentRunnerService } from '../agent/index.js';
|
||||
import type { KloLogger } from '../core/index.js';
|
||||
import type { KloSemanticLayerComputePort } from '../daemon/index.js';
|
||||
import type { KloLocalProject } from '../project/index.js';
|
||||
import { kloLocalStateDbPath } from '../project/index.js';
|
||||
import type { KloQueryResult } from '../sl/index.js';
|
||||
import type { KtxLogger } from '../core/index.js';
|
||||
import type { KtxSemanticLayerComputePort } from '../daemon/index.js';
|
||||
import type { KtxLocalProject } from '../project/index.js';
|
||||
import { ktxLocalStateDbPath } from '../project/index.js';
|
||||
import type { KtxQueryResult } from '../sl/index.js';
|
||||
import { planMetabaseFanoutChildren } from './adapters/metabase/fanout-planner.js';
|
||||
import { LocalMetabaseSourceStateReader } from './adapters/metabase/local-source-state-store.js';
|
||||
import { localPullConfigForAdapter, type DefaultLocalIngestAdaptersOptions } from './local-adapters.js';
|
||||
import { createLocalBundleIngestRuntime } from './local-bundle-runtime.js';
|
||||
import { seedLocalMappingStateFromKloYaml } from './local-mapping-reconcile.js';
|
||||
import { seedLocalMappingStateFromKtxYaml } from './local-mapping-reconcile.js';
|
||||
import type { MemoryFlowEventSink } from './memory-flow/types.js';
|
||||
import { buildSyncId } from './raw-sources-paths.js';
|
||||
import type { IngestReportBody, IngestReportSnapshot } from './reports.js';
|
||||
|
|
@ -20,7 +20,7 @@ import { SqliteBundleIngestStore } from './sqlite-bundle-ingest-store.js';
|
|||
import type { IngestBundleResult, IngestJobContext, IngestJobPhase, IngestTrigger, SourceAdapter } from './types.js';
|
||||
|
||||
export interface RunLocalIngestOptions {
|
||||
project: KloLocalProject;
|
||||
project: KtxLocalProject;
|
||||
adapters: SourceAdapter[];
|
||||
adapter: string;
|
||||
connectionId: string;
|
||||
|
|
@ -30,12 +30,12 @@ export interface RunLocalIngestOptions {
|
|||
jobId?: string;
|
||||
memoryFlow?: MemoryFlowEventSink;
|
||||
agentRunner?: AgentRunnerService;
|
||||
llmProvider?: KloLlmProvider;
|
||||
llmProvider?: KtxLlmProvider;
|
||||
llmDebugRequestFile?: string;
|
||||
memoryModel?: string;
|
||||
semanticLayerCompute?: KloSemanticLayerComputePort;
|
||||
queryExecutor?: { execute(input: { connectionId: string; sql: string; maxRows?: number }): Promise<KloQueryResult> };
|
||||
logger?: KloLogger;
|
||||
semanticLayerCompute?: KtxSemanticLayerComputePort;
|
||||
queryExecutor?: { execute(input: { connectionId: string; sql: string; maxRows?: number }): Promise<KtxQueryResult> };
|
||||
logger?: KtxLogger;
|
||||
}
|
||||
|
||||
export interface LocalIngestMcpOptions
|
||||
|
|
@ -116,12 +116,12 @@ function safeSegment(kind: string, value: string): string {
|
|||
return value;
|
||||
}
|
||||
|
||||
function assertConfigured(project: KloLocalProject, adapter: string, connectionId: string): void {
|
||||
function assertConfigured(project: KtxLocalProject, adapter: string, connectionId: string): void {
|
||||
if (!project.config.connections[connectionId]) {
|
||||
throw new Error(`Connection "${connectionId}" is not configured in klo.yaml`);
|
||||
throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
|
||||
}
|
||||
if (!project.config.ingest.adapters.includes(adapter)) {
|
||||
throw new Error(`Adapter "${adapter}" is not enabled in klo.yaml`);
|
||||
throw new Error(`Adapter "${adapter}" is not enabled in ktx.yaml`);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -153,7 +153,7 @@ async function copySourceDirToUpload(sourceDir: string, uploadDir: string): Prom
|
|||
}
|
||||
|
||||
async function runScheduledPullJob(options: {
|
||||
project: KloLocalProject;
|
||||
project: KtxLocalProject;
|
||||
adapters: SourceAdapter[];
|
||||
adapter: SourceAdapter;
|
||||
connectionId: string;
|
||||
|
|
@ -162,11 +162,11 @@ async function runScheduledPullJob(options: {
|
|||
jobId?: string;
|
||||
memoryFlow?: MemoryFlowEventSink;
|
||||
agentRunner?: AgentRunnerService;
|
||||
llmProvider?: KloLlmProvider;
|
||||
llmProvider?: KtxLlmProvider;
|
||||
memoryModel?: string;
|
||||
semanticLayerCompute?: KloSemanticLayerComputePort;
|
||||
queryExecutor?: { execute(input: { connectionId: string; sql: string; maxRows?: number }): Promise<KloQueryResult> };
|
||||
logger?: KloLogger;
|
||||
semanticLayerCompute?: KtxSemanticLayerComputePort;
|
||||
queryExecutor?: { execute(input: { connectionId: string; sql: string; maxRows?: number }): Promise<KtxQueryResult> };
|
||||
logger?: KtxLogger;
|
||||
}): Promise<LocalIngestResult> {
|
||||
const runtime = createLocalBundleIngestRuntime(options);
|
||||
const jobId = options.jobId ?? runtime.nextJobId();
|
||||
|
|
@ -269,14 +269,14 @@ function metabaseChildJobId(metabaseDatabaseId: number): string {
|
|||
}
|
||||
|
||||
async function recordLocalMetabaseChildFailure(options: {
|
||||
project: KloLocalProject;
|
||||
project: KtxLocalProject;
|
||||
jobId: string;
|
||||
targetConnectionId: string;
|
||||
metabaseDatabaseId: number;
|
||||
trigger?: IngestTrigger;
|
||||
error: unknown;
|
||||
}): Promise<LocalIngestResult> {
|
||||
const store = new SqliteBundleIngestStore({ dbPath: kloLocalStateDbPath(options.project) });
|
||||
const store = new SqliteBundleIngestStore({ dbPath: ktxLocalStateDbPath(options.project) });
|
||||
const syncId = buildSyncId(new Date(), options.jobId);
|
||||
const diffSummary = { added: 0, modified: 0, deleted: 0, unchanged: 0 };
|
||||
const reason = errorMessage(options.error);
|
||||
|
|
@ -357,14 +357,14 @@ export async function runLocalMetabaseIngest(
|
|||
|
||||
const metabaseConnectionId = safeSegment('metabase connection id', options.metabaseConnectionId);
|
||||
assertConfigured(options.project, 'metabase', metabaseConnectionId);
|
||||
await seedLocalMappingStateFromKloYaml(options.project, metabaseConnectionId);
|
||||
await seedLocalMappingStateFromKtxYaml(options.project, metabaseConnectionId);
|
||||
const adapter = findAdapter(options.adapters, 'metabase');
|
||||
const sourceStateReader = new LocalMetabaseSourceStateReader({ dbPath: kloLocalStateDbPath(options.project) });
|
||||
const sourceStateReader = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(options.project) });
|
||||
|
||||
const unhydrated = await sourceStateReader.getUnhydratedSyncEnabledMappingIds(metabaseConnectionId);
|
||||
if (unhydrated.length > 0) {
|
||||
throw new Error(
|
||||
`Metabase mappings ${unhydrated.join(', ')} are not hydrated; run \`klo connection mapping refresh ${metabaseConnectionId}\` before local Metabase ingest.`,
|
||||
`Metabase mappings ${unhydrated.join(', ')} are not hydrated; run \`ktx connection mapping refresh ${metabaseConnectionId}\` before local Metabase ingest.`,
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -385,7 +385,7 @@ export async function runLocalMetabaseIngest(
|
|||
for (const childPlan of childPlans) {
|
||||
const targetConnectionId = safeSegment('target connection id', childPlan.targetConnectionId);
|
||||
if (!options.project.config.connections[targetConnectionId]) {
|
||||
throw new Error(`Target connection "${targetConnectionId}" is not configured in klo.yaml`);
|
||||
throw new Error(`Target connection "${targetConnectionId}" is not configured in ktx.yaml`);
|
||||
}
|
||||
const childJobId = options.jobIdFactory?.() ?? metabaseChildJobId(childPlan.metabaseDatabaseId);
|
||||
options.progress?.onMetabaseChildStarted?.({
|
||||
|
|
@ -448,12 +448,12 @@ export async function runLocalMetabaseIngest(
|
|||
}
|
||||
|
||||
export async function getLocalIngestStatus(
|
||||
project: KloLocalProject,
|
||||
project: KtxLocalProject,
|
||||
id: string,
|
||||
): Promise<IngestReportSnapshot | null> {
|
||||
return new SqliteBundleIngestStore({ dbPath: kloLocalStateDbPath(project) }).findReportByAnyId(id);
|
||||
return new SqliteBundleIngestStore({ dbPath: ktxLocalStateDbPath(project) }).findReportByAnyId(id);
|
||||
}
|
||||
|
||||
export async function getLatestLocalIngestStatus(project: KloLocalProject): Promise<IngestReportSnapshot | null> {
|
||||
return new SqliteBundleIngestStore({ dbPath: kloLocalStateDbPath(project) }).findLatestReport();
|
||||
export async function getLatestLocalIngestStatus(project: KtxLocalProject): Promise<IngestReportSnapshot | null> {
|
||||
return new SqliteBundleIngestStore({ dbPath: ktxLocalStateDbPath(project) }).findLatestReport();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@ import { mkdtemp, rm } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import { kloLocalStateDbPath, type KloLocalProject } from '../project/index.js';
|
||||
import { ktxLocalStateDbPath, type KtxLocalProject } from '../project/index.js';
|
||||
import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js';
|
||||
import { LocalMetabaseSourceStateReader } from './adapters/metabase/local-source-state-store.js';
|
||||
import { seedLocalMappingStateFromKloYaml } from './local-mapping-reconcile.js';
|
||||
import { seedLocalMappingStateFromKtxYaml } from './local-mapping-reconcile.js';
|
||||
|
||||
describe('local mapping yaml reconciliation bridge', () => {
|
||||
let tempDir: string;
|
||||
|
|
@ -16,15 +16,15 @@ describe('local mapping yaml reconciliation bridge', () => {
|
|||
}
|
||||
});
|
||||
|
||||
function projectWithConnections(connections: KloLocalProject['config']['connections']): KloLocalProject {
|
||||
function projectWithConnections(connections: KtxLocalProject['config']['connections']): KtxLocalProject {
|
||||
return {
|
||||
projectDir: tempDir,
|
||||
config: { connections },
|
||||
} as KloLocalProject;
|
||||
} as KtxLocalProject;
|
||||
}
|
||||
|
||||
it('seeds Metabase local state from klo.yaml mapping intent', async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-metabase-yaml-seed-'));
|
||||
it('seeds Metabase local state from ktx.yaml mapping intent', async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-metabase-yaml-seed-'));
|
||||
const project = projectWithConnections({
|
||||
'prod-metabase': {
|
||||
driver: 'metabase',
|
||||
|
|
@ -33,27 +33,27 @@ describe('local mapping yaml reconciliation bridge', () => {
|
|||
syncEnabled: { '1': true },
|
||||
syncMode: 'ONLY',
|
||||
selections: { collections: [12] },
|
||||
defaultTagNames: ['klo'],
|
||||
defaultTagNames: ['ktx'],
|
||||
},
|
||||
},
|
||||
'prod-warehouse': { driver: 'postgres', url: 'postgresql://readonly@db.test/analytics' },
|
||||
});
|
||||
|
||||
await seedLocalMappingStateFromKloYaml(project, 'prod-metabase');
|
||||
await seedLocalMappingStateFromKtxYaml(project, 'prod-metabase');
|
||||
|
||||
const store = new LocalMetabaseSourceStateReader({ dbPath: kloLocalStateDbPath(project) });
|
||||
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) });
|
||||
await expect(store.listDatabaseMappings('prod-metabase')).resolves.toMatchObject([
|
||||
{ metabaseDatabaseId: 1, targetConnectionId: 'prod-warehouse', syncEnabled: true, source: 'klo.yaml' },
|
||||
{ metabaseDatabaseId: 1, targetConnectionId: 'prod-warehouse', syncEnabled: true, source: 'ktx.yaml' },
|
||||
]);
|
||||
await expect(store.getSourceState('prod-metabase')).resolves.toMatchObject({
|
||||
syncMode: 'ONLY',
|
||||
selections: [{ selectionType: 'collection', metabaseObjectId: 12 }],
|
||||
defaultTagNames: ['klo'],
|
||||
defaultTagNames: ['ktx'],
|
||||
});
|
||||
});
|
||||
|
||||
it('seeds Looker local mappings from klo.yaml mapping intent', async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-looker-yaml-seed-'));
|
||||
it('seeds Looker local mappings from ktx.yaml mapping intent', async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-looker-yaml-seed-'));
|
||||
const project = projectWithConnections({
|
||||
'prod-looker': {
|
||||
driver: 'looker',
|
||||
|
|
@ -62,18 +62,18 @@ describe('local mapping yaml reconciliation bridge', () => {
|
|||
'prod-warehouse': { driver: 'postgres', url: 'postgresql://readonly@db.test/analytics' },
|
||||
});
|
||||
|
||||
await seedLocalMappingStateFromKloYaml(project, 'prod-looker');
|
||||
await seedLocalMappingStateFromKtxYaml(project, 'prod-looker');
|
||||
|
||||
const store = new LocalLookerRuntimeStore({ dbPath: kloLocalStateDbPath(project) });
|
||||
const store = new LocalLookerRuntimeStore({ dbPath: ktxLocalStateDbPath(project) });
|
||||
await expect(store.listConnectionMappings('prod-looker')).resolves.toMatchObject([
|
||||
{ lookerConnectionName: 'analytics', kloConnectionId: 'prod-warehouse', source: 'klo.yaml' },
|
||||
{ lookerConnectionName: 'analytics', ktxConnectionId: 'prod-warehouse', source: 'ktx.yaml' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('does nothing for connections without mapping bootstrap intent', async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-no-yaml-seed-'));
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-no-yaml-seed-'));
|
||||
const project = projectWithConnections({ warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' } });
|
||||
|
||||
await expect(seedLocalMappingStateFromKloYaml(project, 'warehouse')).resolves.toBeUndefined();
|
||||
await expect(seedLocalMappingStateFromKtxYaml(project, 'warehouse')).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import {
|
||||
kloLocalStateDbPath,
|
||||
ktxLocalStateDbPath,
|
||||
parseConnectionMappingBootstrap,
|
||||
type KloLocalProject,
|
||||
type KtxLocalProject,
|
||||
type LookerMappingBootstrap,
|
||||
type MetabaseMappingBootstrap,
|
||||
} from '../project/index.js';
|
||||
|
|
@ -30,10 +30,10 @@ function metabaseMappings(bootstrap: MetabaseMappingBootstrap) {
|
|||
function lookerMappings(bootstrap: LookerMappingBootstrap) {
|
||||
return Object.entries(bootstrap.connectionMappings)
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([lookerConnectionName, kloConnectionId]) => ({ lookerConnectionName, kloConnectionId }));
|
||||
.map(([lookerConnectionName, ktxConnectionId]) => ({ lookerConnectionName, ktxConnectionId }));
|
||||
}
|
||||
|
||||
export async function seedLocalMappingStateFromKloYaml(project: KloLocalProject, connectionId: string): Promise<void> {
|
||||
export async function seedLocalMappingStateFromKtxYaml(project: KtxLocalProject, connectionId: string): Promise<void> {
|
||||
const connection = project.config.connections[connectionId];
|
||||
if (!connection) {
|
||||
return;
|
||||
|
|
@ -44,7 +44,7 @@ export async function seedLocalMappingStateFromKloYaml(project: KloLocalProject,
|
|||
return;
|
||||
}
|
||||
|
||||
const dbPath = kloLocalStateDbPath(project);
|
||||
const dbPath = ktxLocalStateDbPath(project);
|
||||
if (bootstrap.adapter === 'metabase') {
|
||||
await new LocalMetabaseSourceStateReader({ dbPath }).applyYamlBootstrap({
|
||||
connectionId,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { tmpdir } from 'node:os';
|
|||
import { join } from 'node:path';
|
||||
import { AgentRunnerService } from '../agent/index.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { initKloProject, type KloLocalProject } from '../project/index.js';
|
||||
import { initKtxProject, type KtxLocalProject } from '../project/index.js';
|
||||
import { LocalMetabaseSourceStateReader } from './adapters/metabase/local-source-state-store.js';
|
||||
import { getLocalIngestStatus, runLocalMetabaseIngest } from './local-ingest.js';
|
||||
import type { ChunkResult, FetchContext, SourceAdapter } from './types.js';
|
||||
|
|
@ -72,11 +72,11 @@ class ThrowingFetchMetabaseSourceAdapter extends FakeMetabaseSourceAdapter {
|
|||
|
||||
describe('runLocalMetabaseIngest', () => {
|
||||
let tempDir: string;
|
||||
let project: KloLocalProject;
|
||||
let project: KtxLocalProject;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-metabase-fanout-'));
|
||||
project = await initKloProject({ projectDir: tempDir, force: true });
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-metabase-fanout-'));
|
||||
project = await initKtxProject({ projectDir: tempDir, force: true });
|
||||
project.config.connections = {
|
||||
'prod-metabase': {
|
||||
driver: 'metabase',
|
||||
|
|
@ -94,11 +94,11 @@ describe('runLocalMetabaseIngest', () => {
|
|||
});
|
||||
|
||||
async function seedMetabaseState(): Promise<void> {
|
||||
const store = new LocalMetabaseSourceStateReader({ dbPath: join(tempDir, '.klo', 'db.sqlite') });
|
||||
const store = new LocalMetabaseSourceStateReader({ dbPath: join(tempDir, '.ktx', 'db.sqlite') });
|
||||
await store.replaceSourceState({
|
||||
connectionId: 'prod-metabase',
|
||||
syncMode: 'ALL',
|
||||
defaultTagNames: ['klo'],
|
||||
defaultTagNames: ['ktx'],
|
||||
selections: [],
|
||||
mappings: [
|
||||
{
|
||||
|
|
@ -151,7 +151,7 @@ describe('runLocalMetabaseIngest', () => {
|
|||
});
|
||||
|
||||
it('throws before runner work when there are no sync-enabled mapped rows', async () => {
|
||||
const store = new LocalMetabaseSourceStateReader({ dbPath: join(tempDir, '.klo', 'db.sqlite') });
|
||||
const store = new LocalMetabaseSourceStateReader({ dbPath: join(tempDir, '.ktx', 'db.sqlite') });
|
||||
await store.replaceSourceState({
|
||||
connectionId: 'prod-metabase',
|
||||
mappings: [
|
||||
|
|
@ -179,7 +179,7 @@ describe('runLocalMetabaseIngest', () => {
|
|||
});
|
||||
|
||||
it('throws with refresh guidance for unhydrated sync-enabled rows', async () => {
|
||||
const store = new LocalMetabaseSourceStateReader({ dbPath: join(tempDir, '.klo', 'db.sqlite') });
|
||||
const store = new LocalMetabaseSourceStateReader({ dbPath: join(tempDir, '.ktx', 'db.sqlite') });
|
||||
await store.replaceSourceState({
|
||||
connectionId: 'prod-metabase',
|
||||
mappings: [
|
||||
|
|
@ -191,7 +191,7 @@ describe('runLocalMetabaseIngest', () => {
|
|||
metabaseDbName: null,
|
||||
targetConnectionId: 'warehouse_a',
|
||||
syncEnabled: true,
|
||||
source: 'klo.yaml',
|
||||
source: 'ktx.yaml',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
|
@ -203,7 +203,7 @@ describe('runLocalMetabaseIngest', () => {
|
|||
metabaseConnectionId: 'prod-metabase',
|
||||
agentRunner: new TestAgentRunner(),
|
||||
}),
|
||||
).rejects.toThrow('run `klo connection mapping refresh prod-metabase`');
|
||||
).rejects.toThrow('run `ktx connection mapping refresh prod-metabase`');
|
||||
});
|
||||
|
||||
it('seeds yaml-only Metabase mappings before the unhydrated fan-out preflight', async () => {
|
||||
|
|
@ -230,7 +230,7 @@ describe('runLocalMetabaseIngest', () => {
|
|||
adapters: [new FakeMetabaseSourceAdapter()],
|
||||
metabaseConnectionId: 'prod-metabase',
|
||||
}),
|
||||
).rejects.toThrow('run `klo connection mapping refresh prod-metabase`');
|
||||
).rejects.toThrow('run `ktx connection mapping refresh prod-metabase`');
|
||||
});
|
||||
|
||||
it('rejects source-dir uploads through the Metabase fan-out runner', async () => {
|
||||
|
|
@ -266,7 +266,7 @@ describe('runLocalMetabaseIngest', () => {
|
|||
it('captures fetch-time child failures and continues later mappings', async () => {
|
||||
await seedMetabaseState();
|
||||
project.config.connections.warehouse_c = { driver: 'postgres', url: 'postgres://localhost/c' };
|
||||
const store = new LocalMetabaseSourceStateReader({ dbPath: join(tempDir, '.klo', 'db.sqlite') });
|
||||
const store = new LocalMetabaseSourceStateReader({ dbPath: join(tempDir, '.ktx', 'db.sqlite') });
|
||||
await store.upsertDatabaseMapping({
|
||||
connectionId: 'prod-metabase',
|
||||
metabaseDatabaseId: 3,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promise
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { initKloProject, type KloLocalProject, loadKloProject } from '../project/index.js';
|
||||
import { initKtxProject, type KtxLocalProject, loadKtxProject } from '../project/index.js';
|
||||
import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js';
|
||||
import { createDefaultLocalIngestAdapters } from './local-adapters.js';
|
||||
import {
|
||||
|
|
@ -15,7 +15,7 @@ import type { SourceAdapter } from './types.js';
|
|||
|
||||
async function writeWarehouseConfig(projectDir: string): Promise<void> {
|
||||
await writeFile(
|
||||
join(projectDir, 'klo.yaml'),
|
||||
join(projectDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
|
|
@ -32,7 +32,7 @@ async function writeWarehouseConfig(projectDir: string): Promise<void> {
|
|||
|
||||
async function writeLiveDatabaseConfig(projectDir: string): Promise<void> {
|
||||
await writeFile(
|
||||
join(projectDir, 'klo.yaml'),
|
||||
join(projectDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
|
|
@ -84,14 +84,14 @@ function fetchOnlyAdapter(): SourceAdapter {
|
|||
|
||||
describe('local ingest', () => {
|
||||
let tempDir: string;
|
||||
let project: KloLocalProject;
|
||||
let project: KtxLocalProject;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-local-ingest-'));
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-ingest-'));
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKloProject({ projectDir, projectName: 'warehouse' });
|
||||
await initKtxProject({ projectDir, projectName: 'warehouse' });
|
||||
await writeWarehouseConfig(projectDir);
|
||||
project = await loadKloProject({ projectDir });
|
||||
project = await loadKtxProject({ projectDir });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
@ -158,12 +158,12 @@ describe('local ingest', () => {
|
|||
const status = await getLocalStageOnlyIngestStatus(project, 'local-job-1');
|
||||
expect(status).toEqual(result);
|
||||
|
||||
await expect(access(join(project.projectDir, '.klo', 'db.sqlite'))).resolves.toBeUndefined();
|
||||
await expect(access(join(project.projectDir, '.ktx', 'db.sqlite'))).resolves.toBeUndefined();
|
||||
await expect(
|
||||
readFile(join(project.projectDir, '.klo', 'ingest-runs', 'local-job-1.json'), 'utf-8'),
|
||||
readFile(join(project.projectDir, '.ktx', 'ingest-runs', 'local-job-1.json'), 'utf-8'),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
readFile(join(project.projectDir, '.klo', 'ingest-reports', 'local-job-1.json'), 'utf-8'),
|
||||
readFile(join(project.projectDir, '.ktx', 'ingest-reports', 'local-job-1.json'), 'utf-8'),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
|
|
@ -345,12 +345,12 @@ describe('local ingest', () => {
|
|||
const status = await getLocalStageOnlyIngestStatus(project, 'local-job-3');
|
||||
expect(status).toEqual(changed);
|
||||
|
||||
await expect(access(join(project.projectDir, '.klo', 'db.sqlite'))).resolves.toBeUndefined();
|
||||
await expect(access(join(project.projectDir, '.ktx', 'db.sqlite'))).resolves.toBeUndefined();
|
||||
await expect(
|
||||
readFile(join(project.projectDir, '.klo', 'ingest-runs', 'local-job-3.json'), 'utf-8'),
|
||||
readFile(join(project.projectDir, '.ktx', 'ingest-runs', 'local-job-3.json'), 'utf-8'),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
readFile(join(project.projectDir, '.klo', 'ingest-reports', 'local-job-3.json'), 'utf-8'),
|
||||
readFile(join(project.projectDir, '.ktx', 'ingest-reports', 'local-job-3.json'), 'utf-8'),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
|
|
@ -430,7 +430,7 @@ describe('local ingest', () => {
|
|||
|
||||
it('runs fetch-capable adapters without a source directory', async () => {
|
||||
await writeLiveDatabaseConfig(project.projectDir);
|
||||
project = await loadKloProject({ projectDir: project.projectDir });
|
||||
project = await loadKtxProject({ projectDir: project.projectDir });
|
||||
|
||||
const result = await runLocalStageOnlyIngest({
|
||||
project,
|
||||
|
|
@ -470,7 +470,7 @@ describe('local ingest', () => {
|
|||
|
||||
it('supports dry-run planning without writing raw files, status, or commits', async () => {
|
||||
await writeLiveDatabaseConfig(project.projectDir);
|
||||
project = await loadKloProject({ projectDir: project.projectDir });
|
||||
project = await loadKtxProject({ projectDir: project.projectDir });
|
||||
|
||||
const result = await runLocalStageOnlyIngest({
|
||||
project,
|
||||
|
|
@ -516,7 +516,7 @@ describe('local ingest', () => {
|
|||
|
||||
it('uses daemon-backed live-database introspection in default local adapters', async () => {
|
||||
await writeLiveDatabaseConfig(project.projectDir);
|
||||
project = await loadKloProject({ projectDir: project.projectDir });
|
||||
project = await loadKtxProject({ projectDir: project.projectDir });
|
||||
const runJson = vi.fn(async () => ({
|
||||
connection_id: 'warehouse',
|
||||
extracted_at: '2026-04-28T10:00:00+00:00',
|
||||
|
|
@ -562,7 +562,7 @@ describe('local ingest', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('includes upload-capable KLO adapters in default local ingest adapters', () => {
|
||||
it('includes upload-capable KTX adapters in default local ingest adapters', () => {
|
||||
expect(createDefaultLocalIngestAdapters(project).map((adapter) => adapter.source)).toEqual(
|
||||
expect.arrayContaining(['dbt', 'metricflow', 'notion']),
|
||||
);
|
||||
|
|
@ -573,7 +573,7 @@ describe('local ingest', () => {
|
|||
process.env.NOTION_AUTH_TOKEN = 'ntn_local_test_token';
|
||||
try {
|
||||
await writeFile(
|
||||
join(project.projectDir, 'klo.yaml'),
|
||||
join(project.projectDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: warehouse',
|
||||
'connections:',
|
||||
|
|
@ -590,7 +590,7 @@ describe('local ingest', () => {
|
|||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
project = await loadKloProject({ projectDir: project.projectDir });
|
||||
project = await loadKtxProject({ projectDir: project.projectDir });
|
||||
|
||||
const fetch = vi.fn(async (_pullConfig: unknown, stagedDir: string) => {
|
||||
await mkdir(join(stagedDir, 'pages', 'page-1'), { recursive: true });
|
||||
|
|
@ -686,7 +686,7 @@ describe('local ingest', () => {
|
|||
).rejects.toThrow('Local ingest adapter "fake" requires sourceDir because it does not implement fetch().');
|
||||
});
|
||||
|
||||
it('rejects adapters that are not enabled in klo.yaml', async () => {
|
||||
it('rejects adapters that are not enabled in ktx.yaml', async () => {
|
||||
const sourceDir = join(tempDir, 'source');
|
||||
await mkdir(join(sourceDir, 'orders'), { recursive: true });
|
||||
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\n', 'utf-8');
|
||||
|
|
@ -701,6 +701,6 @@ describe('local ingest', () => {
|
|||
jobId: 'local-job-2',
|
||||
now: () => new Date('2026-04-27T12:00:00.000Z'),
|
||||
}),
|
||||
).rejects.toThrow('Adapter "metricflow" is not enabled in klo.yaml');
|
||||
).rejects.toThrow('Adapter "metricflow" is not enabled in ktx.yaml');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { cp, mkdir, readdir, readFile, rm } from 'node:fs/promises';
|
||||
import { isAbsolute, join, relative, resolve, sep } from 'node:path';
|
||||
import type { KloLocalProject } from '../project/index.js';
|
||||
import { kloLocalStateDbPath } from '../project/local-state-db.js';
|
||||
import type { KtxLocalProject } from '../project/index.js';
|
||||
import { ktxLocalStateDbPath } from '../project/local-state-db.js';
|
||||
import { computeDiffSetFromHashes } from './diff-set.service.js';
|
||||
import { localPullConfigForAdapter } from './local-adapters.js';
|
||||
import { sanitizeMemoryFlowError } from './memory-flow/live-buffer.js';
|
||||
|
|
@ -52,7 +52,7 @@ export type LocalIngestReport = LocalIngestRunRecord & {
|
|||
};
|
||||
|
||||
export interface RunLocalStageOnlyIngestOptions {
|
||||
project: KloLocalProject;
|
||||
project: KtxLocalProject;
|
||||
adapters: SourceAdapter[];
|
||||
adapter: string;
|
||||
connectionId: string;
|
||||
|
|
@ -64,8 +64,8 @@ export interface RunLocalStageOnlyIngestOptions {
|
|||
memoryFlow?: MemoryFlowEventSink;
|
||||
}
|
||||
|
||||
const LOCAL_AUTHOR = 'klo';
|
||||
const LOCAL_AUTHOR_EMAIL = 'klo@example.com';
|
||||
const LOCAL_AUTHOR = 'ktx';
|
||||
const LOCAL_AUTHOR_EMAIL = 'ktx@example.com';
|
||||
|
||||
function safeSegment(kind: string, value: string): string {
|
||||
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
|
||||
|
|
@ -143,17 +143,17 @@ function findAdapter(adapters: SourceAdapter[], source: string): SourceAdapter {
|
|||
return adapter;
|
||||
}
|
||||
|
||||
function assertConfigured(project: KloLocalProject, adapter: string, connectionId: string): void {
|
||||
function assertConfigured(project: KtxLocalProject, adapter: string, connectionId: string): void {
|
||||
if (!project.config.connections[connectionId]) {
|
||||
throw new Error(`Connection "${connectionId}" is not configured in klo.yaml`);
|
||||
throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
|
||||
}
|
||||
if (!project.config.ingest.adapters.includes(adapter)) {
|
||||
throw new Error(`Adapter "${adapter}" is not enabled in klo.yaml`);
|
||||
throw new Error(`Adapter "${adapter}" is not enabled in ktx.yaml`);
|
||||
}
|
||||
}
|
||||
|
||||
function createLocalIngestStore(project: KloLocalProject): SqliteLocalIngestStore {
|
||||
return new SqliteLocalIngestStore({ dbPath: kloLocalStateDbPath(project) });
|
||||
function createLocalIngestStore(project: KtxLocalProject): SqliteLocalIngestStore {
|
||||
return new SqliteLocalIngestStore({ dbPath: ktxLocalStateDbPath(project) });
|
||||
}
|
||||
|
||||
function buildLocalJobId(now: Date): string {
|
||||
|
|
@ -189,7 +189,7 @@ function memoryFlowPlannedWorkUnits(
|
|||
}
|
||||
|
||||
async function pruneStaleRawFiles(input: {
|
||||
project: KloLocalProject;
|
||||
project: KtxLocalProject;
|
||||
rawPrefix: string;
|
||||
nextRawPaths: string[];
|
||||
adapter: string;
|
||||
|
|
@ -210,7 +210,7 @@ async function pruneStaleRawFiles(input: {
|
|||
}
|
||||
|
||||
async function prepareLocalStagedDir(
|
||||
project: KloLocalProject,
|
||||
project: KtxLocalProject,
|
||||
adapter: SourceAdapter,
|
||||
stagedDir: string,
|
||||
sourceDir: string | undefined,
|
||||
|
|
@ -263,7 +263,7 @@ async function runLocalStageOnlyIngestInner(options: RunLocalStageOnlyIngestOpti
|
|||
const existingRun = options.dryRun ? null : store.findRunById(runId);
|
||||
assertCompatibleExistingRun(existingRun, runId, adapter.source, connectionId);
|
||||
|
||||
const stagedDir = join(options.project.projectDir, '.klo/cache/local-ingest', runId, 'staged');
|
||||
const stagedDir = join(options.project.projectDir, '.ktx/cache/local-ingest', runId, 'staged');
|
||||
const sourceDir = await prepareLocalStagedDir(options.project, adapter, stagedDir, options.sourceDir, connectionId);
|
||||
|
||||
const detected = await adapter.detect(stagedDir);
|
||||
|
|
@ -404,7 +404,7 @@ async function runLocalStageOnlyIngestInner(options: RunLocalStageOnlyIngestOpti
|
|||
}
|
||||
|
||||
export async function getLocalStageOnlyIngestStatus(
|
||||
project: KloLocalProject,
|
||||
project: KtxLocalProject,
|
||||
runId: string,
|
||||
): Promise<LocalIngestRunRecord | null> {
|
||||
return createLocalIngestStore(project).findRunById(runId);
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ describe('memory-flow acceptance scenarios', () => {
|
|||
it('renders a completed replay with a clear saved-memory completion line', () => {
|
||||
const output = renderScenario(successfulReplayScenario());
|
||||
|
||||
expect(output).toContain('KLO memory flow warehouse/metricflow done');
|
||||
expect(output).toContain('KTX memory flow warehouse/metricflow done');
|
||||
expect(output).toContain('Saved 3 memories from 4 raw files: 2 wiki pages, 1 SL updates.');
|
||||
expect(output).toContain('Commit: abc12345 Run: run-success Report: ingest-report.json');
|
||||
});
|
||||
|
|
@ -48,7 +48,7 @@ describe('memory-flow acceptance scenarios', () => {
|
|||
it('renders no ANSI color codes in the text fallback for terminals without color support', () => {
|
||||
const output = renderScenario(successfulReplayScenario(), 80);
|
||||
|
||||
expect(output).toContain('KLO memory flow warehouse/metricflow done');
|
||||
expect(output).toContain('KTX memory flow warehouse/metricflow done');
|
||||
expect(output).not.toMatch(/\u001b\[[0-9;]*m/);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -96,14 +96,14 @@ function reportSnapshot(): IngestReportSnapshot {
|
|||
toolTranscripts: [
|
||||
{
|
||||
unitKey: 'orders',
|
||||
path: '/tmp/klo/run/wu-transcripts/job-1/orders.jsonl',
|
||||
path: '/tmp/ktx/run/wu-transcripts/job-1/orders.jsonl',
|
||||
toolCallCount: 3,
|
||||
errorCount: 0,
|
||||
toolNames: ['read_raw_span', 'wiki_write', 'sl_write_source'],
|
||||
},
|
||||
{
|
||||
unitKey: 'customers',
|
||||
path: '/tmp/klo/run/wu-transcripts/job-1/customers.jsonl',
|
||||
path: '/tmp/ktx/run/wu-transcripts/job-1/customers.jsonl',
|
||||
toolCallCount: 2,
|
||||
errorCount: 1,
|
||||
toolNames: ['read_raw_span', 'sl_write_source'],
|
||||
|
|
@ -244,14 +244,14 @@ describe('memory-flow event mapping', () => {
|
|||
expect(replay.details.transcripts).toEqual([
|
||||
{
|
||||
unitKey: 'orders',
|
||||
path: '/tmp/klo/run/wu-transcripts/job-1/orders.jsonl',
|
||||
path: '/tmp/ktx/run/wu-transcripts/job-1/orders.jsonl',
|
||||
toolCallCount: 3,
|
||||
errorCount: 0,
|
||||
toolNames: ['read_raw_span', 'wiki_write', 'sl_write_source'],
|
||||
},
|
||||
{
|
||||
unitKey: 'customers',
|
||||
path: '/tmp/klo/run/wu-transcripts/job-1/customers.jsonl',
|
||||
path: '/tmp/ktx/run/wu-transcripts/job-1/customers.jsonl',
|
||||
toolCallCount: 2,
|
||||
errorCount: 1,
|
||||
toolNames: ['read_raw_span', 'sl_write_source'],
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import type { MemoryFlowInteractionState, MemoryFlowViewModel } from './types.js
|
|||
|
||||
function view(): MemoryFlowViewModel {
|
||||
return {
|
||||
title: 'KLO memory flow warehouse/metricflow running',
|
||||
title: 'KTX memory flow warehouse/metricflow running',
|
||||
subtitle: 'Run run-1 Sync sync-1',
|
||||
status: 'running',
|
||||
activeLine: 'active: WorkUnit orders step 2/4',
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue