mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-16 08:25:14 +02:00
Merge origin/main into rename-knowledge-to-wiki
This commit is contained in:
Andrey Avtomonov
commit
5831bc0242
84 changed files with 454 additions and 503 deletions
|
|
@ -57,4 +57,4 @@ Always join through `customer.id`. Do not join on `email`.
|
|||
- **Join key:** Always use `customer.id`, never `email`.
|
||||
- **Timezone:** `created_at` and `last_seen_at` are UTC. Confirm whether a question expects UTC or a local business day before filtering.
|
||||
- **Paying vs. all:** `free` customers must be excluded from paying-customer follow-ups. Use `paying_customer_count`, not `customer_count`.
|
||||
- **plan_tier values:** `free`, `pro`, `enterprise`. Note: `pro_plus` is a legacy alias for `growth` in the account/contract layer (see `orbit-plan-segment-normalization`), but `plan_tier` on this table uses `pro` not `pro_plus`.
|
||||
- **plan_tier values:** `free`, `pro`, `enterprise`. Note: use the canonical plan names from the account/contract layer (see `orbit-plan-segment-normalization`); `plan_tier` on this table uses `pro` rather than `growth`.
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ Sales Ops must complete the handoff **before the first implementation call**. Cu
|
|||
|
||||
| Field | Notes |
|
||||
|---|---|
|
||||
| Current plan | Starter / Growth / Enterprise — use canonical plan name, not legacy aliases |
|
||||
| Current plan | Starter / Growth / Enterprise — use canonical plan name |
|
||||
| Account segment | self_serve / commercial / enterprise (see `orbit-plan-segment-normalization`) |
|
||||
| Contract shape | Term, ARR, any discounts or custom terms |
|
||||
| Renewal contact | Named person on the customer side responsible for renewal |
|
||||
|
|
|
|||
|
|
@ -216,7 +216,7 @@ export function resolveCommandProjectDirOverride(command: CommandWithGlobalOptio
|
|||
function createBaseProgram(info: KtxCliPackageInfo, io: KtxCliIo): Command {
|
||||
return new Command()
|
||||
.name('ktx')
|
||||
.description('Standalone KTX developer CLI')
|
||||
.description('KTX data agent context layer CLI')
|
||||
.option('--project-dir <path>', 'KTX project directory (default: KTX_PROJECT_DIR, nearest ktx.yaml, or cwd)')
|
||||
.option('--debug', 'Enable diagnostic logging to stderr')
|
||||
.version(`${info.name} ${info.version}`, '-v, --version', 'Show CLI version')
|
||||
|
|
|
|||
|
|
@ -121,7 +121,6 @@ function shouldShowSetupEntryMenu(
|
|||
disableHistoricSql?: boolean;
|
||||
historicSqlWindowDays?: number;
|
||||
historicSqlMinExecutions?: number;
|
||||
historicSqlMinCalls?: number;
|
||||
historicSqlServiceAccountPattern?: string[];
|
||||
historicSqlRedactionPattern?: string[];
|
||||
skipDatabases?: boolean;
|
||||
|
|
@ -194,7 +193,6 @@ function shouldShowSetupEntryMenu(
|
|||
'disableHistoricSql',
|
||||
'historicSqlWindowDays',
|
||||
'historicSqlMinExecutions',
|
||||
'historicSqlMinCalls',
|
||||
'skipDatabases',
|
||||
'source',
|
||||
'sourceConnectionId',
|
||||
|
|
@ -283,11 +281,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
|
|||
.option('--disable-historic-sql', 'Disable Historic SQL for the selected database', false)
|
||||
.option('--historic-sql-window-days <number>', 'Historic SQL query-history window', positiveInteger)
|
||||
.option('--historic-sql-min-executions <number>', 'Minimum Historic SQL executions for a template', positiveInteger)
|
||||
.option(
|
||||
'--historic-sql-min-calls <number>',
|
||||
'Alias for --historic-sql-min-executions',
|
||||
positiveInteger,
|
||||
)
|
||||
.option(
|
||||
'--historic-sql-service-account-pattern <pattern>',
|
||||
'Historic SQL service-account regex; repeatable',
|
||||
|
|
@ -379,7 +372,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
|
|||
|
||||
const mode = options.new ? 'new' : options.existing ? 'existing' : 'auto';
|
||||
const resolvedAgentScope = options.global ? 'global' : options.agentScope;
|
||||
const historicSqlMinExecutions = options.historicSqlMinExecutions ?? options.historicSqlMinCalls;
|
||||
await runSetupArgs(context, {
|
||||
command: 'run',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
|
|
@ -410,7 +402,9 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
|
|||
...(options.enableHistoricSql ? { enableHistoricSql: true } : {}),
|
||||
...(options.disableHistoricSql ? { disableHistoricSql: true } : {}),
|
||||
...(options.historicSqlWindowDays !== undefined ? { historicSqlWindowDays: options.historicSqlWindowDays } : {}),
|
||||
...(historicSqlMinExecutions !== undefined ? { historicSqlMinExecutions } : {}),
|
||||
...(options.historicSqlMinExecutions !== undefined
|
||||
? { historicSqlMinExecutions: options.historicSqlMinExecutions }
|
||||
: {}),
|
||||
...(options.historicSqlServiceAccountPattern.length > 0
|
||||
? { historicSqlServiceAccountPatterns: options.historicSqlServiceAccountPattern }
|
||||
: {}),
|
||||
|
|
|
|||
|
|
@ -168,6 +168,15 @@ describe('renderContextBuildView', () => {
|
|||
expect(output).toContain('(0/1 · 1m05s)');
|
||||
});
|
||||
|
||||
it('renders project directory when provided', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
]);
|
||||
|
||||
const output = renderContextBuildView(state, { styled: false, projectDir: '/tmp/project' });
|
||||
expect(output).toContain('Project: /tmp/project');
|
||||
});
|
||||
|
||||
it('renders dynamic separator matching header width', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
|
|
@ -448,6 +457,7 @@ describe('runContextBuild', () => {
|
|||
|
||||
const output = io.stdout();
|
||||
expect(output).toContain('Building KTX context');
|
||||
expect(output).toContain('Project: /tmp/project');
|
||||
expect(output).toContain('Primary sources:');
|
||||
expect(output).toContain('warehouse');
|
||||
expect(output).toContain('Context sources:');
|
||||
|
|
|
|||
|
|
@ -204,6 +204,7 @@ export function renderContextBuildView(
|
|||
'',
|
||||
header,
|
||||
separator,
|
||||
...(options.projectDir ? [` Project: ${options.projectDir}`] : []),
|
||||
...renderTargetGroup('Primary sources', state.primarySources, state.frame, styled, width),
|
||||
...renderTargetGroup('Context sources', state.contextSources, state.frame, styled, width),
|
||||
'',
|
||||
|
|
@ -684,7 +685,7 @@ export async function runContextBuild(
|
|||
}
|
||||
|
||||
if (!repainter) {
|
||||
io.stdout.write(renderContextBuildView(state, { styled: false }));
|
||||
io.stdout.write(renderContextBuildView(state, { ...viewOpts, styled: false }));
|
||||
} else {
|
||||
paint(false);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -123,12 +123,12 @@ describe('runKtxCli', () => {
|
|||
await expect(runKtxCli(['--help'], testIo.io)).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain('Usage: ktx [options] [command]');
|
||||
expect(testIo.stdout()).toContain('KTX data agent context layer CLI');
|
||||
for (const command of ['setup', 'connection', 'ingest', 'wiki', 'sl', 'status', 'scan']) {
|
||||
expect(testIo.stdout()).toContain(`${command}`);
|
||||
}
|
||||
for (const removed of ['demo', 'init', 'connect', 'ask', 'knowledge', 'agent', 'completion', 'serve']) {
|
||||
expect(testIo.stdout()).not.toContain(`${removed} [`);
|
||||
expect(testIo.stdout()).not.toContain(`${removed} `);
|
||||
expect(testIo.stdout()).not.toMatch(new RegExp(`^\\s+${removed}(?:\\s|\\[|$)`, 'm'));
|
||||
}
|
||||
expect(testIo.stdout()).toContain('--project-dir <path>');
|
||||
expect(testIo.stdout()).toContain('KTX_PROJECT_DIR');
|
||||
|
|
|
|||
|
|
@ -376,7 +376,7 @@ const SYNC_MODE_METABASE_CARDS: MetabaseCard[] = [
|
|||
collection_id: 12,
|
||||
archived: false,
|
||||
result_metadata: [],
|
||||
dataset_query: { type: 'native', database: 1, native: { query: 'select 101 as id' } },
|
||||
dataset_query: { type: 'native', database: 1, stages: [{ 'lib/type': 'mbql.stage/native', native: 'select 101 as id' }] },
|
||||
parameters: [],
|
||||
dashboard_count: 0,
|
||||
},
|
||||
|
|
@ -390,7 +390,7 @@ const SYNC_MODE_METABASE_CARDS: MetabaseCard[] = [
|
|||
collection_id: 12,
|
||||
archived: false,
|
||||
result_metadata: [],
|
||||
dataset_query: { type: 'native', database: 1, native: { query: 'select 102 as id' } },
|
||||
dataset_query: { type: 'native', database: 1, stages: [{ 'lib/type': 'mbql.stage/native', native: 'select 102 as id' }] },
|
||||
parameters: [],
|
||||
dashboard_count: 0,
|
||||
},
|
||||
|
|
@ -404,7 +404,7 @@ const SYNC_MODE_METABASE_CARDS: MetabaseCard[] = [
|
|||
collection_id: 13,
|
||||
archived: false,
|
||||
result_metadata: [],
|
||||
dataset_query: { type: 'native', database: 1, native: { query: 'select 103 as id' } },
|
||||
dataset_query: { type: 'native', database: 1, stages: [{ 'lib/type': 'mbql.stage/native', native: 'select 103 as id' }] },
|
||||
parameters: [],
|
||||
dashboard_count: 0,
|
||||
},
|
||||
|
|
@ -454,11 +454,11 @@ function createSyncModeMetabaseClient(): MetabaseRuntimeClient {
|
|||
},
|
||||
getAllCards: async () => SYNC_MODE_METABASE_CARDS.map(metabaseCardSummary),
|
||||
convertMbqlToNative: async () => ({ query: 'select 1' }),
|
||||
getNativeSql: (card) => card.dataset_query?.native?.query ?? null,
|
||||
getNativeSql: (card) => card.dataset_query?.stages?.[0]?.native ?? null,
|
||||
getTemplateTags: () => ({}),
|
||||
getCardSql: async (card) => card.dataset_query?.native?.query ?? null,
|
||||
getCardSql: async (card) => card.dataset_query?.stages?.[0]?.native ?? null,
|
||||
getResolvedSql: async (card) => ({
|
||||
resolvedSql: card.dataset_query?.native?.query ?? `select ${card.id} as id`,
|
||||
resolvedSql: card.dataset_query?.stages?.[0]?.native ?? `select ${card.id} as id`,
|
||||
templateTags: [],
|
||||
resolutionStatus: 'resolved',
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -705,7 +705,6 @@ describe('runKtxIngest', () => {
|
|||
patternPagesWritten: 30,
|
||||
stalePatternPagesMarked: 2,
|
||||
archivedPatternPages: 3,
|
||||
legacyPagesDeleted: 4,
|
||||
},
|
||||
errors: [],
|
||||
warnings: [],
|
||||
|
|
@ -739,7 +738,7 @@ describe('runKtxIngest', () => {
|
|||
|
||||
expect(io.stderr()).toBe('');
|
||||
expect(io.stdout()).toContain('Adapter: historic-sql\n');
|
||||
expect(io.stdout()).toContain('Saved memory: 39 wiki, 57 SL\n');
|
||||
expect(io.stdout()).toContain('Saved memory: 35 wiki, 57 SL\n');
|
||||
});
|
||||
|
||||
it('returns a non-zero code when local ingest reports failed work units', async () => {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import { join } from 'node:path';
|
||||
import {
|
||||
createBigQueryLiveDatabaseIntrospection,
|
||||
isKtxBigQueryConnectionConfig,
|
||||
|
|
@ -298,7 +297,6 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli
|
|||
|
||||
const base = {
|
||||
sqlAnalysis: ktxCliHistoricSqlAnalysis(options),
|
||||
postgresBaselineRootDir: join(project.projectDir, '.ktx/cache/historic-sql'),
|
||||
};
|
||||
|
||||
if (dialect === 'postgres') {
|
||||
|
|
|
|||
|
|
@ -62,10 +62,7 @@ describe('createKtxCliScanConnector', () => {
|
|||
expect(connector.driver).toBe('sqlite');
|
||||
});
|
||||
|
||||
it.each([
|
||||
['maxBytesBilled', ' maxBytesBilled: 123456789', 123456789],
|
||||
['max_bytes_billed', ' max_bytes_billed: "987654321"', '987654321'],
|
||||
])('passes BigQuery %s from standalone config', async (_label, byteCapLine, expectedMaxBytesBilled) => {
|
||||
it('passes BigQuery max_bytes_billed from standalone config', async () => {
|
||||
await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
|
|
@ -76,7 +73,7 @@ describe('createKtxCliScanConnector', () => {
|
|||
' driver: bigquery',
|
||||
' dataset_id: analytics',
|
||||
' readonly: true',
|
||||
byteCapLine,
|
||||
' max_bytes_billed: "987654321"',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
@ -90,7 +87,7 @@ describe('createKtxCliScanConnector', () => {
|
|||
expect(bigQueryMock.constructorInputs).toEqual([
|
||||
expect.objectContaining({
|
||||
connectionId: 'warehouse',
|
||||
maxBytesBilled: expectedMaxBytesBilled,
|
||||
maxBytesBilled: '987654321',
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ const SUPPORTED_DRIVERS = 'sqlite, postgres, mysql, clickhouse, sqlserver, bigqu
|
|||
function bigQueryMaxBytesBilled(
|
||||
connection: KtxLocalProject['config']['connections'][string],
|
||||
): number | string | undefined {
|
||||
const raw = connection.maxBytesBilled ?? connection.max_bytes_billed;
|
||||
const raw = connection.max_bytes_billed;
|
||||
if (typeof raw === 'number') {
|
||||
return Number.isFinite(raw) && raw > 0 ? raw : undefined;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,8 +6,6 @@ import {
|
|||
formatSetupNextStepLines,
|
||||
} from './next-steps.js';
|
||||
|
||||
const command = (...parts: string[]) => parts.join(' ');
|
||||
|
||||
describe('KTX demo next steps', () => {
|
||||
it('uses supported context-build commands before agent usage', () => {
|
||||
expect(KTX_CONTEXT_BUILD_COMMANDS).toEqual([
|
||||
|
|
@ -57,29 +55,6 @@ describe('KTX demo next steps', () => {
|
|||
expect(rendered).not.toContain('Optional MCP:');
|
||||
});
|
||||
|
||||
it('does not advertise removed Commander migration commands', () => {
|
||||
const rendered = formatNextStepLines().join('\n');
|
||||
|
||||
expect(rendered).toContain('ktx status --json');
|
||||
expect(rendered).not.toContain('ktx agent');
|
||||
expect(rendered).toContain('ktx sl list');
|
||||
expect(rendered).toContain('ktx wiki list');
|
||||
|
||||
for (const removed of [
|
||||
command('ktx', 'ask'),
|
||||
command('ktx', 'mcp'),
|
||||
command('ktx', 'connect'),
|
||||
command('ktx', 'knowledge'),
|
||||
command('dev', 'model'),
|
||||
command('dev', 'knowledge'),
|
||||
command('ktx', 'ingest', 'run'),
|
||||
command('ktx', 'ingest', 'replay'),
|
||||
command('ktx', 'serve', '--mcp', 'stdio', '--user-id', 'local'),
|
||||
]) {
|
||||
expect(rendered).not.toContain(removed);
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps setup next steps focused on building context when the build is not ready', () => {
|
||||
const rendered = formatSetupNextStepLines({
|
||||
setupReady: true,
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ function normalizedDriver(connection: KtxProjectConnectionConfig): string {
|
|||
}
|
||||
|
||||
function sourceDirForConnection(connection: KtxProjectConnectionConfig): string | undefined {
|
||||
const value = connection.source_dir ?? connection.sourceDir;
|
||||
const value = connection.source_dir;
|
||||
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -64,8 +64,6 @@ function textInputPrompt(message: string): string {
|
|||
return `${title}\n│\n│ ${bodyLines.join('\n│ ')}\n│ Press Escape to go back.\n│`;
|
||||
}
|
||||
|
||||
const legacyHistoricSqlServiceAccountPatternsKey = ['serviceAccount', 'UserPatterns'].join('');
|
||||
|
||||
describe('setup databases step', () => {
|
||||
let tempDir: string;
|
||||
|
||||
|
|
@ -1255,6 +1253,7 @@ describe('setup databases step', () => {
|
|||
io.io,
|
||||
{
|
||||
testConnection: vi.fn(async () => 0),
|
||||
rebuildNativeSqlite: vi.fn(async () => 1),
|
||||
scanConnection: vi.fn(async (_projectDir: string, _connectionId: string, commandIo: KtxCliIo) => {
|
||||
commandIo.stderr.write(
|
||||
[
|
||||
|
|
@ -1280,6 +1279,60 @@ describe('setup databases step', () => {
|
|||
expect(io.stderr()).not.toMatch(/^Native SQLite is built for a different Node.js ABI\./m);
|
||||
});
|
||||
|
||||
it('rebuilds native SQLite once and retries setup scanning after a Node ABI mismatch', async () => {
|
||||
const io = makeIo();
|
||||
const scanConnection = vi.fn(async (_projectDir: string, _connectionId: string, commandIo: KtxCliIo) => {
|
||||
if (scanConnection.mock.calls.length === 1) {
|
||||
commandIo.stderr.write(
|
||||
[
|
||||
"The module '/workspace/node_modules/better-sqlite3/build/Release/better_sqlite3.node'",
|
||||
'was compiled against a different Node.js version using',
|
||||
'NODE_MODULE_VERSION 147. This version of Node.js requires',
|
||||
'NODE_MODULE_VERSION 137. Please try re-compiling or re-installing',
|
||||
'the module (for instance, using `npm rebuild` or `npm install`).',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
return 1;
|
||||
}
|
||||
|
||||
commandIo.stdout.write('What changed\n');
|
||||
commandIo.stdout.write(' Semantic layer comparison found 0 changes across 56 tables\n');
|
||||
commandIo.stdout.write(' New tables: 0\n');
|
||||
commandIo.stdout.write(' Changed tables: 0\n');
|
||||
commandIo.stdout.write(' Removed tables: 0\n');
|
||||
commandIo.stdout.write(' Unchanged tables: 56\n');
|
||||
return 0;
|
||||
});
|
||||
const rebuildNativeSqlite = vi.fn(async () => 0);
|
||||
|
||||
const result = await runKtxSetupDatabasesStep(
|
||||
{
|
||||
projectDir: tempDir,
|
||||
inputMode: 'disabled',
|
||||
databaseDrivers: ['postgres'],
|
||||
databaseConnectionId: 'warehouse',
|
||||
databaseUrl: 'env:DATABASE_URL',
|
||||
databaseSchemas: [],
|
||||
skipDatabases: false,
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
testConnection: vi.fn(async () => 0),
|
||||
scanConnection,
|
||||
rebuildNativeSqlite,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.status).toBe('ready');
|
||||
expect(rebuildNativeSqlite).toHaveBeenCalledOnce();
|
||||
expect(rebuildNativeSqlite).toHaveBeenCalledWith(expect.anything());
|
||||
expect(scanConnection).toHaveBeenCalledTimes(2);
|
||||
expect(io.stderr()).toContain('Native SQLite is built for a different Node.js ABI.');
|
||||
expect(io.stderr()).toContain('Rebuilding Native SQLite with pnpm run native:rebuild…');
|
||||
expect(io.stdout()).toContain('◇ Scan complete for warehouse');
|
||||
});
|
||||
|
||||
it('writes Historic SQL config for supported Snowflake databases after validation succeeds', async () => {
|
||||
const io = makeIo();
|
||||
const result = await runKtxSetupDatabasesStep(
|
||||
|
|
@ -1325,7 +1378,6 @@ describe('setup databases step', () => {
|
|||
redactionPatterns: ['(?i)secret'],
|
||||
},
|
||||
});
|
||||
expect(config.connections.snowflake.historicSql).not.toHaveProperty(legacyHistoricSqlServiceAccountPatternsKey);
|
||||
expect(config.ingest.adapters).toContain('historic-sql');
|
||||
});
|
||||
|
||||
|
|
@ -1373,10 +1425,8 @@ describe('setup databases step', () => {
|
|||
},
|
||||
},
|
||||
});
|
||||
expect(config.connections.warehouse.historicSql).not.toHaveProperty('minCalls');
|
||||
expect(config.connections.warehouse.historicSql).not.toHaveProperty('windowDays');
|
||||
expect(config.connections.warehouse.historicSql).not.toHaveProperty('redactionPatterns');
|
||||
expect(config.connections.warehouse.historicSql).not.toHaveProperty(legacyHistoricSqlServiceAccountPatternsKey);
|
||||
expect(config.ingest.adapters).toContain('historic-sql');
|
||||
expect(config.ingest.workUnits.maxConcurrency).toBe(6);
|
||||
expect(io.stdout()).toContain('Historic SQL probe...');
|
||||
|
|
@ -1430,7 +1480,6 @@ describe('setup databases step', () => {
|
|||
redactionPatterns: [],
|
||||
},
|
||||
});
|
||||
expect(config.connections.analytics.historicSql).not.toHaveProperty(legacyHistoricSqlServiceAccountPatternsKey);
|
||||
expect(config.ingest.adapters).toContain('historic-sql');
|
||||
});
|
||||
|
||||
|
|
@ -1480,7 +1529,6 @@ describe('setup databases step', () => {
|
|||
},
|
||||
},
|
||||
});
|
||||
expect(config.connections.warehouse.historicSql).not.toHaveProperty(legacyHistoricSqlServiceAccountPatternsKey);
|
||||
});
|
||||
|
||||
it('prints a non-blocking Postgres Historic SQL probe failure after connection test succeeds', async () => {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
import { writeFile } from 'node:fs/promises';
|
||||
import { execFile as execFileCallback } from 'node:child_process';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
import { delimiter, dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { promisify } from 'node:util';
|
||||
import { cancel, confirm, isCancel, multiselect, password, select, text } from '@clack/prompts';
|
||||
import type { HistoricSqlDialect } from '@ktx/context/ingest';
|
||||
import {
|
||||
|
|
@ -17,6 +21,7 @@ import { withSetupInterruptConfirmation } from './setup-interrupt.js';
|
|||
import { writeProjectLocalSecretReference } from './setup-secrets.js';
|
||||
|
||||
const HISTORIC_SQL_WORK_UNIT_MAX_CONCURRENCY = 6;
|
||||
const execFileAsync = promisify(execFileCallback);
|
||||
|
||||
export type KtxSetupDatabaseDriver =
|
||||
| 'sqlite'
|
||||
|
|
@ -39,7 +44,6 @@ export interface KtxSetupDatabasesArgs {
|
|||
disableHistoricSql?: boolean;
|
||||
historicSqlWindowDays?: number;
|
||||
historicSqlMinExecutions?: number;
|
||||
historicSqlMinCalls?: number;
|
||||
historicSqlServiceAccountPatterns?: string[];
|
||||
historicSqlRedactionPatterns?: string[];
|
||||
skipDatabases: boolean;
|
||||
|
|
@ -82,6 +86,7 @@ export interface KtxSetupDatabasesDeps {
|
|||
prompts?: KtxSetupDatabasesPromptAdapter;
|
||||
testConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise<number>;
|
||||
scanConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise<number>;
|
||||
rebuildNativeSqlite?: (io: KtxCliIo) => Promise<number>;
|
||||
listSchemas?: (projectDir: string, connectionId: string) => Promise<string[]>;
|
||||
listTables?: (projectDir: string, connectionId: string) => Promise<KtxTableListEntry[]>;
|
||||
historicSqlProbe?: KtxSetupHistoricSqlProbe;
|
||||
|
|
@ -856,14 +861,13 @@ async function maybeApplyHistoricSqlConfig(input: {
|
|||
dialect,
|
||||
filters: historicSqlFiltersForSetup(input.args.historicSqlServiceAccountPatterns),
|
||||
};
|
||||
delete common[['serviceAccount', 'UserPatterns'].join('')];
|
||||
|
||||
if (dialect === 'postgres') {
|
||||
return {
|
||||
...input.connection,
|
||||
historicSql: {
|
||||
...common,
|
||||
minExecutions: input.args.historicSqlMinExecutions ?? input.args.historicSqlMinCalls ?? 5,
|
||||
minExecutions: input.args.historicSqlMinExecutions ?? 5,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -959,6 +963,81 @@ function writePrefixedLines(write: (chunk: string) => void, output: string): voi
|
|||
}
|
||||
}
|
||||
|
||||
function envWithCurrentNodeFirst(env: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv {
|
||||
return {
|
||||
...env,
|
||||
PATH: `${dirname(process.execPath)}${delimiter}${env.PATH ?? ''}`,
|
||||
};
|
||||
}
|
||||
|
||||
function errorTextProperty(error: unknown, property: 'stderr' | 'stdout'): string {
|
||||
if (typeof error !== 'object' || error === null || !(property in error)) {
|
||||
return '';
|
||||
}
|
||||
const value = (error as Record<typeof property, unknown>)[property];
|
||||
return typeof value === 'string' ? value : '';
|
||||
}
|
||||
|
||||
function commandFailureOutput(error: unknown): string {
|
||||
const stderr = errorTextProperty(error, 'stderr');
|
||||
const stdout = errorTextProperty(error, 'stdout');
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return [stderr.trim(), stdout.trim(), message.trim()].filter((line) => line.length > 0).join('\n');
|
||||
}
|
||||
|
||||
type PackageJsonScriptStatus = 'has-script' | 'exists' | 'missing';
|
||||
|
||||
async function packageJsonScriptStatus(
|
||||
packageJsonPath: string,
|
||||
scriptName: string,
|
||||
): Promise<PackageJsonScriptStatus> {
|
||||
try {
|
||||
const parsed = JSON.parse(await readFile(packageJsonPath, 'utf-8')) as unknown;
|
||||
if (typeof parsed !== 'object' || parsed === null || !('scripts' in parsed)) {
|
||||
return 'exists';
|
||||
}
|
||||
const scripts = (parsed as { scripts?: unknown }).scripts;
|
||||
return typeof scripts === 'object' && scripts !== null && scriptName in scripts ? 'has-script' : 'exists';
|
||||
} catch {
|
||||
return 'missing';
|
||||
}
|
||||
}
|
||||
|
||||
async function nativeSqliteRebuildCommand(): Promise<{ cwd: string; args: string[] }> {
|
||||
let dir = dirname(fileURLToPath(import.meta.url));
|
||||
let packageRoot: string | undefined;
|
||||
while (true) {
|
||||
const status = await packageJsonScriptStatus(join(dir, 'package.json'), 'native:rebuild');
|
||||
if (status === 'has-script') {
|
||||
return { cwd: dir, args: ['run', 'native:rebuild'] };
|
||||
}
|
||||
if (status === 'exists') {
|
||||
packageRoot ??= dir;
|
||||
}
|
||||
|
||||
const parent = dirname(dir);
|
||||
if (parent === dir) {
|
||||
return { cwd: packageRoot ?? process.cwd(), args: ['rebuild', 'better-sqlite3'] };
|
||||
}
|
||||
dir = parent;
|
||||
}
|
||||
}
|
||||
|
||||
async function defaultRebuildNativeSqlite(io: KtxCliIo): Promise<number> {
|
||||
const command = await nativeSqliteRebuildCommand();
|
||||
try {
|
||||
await execFileAsync('pnpm', command.args, {
|
||||
cwd: command.cwd,
|
||||
env: envWithCurrentNodeFirst(),
|
||||
maxBuffer: 1024 * 1024 * 16,
|
||||
});
|
||||
return 0;
|
||||
} catch (error) {
|
||||
writePrefixedLines((chunk) => io.stderr.write(chunk), commandFailureOutput(error));
|
||||
return typeof (error as { code?: unknown })?.code === 'number' ? (error as { code: number }).code : 1;
|
||||
}
|
||||
}
|
||||
|
||||
function flushPrefixedBufferedCommandOutput(io: KtxCliIo, bufferedIo: BufferedCommandIo): void {
|
||||
writePrefixedLines((chunk) => io.stdout.write(chunk), bufferedIo.stdoutText());
|
||||
writePrefixedLines((chunk) => io.stderr.write(chunk), bufferedIo.stderrText());
|
||||
|
|
@ -1472,8 +1551,8 @@ async function validateAndScanConnection(input: {
|
|||
writeSetupSection(input.io, `Scanning ${input.connectionId}`, [
|
||||
'Running structural scan…',
|
||||
]);
|
||||
const scanIo = createBufferedCommandIo();
|
||||
const scanCode = await scanConnection(input.projectDir, input.connectionId, scanIo);
|
||||
let scanIo = createBufferedCommandIo();
|
||||
let scanCode = await scanConnection(input.projectDir, input.connectionId, scanIo);
|
||||
if (scanCode !== 0) {
|
||||
const nativeSqliteDetail = nativeSqliteAbiMismatchDetail(`${scanIo.stderrText()}\n${scanIo.stdoutText()}`);
|
||||
if (nativeSqliteDetail) {
|
||||
|
|
@ -1483,10 +1562,32 @@ async function validateAndScanConnection(input: {
|
|||
`Structural scan failed for ${input.connectionId}.`,
|
||||
'Native SQLite is built for a different Node.js ABI.',
|
||||
`Detail: ${nativeSqliteDetail}`,
|
||||
'Fix: pnpm run native:rebuild',
|
||||
`Retry: ktx scan --project-dir ${input.projectDir} ${input.connectionId}`,
|
||||
'Rebuilding Native SQLite with pnpm run native:rebuild…',
|
||||
].join('\n'),
|
||||
);
|
||||
const rebuildNativeSqlite = input.deps.rebuildNativeSqlite ?? defaultRebuildNativeSqlite;
|
||||
const rebuildCode = await rebuildNativeSqlite(input.io);
|
||||
if (rebuildCode === 0) {
|
||||
writePrefixedLines(
|
||||
(chunk) => input.io.stderr.write(chunk),
|
||||
'Native SQLite rebuild complete. Retrying structural scan…',
|
||||
);
|
||||
const retryScanIo = createBufferedCommandIo();
|
||||
scanCode = await scanConnection(input.projectDir, input.connectionId, retryScanIo);
|
||||
scanIo = retryScanIo;
|
||||
}
|
||||
if (scanCode !== 0) {
|
||||
writePrefixedLines(
|
||||
(chunk) => input.io.stderr.write(chunk),
|
||||
[
|
||||
rebuildCode === 0
|
||||
? `Structural scan still failed for ${input.connectionId} after rebuilding Native SQLite.`
|
||||
: `Native SQLite rebuild failed for ${input.connectionId}.`,
|
||||
'Fix: pnpm run native:rebuild',
|
||||
`Retry: ktx scan --project-dir ${input.projectDir} ${input.connectionId}`,
|
||||
].join('\n'),
|
||||
);
|
||||
}
|
||||
} else {
|
||||
flushPrefixedBufferedCommandOutput(input.io, scanIo);
|
||||
writePrefixedLines(
|
||||
|
|
@ -1497,7 +1598,9 @@ async function validateAndScanConnection(input: {
|
|||
].join('\n'),
|
||||
);
|
||||
}
|
||||
return false;
|
||||
if (scanCode !== 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
const scanOutput = scanIo.stdoutText();
|
||||
const reportPath = readOutputValue(scanOutput, 'Report');
|
||||
|
|
|
|||
|
|
@ -545,8 +545,8 @@ function sourcePathFromFileRepoUrl(repoUrl: string, subpath?: string): string {
|
|||
}
|
||||
|
||||
function repoAuthToken(connection: KtxProjectConnectionConfig | Record<string, unknown>): string | null {
|
||||
const ref = stringField(connection.auth_token_ref) ?? stringField(connection.authTokenRef);
|
||||
const literal = stringField(connection.authToken) ?? stringField(connection.auth_token);
|
||||
const ref = stringField(connection.auth_token_ref);
|
||||
const literal = stringField(connection.auth_token);
|
||||
return literal ?? resolveKtxConfigReference(ref, process.env) ?? null;
|
||||
}
|
||||
|
||||
|
|
@ -564,8 +564,8 @@ async function collectYamlFilesRecursive(sourceRoot: string): Promise<Array<{ co
|
|||
}
|
||||
|
||||
async function defaultValidateDbt(connection: KtxProjectConnectionConfig): Promise<SourceValidationResult> {
|
||||
let sourceDir = stringField(connection.source_dir) ?? stringField(connection.sourceDir);
|
||||
const repoUrl = stringField(connection.repo_url) ?? stringField(connection.repoUrl);
|
||||
let sourceDir = stringField(connection.source_dir);
|
||||
const repoUrl = stringField(connection.repo_url);
|
||||
if (!sourceDir && repoUrl?.startsWith('file:')) {
|
||||
sourceDir = sourcePathFromFileRepoUrl(repoUrl, stringField(connection.path));
|
||||
}
|
||||
|
|
@ -625,7 +625,7 @@ async function defaultValidateLooker(projectDir: string, connectionId: string):
|
|||
}
|
||||
|
||||
async function defaultValidateLookml(connection: KtxProjectConnectionConfig): Promise<SourceValidationResult> {
|
||||
const repoUrl = stringField(connection.repoUrl) ?? stringField(connection.repo_url);
|
||||
const repoUrl = stringField(connection.repoUrl);
|
||||
if (!repoUrl) {
|
||||
return { ok: false, message: 'LookML setup requires repoUrl.' };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -90,7 +90,6 @@ export type KtxSetupArgs =
|
|||
disableHistoricSql?: boolean;
|
||||
historicSqlWindowDays?: number;
|
||||
historicSqlMinExecutions?: number;
|
||||
historicSqlMinCalls?: number;
|
||||
historicSqlServiceAccountPatterns?: string[];
|
||||
historicSqlRedactionPatterns?: string[];
|
||||
skipDatabases: boolean;
|
||||
|
|
@ -636,7 +635,6 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup
|
|||
...(args.historicSqlMinExecutions !== undefined
|
||||
? { historicSqlMinExecutions: args.historicSqlMinExecutions }
|
||||
: {}),
|
||||
...(args.historicSqlMinCalls !== undefined ? { historicSqlMinCalls: args.historicSqlMinCalls } : {}),
|
||||
...(args.historicSqlServiceAccountPatterns
|
||||
? { historicSqlServiceAccountPatterns: args.historicSqlServiceAccountPatterns }
|
||||
: {}),
|
||||
|
|
|
|||
|
|
@ -129,7 +129,7 @@ describe('runKtxSl', () => {
|
|||
connectionId: 'warehouse',
|
||||
name: 'orders',
|
||||
score: expect.any(Number),
|
||||
matchReasons: expect.arrayContaining(['token']),
|
||||
matchReasons: expect.any(Array),
|
||||
}),
|
||||
],
|
||||
},
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue