mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-22 08:38:08 +02:00
Merge branch 'main' into review-workspace-changes
This commit is contained in:
Andrey Avtomonov
GitHub
commit
b30c320408
50 changed files with 745 additions and 922 deletions
|
|
@ -29,8 +29,6 @@ export function registerIngestCommands(
|
|||
.usage('[options] [connectionId]')
|
||||
.argument('[connectionId]', 'Configured connection id to ingest (omit to ingest all)')
|
||||
.option('--all', 'Ingest all configured connections', false)
|
||||
.addOption(new Option('--fast', 'Use deterministic database schema ingest').conflicts('deep'))
|
||||
.addOption(new Option('--deep', 'Use AI-enriched database ingest').conflicts('fast'))
|
||||
.addOption(new Option('--query-history', 'Include database query-history usage patterns').conflicts('noQueryHistory'))
|
||||
.addOption(new Option('--no-query-history', 'Skip database query-history usage patterns'))
|
||||
.option('--query-history-window-days <days>', 'Query-history lookback window for this run', parsePositiveIntegerOption)
|
||||
|
|
@ -87,8 +85,6 @@ export function registerIngestCommands(
|
|||
all: selection.kind === 'all',
|
||||
json: options.json === true,
|
||||
inputMode: options.input === false ? 'disabled' : 'auto',
|
||||
...(options.fast === true ? { depth: 'fast' as const } : {}),
|
||||
...(options.deep === true ? { depth: 'deep' as const } : {}),
|
||||
queryHistory,
|
||||
...(options.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: options.queryHistoryWindowDays } : {}),
|
||||
cliVersion: context.packageInfo.version,
|
||||
|
|
|
|||
|
|
@ -308,9 +308,14 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
|
|||
.addOption(new Option('--source-git-url <url>', 'Git URL for dbt, MetricFlow, or LookML').hideHelp())
|
||||
.addOption(new Option('--source-branch <branch>', 'Git branch for source setup').hideHelp())
|
||||
.addOption(new Option('--source-subpath <path>', 'Repo subpath for source setup').hideHelp())
|
||||
.addOption(new Option('--source-auth-token-ref <ref>', 'env: or file: credential ref for source repo auth').hideHelp())
|
||||
.addOption(
|
||||
new Option(
|
||||
'--source-auth-token-ref <ref>',
|
||||
'env: or file: credential ref for source repo auth or Notion integration token',
|
||||
).hideHelp(),
|
||||
)
|
||||
.addOption(new Option('--source-url <url>', 'Source service URL for Metabase or Looker').hideHelp())
|
||||
.addOption(new Option('--source-api-key-ref <ref>', 'env: or file: API key ref for Metabase or Notion').hideHelp())
|
||||
.addOption(new Option('--source-api-key-ref <ref>', 'env: or file: API key ref for Metabase').hideHelp())
|
||||
.addOption(new Option('--source-client-id <id>', 'Looker client id').hideHelp())
|
||||
.addOption(new Option('--source-client-secret-ref <ref>', 'env: or file: Looker client secret ref').hideHelp())
|
||||
.addOption(new Option('--source-warehouse-connection-id <id>', 'Mapped warehouse connection id').hideHelp())
|
||||
|
|
|
|||
21
packages/cli/src/connection-drivers.ts
Normal file
21
packages/cli/src/connection-drivers.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import type { KtxProjectConnectionConfig } from './context/project/config.js';
|
||||
|
||||
const KTX_DATABASE_DRIVER_IDS = new Set([
|
||||
'sqlite',
|
||||
'postgres',
|
||||
'mysql',
|
||||
'clickhouse',
|
||||
'sqlserver',
|
||||
'bigquery',
|
||||
'snowflake',
|
||||
]);
|
||||
|
||||
export function normalizeConnectionDriver(connection: KtxProjectConnectionConfig): string {
|
||||
return String(connection.driver ?? '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
export function isDatabaseDriver(driver: string): boolean {
|
||||
return KTX_DATABASE_DRIVER_IDS.has(driver.trim().toLowerCase());
|
||||
}
|
||||
|
|
@ -88,7 +88,6 @@ export interface ContextBuildArgs {
|
|||
targetConnectionId?: string;
|
||||
all?: boolean;
|
||||
entrypoint?: 'setup' | 'ingest';
|
||||
depth?: Extract<KtxPublicIngestArgs, { command: 'run' }>['depth'];
|
||||
queryHistory?: Extract<KtxPublicIngestArgs, { command: 'run' }>['queryHistory'];
|
||||
queryHistoryWindowDays?: number;
|
||||
scanMode?: Extract<KtxPublicIngestArgs, { command: 'run' }>['scanMode'];
|
||||
|
|
@ -371,19 +370,17 @@ function retryCommand(input: {
|
|||
projectDir?: string;
|
||||
entrypoint?: 'setup' | 'ingest';
|
||||
connectionId?: string;
|
||||
depth?: 'fast' | 'deep';
|
||||
queryHistory?: boolean;
|
||||
queryHistoryWindowDays?: number;
|
||||
}): string {
|
||||
const projectPart = input.projectDir ? ` --project-dir ${input.projectDir}` : '';
|
||||
if (input.entrypoint === 'ingest' && input.connectionId) {
|
||||
const depthPart = input.depth ? ` --${input.depth}` : '';
|
||||
const queryHistoryPart = input.queryHistory ? ' --query-history' : '';
|
||||
const windowPart =
|
||||
input.queryHistory && input.queryHistoryWindowDays !== undefined
|
||||
? ` --query-history-window-days ${input.queryHistoryWindowDays}`
|
||||
: '';
|
||||
return `ktx ingest ${input.connectionId}${projectPart}${depthPart}${queryHistoryPart}${windowPart}`;
|
||||
return `ktx ingest ${input.connectionId}${projectPart}${queryHistoryPart}${windowPart}`;
|
||||
}
|
||||
return input.projectDir ? `ktx setup --project-dir ${input.projectDir}` : 'ktx setup';
|
||||
}
|
||||
|
|
@ -746,7 +743,6 @@ function appendRetryIfNeeded(input: {
|
|||
projectDir: input.projectDir,
|
||||
entrypoint: input.entrypoint,
|
||||
connectionId: input.target.connectionId,
|
||||
depth: input.target.databaseDepth,
|
||||
queryHistory: input.target.queryHistory?.enabled === true,
|
||||
queryHistoryWindowDays: input.target.queryHistory?.windowDays,
|
||||
})}`;
|
||||
|
|
@ -769,7 +765,6 @@ function failureTextForTarget(input: {
|
|||
projectDir: input.projectDir,
|
||||
entrypoint: input.entrypoint,
|
||||
connectionId: input.target.connectionId,
|
||||
depth: input.target.databaseDepth,
|
||||
queryHistory: input.target.queryHistory?.enabled === true,
|
||||
queryHistoryWindowDays: input.target.queryHistory?.windowDays,
|
||||
})}`,
|
||||
|
|
@ -784,7 +779,6 @@ function failureTextForTarget(input: {
|
|||
projectDir: input.projectDir,
|
||||
entrypoint: input.entrypoint,
|
||||
connectionId: input.target.connectionId,
|
||||
depth: input.target.databaseDepth,
|
||||
queryHistory: input.target.queryHistory?.enabled === true,
|
||||
queryHistoryWindowDays: input.target.queryHistory?.windowDays,
|
||||
})}`,
|
||||
|
|
@ -868,7 +862,6 @@ export async function runContextBuild(
|
|||
projectDir: args.projectDir,
|
||||
...(args.targetConnectionId ? { targetConnectionId: args.targetConnectionId } : {}),
|
||||
all: args.all ?? true,
|
||||
...(args.depth ? { depth: args.depth } : {}),
|
||||
...(args.queryHistory ? { queryHistory: args.queryHistory } : {}),
|
||||
...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
|
||||
...(args.scanMode ? { scanMode: args.scanMode } : {}),
|
||||
|
|
@ -935,7 +928,6 @@ export async function runContextBuild(
|
|||
all: args.all ?? true,
|
||||
json: false,
|
||||
inputMode: args.inputMode,
|
||||
...(args.depth ? { depth: args.depth } : {}),
|
||||
...(args.queryHistory ? { queryHistory: args.queryHistory } : {}),
|
||||
...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
|
||||
...(args.scanMode ? { scanMode: args.scanMode } : {}),
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import { localPullConfigForAdapter, type DefaultLocalIngestAdaptersOptions } fro
|
|||
import { createLocalBundleIngestRuntime } from './local-bundle-runtime.js';
|
||||
import type { MemoryFlowEventSink } from './memory-flow/types.js';
|
||||
import { buildSyncId } from './raw-sources-paths.js';
|
||||
import { ingestReportOutcome } from './reports.js';
|
||||
import type { IngestReportBody, IngestReportSnapshot } from './reports.js';
|
||||
import { SqliteBundleIngestStore } from './sqlite-bundle-ingest-store.js';
|
||||
import type { IngestBundleResult, IngestJobContext, IngestJobPhase, IngestTrigger, SourceAdapter } from './types.js';
|
||||
|
|
@ -79,7 +80,7 @@ export interface LocalMetabaseFanoutProgress {
|
|||
metabaseDatabaseId: number;
|
||||
targetConnectionId: string;
|
||||
jobId: string;
|
||||
status: 'done' | 'failed';
|
||||
status: 'done' | 'partial' | 'failed';
|
||||
}): void;
|
||||
}
|
||||
|
||||
|
|
@ -232,11 +233,11 @@ export async function runLocalIngest(options: RunLocalIngestOptions): Promise<Lo
|
|||
}
|
||||
|
||||
function metabaseFanoutStatus(children: LocalMetabaseFanoutChild[]): LocalMetabaseFanoutResult['status'] {
|
||||
const succeeded = children.filter((child) => child.report.body.failedWorkUnits.length === 0).length;
|
||||
if (succeeded === children.length) {
|
||||
const outcomes = children.map((child) => ingestReportOutcome(child.report));
|
||||
if (outcomes.every((outcome) => outcome === 'done')) {
|
||||
return 'all_succeeded';
|
||||
}
|
||||
if (succeeded === 0) {
|
||||
if (outcomes.every((outcome) => outcome === 'error')) {
|
||||
return 'all_failed';
|
||||
}
|
||||
return 'partial_failure';
|
||||
|
|
@ -401,12 +402,13 @@ export async function runLocalMetabaseIngest(
|
|||
error,
|
||||
});
|
||||
}
|
||||
const childOutcome = ingestReportOutcome(child.report);
|
||||
options.progress?.onMetabaseChildCompleted?.({
|
||||
metabaseConnectionId,
|
||||
metabaseDatabaseId: childPlan.metabaseDatabaseId,
|
||||
targetConnectionId,
|
||||
jobId: child.report.jobId,
|
||||
status: child.report.body.failedWorkUnits.length > 0 ? 'failed' : 'done',
|
||||
status: childOutcome === 'error' ? 'failed' : childOutcome,
|
||||
});
|
||||
children.push({
|
||||
jobId: child.report.jobId,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import type { MemoryAction } from '../../../context/memory/types.js';
|
||||
import type { LocalIngestRunRecord } from '../local-stage-ingest.js';
|
||||
import { ingestReportOutcome } from '../reports.js';
|
||||
import type { IngestReportSnapshot } from '../reports.js';
|
||||
import type {
|
||||
MemoryFlowActionDetail,
|
||||
|
|
@ -72,7 +73,7 @@ function fullModeMetadata(input: {
|
|||
}
|
||||
|
||||
function reportStatus(report: IngestReportSnapshot): MemoryFlowReplayInput['status'] {
|
||||
return report.body.failedWorkUnits.length > 0 ? 'error' : 'done';
|
||||
return ingestReportOutcome(report) === 'error' ? 'error' : 'done';
|
||||
}
|
||||
|
||||
function reportCreatedEvent(report: IngestReportSnapshot): MemoryFlowEvent {
|
||||
|
|
|
|||
|
|
@ -146,6 +146,20 @@ export function savedMemoryCountsForReport(report: IngestReportSnapshot): Ingest
|
|||
};
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export type IngestReportOutcome = 'done' | 'partial' | 'error';
|
||||
|
||||
export function ingestReportOutcome(report: IngestReportSnapshot): IngestReportOutcome {
|
||||
if (report.body.status === 'failed') {
|
||||
return 'error';
|
||||
}
|
||||
if (report.body.failedWorkUnits.length === 0) {
|
||||
return 'done';
|
||||
}
|
||||
const { wikiCount, slCount } = savedMemoryCountsForReport(report);
|
||||
return wikiCount + slCount > 0 ? 'partial' : 'error';
|
||||
}
|
||||
|
||||
export function buildStageIndexFromReportBody(jobId: string, connectionId: string, body: IngestReportBody): StageIndex {
|
||||
return {
|
||||
jobId,
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ function warehouseConnectionSchema<const Driver extends WarehouseDriver>(driver:
|
|||
.array(z.string().min(1))
|
||||
.optional()
|
||||
.describe(
|
||||
'Optional allowlist of fully-qualified table names ("schema.table") to ingest. When set, live-database ingest discards any table whose schema-qualified name is not in this list. Useful for smoke-testing deep ingest on a single table.',
|
||||
'Optional allowlist of fully-qualified table names ("schema.table") to ingest. When set, live-database ingest discards any table whose schema-qualified name is not in this list. Useful for smoke-testing ingest on a single table.',
|
||||
),
|
||||
})
|
||||
.describe(
|
||||
|
|
|
|||
|
|
@ -1,75 +0,0 @@
|
|||
import type { KtxProjectConfig, KtxProjectConnectionConfig } from './context/project/config.js';
|
||||
|
||||
export type KtxDatabaseContextDepth = 'fast' | 'deep';
|
||||
|
||||
const KTX_DATABASE_DRIVER_IDS = new Set([
|
||||
'sqlite',
|
||||
'postgres',
|
||||
'mysql',
|
||||
'clickhouse',
|
||||
'sqlserver',
|
||||
'bigquery',
|
||||
'snowflake',
|
||||
]);
|
||||
|
||||
export function normalizeConnectionDriver(connection: KtxProjectConnectionConfig): string {
|
||||
return String(connection.driver ?? '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
export function isDatabaseDriver(driver: string): boolean {
|
||||
return KTX_DATABASE_DRIVER_IDS.has(driver.trim().toLowerCase());
|
||||
}
|
||||
|
||||
function connectionContextRecord(connection: KtxProjectConnectionConfig): Record<string, unknown> {
|
||||
const context = connection.context;
|
||||
return typeof context === 'object' && context !== null && !Array.isArray(context)
|
||||
? (context as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
export function databaseContextDepth(connection: KtxProjectConnectionConfig): KtxDatabaseContextDepth | undefined {
|
||||
const depth = connectionContextRecord(connection).depth;
|
||||
return depth === 'fast' || depth === 'deep' ? depth : undefined;
|
||||
}
|
||||
|
||||
export function withDatabaseContextDepth(
|
||||
connection: KtxProjectConnectionConfig,
|
||||
depth: KtxDatabaseContextDepth,
|
||||
): KtxProjectConnectionConfig {
|
||||
return {
|
||||
...connection,
|
||||
context: {
|
||||
...connectionContextRecord(connection),
|
||||
depth,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function deepReadinessGaps(config: KtxProjectConfig): string[] {
|
||||
const gaps: string[] = [];
|
||||
if (config.llm.provider.backend === 'none' || !config.llm.models.default) {
|
||||
gaps.push('model configuration');
|
||||
}
|
||||
|
||||
if (config.scan.enrichment.mode !== 'llm') {
|
||||
gaps.push('scan enrichment mode');
|
||||
}
|
||||
|
||||
const embeddings = config.scan.enrichment.embeddings;
|
||||
if (
|
||||
!embeddings ||
|
||||
embeddings.backend === 'none' ||
|
||||
!embeddings.model ||
|
||||
embeddings.dimensions <= 0
|
||||
) {
|
||||
gaps.push('scan embeddings');
|
||||
}
|
||||
|
||||
return gaps;
|
||||
}
|
||||
|
||||
export function recommendedDatabaseContextDepth(config: KtxProjectConfig): KtxDatabaseContextDepth {
|
||||
return deepReadinessGaps(config).length === 0 ? 'deep' : 'fast';
|
||||
}
|
||||
|
|
@ -2,7 +2,7 @@ import { buildMemoryFlowViewModel } from './context/ingest/memory-flow/view-mode
|
|||
import { createMemoryFlowLiveBuffer, sanitizeMemoryFlowError } from './context/ingest/memory-flow/live-buffer.js';
|
||||
import { formatMemoryFlowFinalSummary } from './context/ingest/memory-flow/summary.js';
|
||||
import { getLatestLocalIngestStatus, getLocalIngestStatus, type LocalMetabaseFanoutResult, type LocalMetabaseFanoutProgress, type RunLocalIngestOptions, runLocalIngest, runLocalMetabaseIngest } from './context/ingest/local-ingest.js';
|
||||
import { type IngestReportSnapshot, savedMemoryCountsForReport } from './context/ingest/reports.js';
|
||||
import { type IngestReportSnapshot, ingestReportOutcome, savedMemoryCountsForReport } from './context/ingest/reports.js';
|
||||
import { ingestReportToMemoryFlowReplay } from './context/ingest/memory-flow/events.js';
|
||||
import type { MemoryFlowEvent, MemoryFlowReplayInput } from './context/ingest/memory-flow/types.js';
|
||||
import { renderMemoryFlowReplay } from './context/ingest/memory-flow/render.js';
|
||||
|
|
@ -93,10 +93,6 @@ export interface KtxIngestDeps {
|
|||
runtimeIo?: KtxIngestIo;
|
||||
}
|
||||
|
||||
function reportStatus(report: IngestReportSnapshot): 'done' | 'error' {
|
||||
return report.body.status === 'failed' || report.body.failedWorkUnits.length > 0 ? 'error' : 'done';
|
||||
}
|
||||
|
||||
const REPORT_SOURCE_LABELS = new Map<string, string>([
|
||||
['live-database', 'Database schema'],
|
||||
['historic-sql', 'Query history'],
|
||||
|
|
@ -193,7 +189,7 @@ function writeReportStatus(report: IngestReportSnapshot, io: KtxIngestIo): void
|
|||
if (report.body.tracePath) {
|
||||
io.stdout.write(`Trace: ${report.body.tracePath}\n`);
|
||||
}
|
||||
io.stdout.write(`Status: ${reportStatus(report)}\n`);
|
||||
io.stdout.write(`Status: ${ingestReportOutcome(report)}\n`);
|
||||
io.stdout.write(`Source: ${reportSourceLabel(report.sourceKey)}\n`);
|
||||
io.stdout.write(`Connection: ${report.connectionId}\n`);
|
||||
io.stdout.write(`Sync: ${report.body.syncId}\n`);
|
||||
|
|
@ -231,7 +227,7 @@ function writeMetabaseFanoutStatus(result: LocalMetabaseFanoutResult, io: KtxIng
|
|||
}
|
||||
io.stdout.write(`Saved memory: ${counts.wikiCount} wiki, ${counts.slCount} SL\n`);
|
||||
for (const child of result.children) {
|
||||
const status = reportStatus(child.report);
|
||||
const status = ingestReportOutcome(child.report);
|
||||
io.stdout.write(
|
||||
`- target=${child.targetConnectionId} database=${child.metabaseDatabaseId} status=${status} job=${child.jobId} report=${child.report.id}\n`,
|
||||
);
|
||||
|
|
@ -595,7 +591,7 @@ function initialRunMemoryFlowInput(
|
|||
}
|
||||
|
||||
function finalRunMemoryFlowInput(snapshot: MemoryFlowReplayInput, report: IngestReportSnapshot): MemoryFlowReplayInput {
|
||||
const status = reportStatus(report);
|
||||
const status = ingestReportOutcome(report) === 'error' ? 'error' : 'done';
|
||||
return {
|
||||
...snapshot,
|
||||
runId: report.runId,
|
||||
|
|
@ -777,7 +773,7 @@ export async function runKtxIngest(
|
|||
} finally {
|
||||
plainProgress?.flush();
|
||||
}
|
||||
return result.status === 'all_succeeded' ? 0 : 1;
|
||||
return result.status === 'all_failed' ? 1 : 0;
|
||||
}
|
||||
|
||||
const jobId = deps.jobIdFactory?.();
|
||||
|
|
@ -846,7 +842,7 @@ export async function runKtxIngest(
|
|||
liveTui?.close();
|
||||
liveTui = null;
|
||||
io.stdout.write(formatMemoryFlowFinalSummary(latestMemoryFlowSnapshot));
|
||||
return reportStatus(result.report) === 'done' ? 0 : 1;
|
||||
return ingestReportOutcome(result.report) === 'error' ? 1 : 0;
|
||||
}
|
||||
plainProgress?.flush();
|
||||
await writeReportRecord(result.report, runOutputMode, io, {
|
||||
|
|
@ -854,7 +850,7 @@ export async function runKtxIngest(
|
|||
renderStoredMemoryFlow: deps.renderStoredMemoryFlow,
|
||||
env,
|
||||
});
|
||||
return reportStatus(result.report) === 'done' ? 0 : 1;
|
||||
return ingestReportOutcome(result.report) === 'error' ? 1 : 0;
|
||||
} finally {
|
||||
plainProgress?.flush();
|
||||
liveTui?.close();
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ const DATABASE_INGEST_REPLACEMENTS: Array<[RegExp, string]> = [
|
|||
'Database enrichment failed after schema context completed',
|
||||
],
|
||||
[/\bstructural scan\b/gi, 'schema context'],
|
||||
[/\benriched scan\b/gi, 'deep database ingest'],
|
||||
[/\benriched scan\b/gi, 'database ingest'],
|
||||
[/\bscan results\b/gi, 'database context'],
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -1,16 +1,10 @@
|
|||
import { getKtxCliPackageInfo } from './cli-runtime.js';
|
||||
import { loadKtxProject, type KtxLocalProject } from './context/project/project.js';
|
||||
import type { KtxProjectConnectionConfig } from './context/project/config.js';
|
||||
import type { KtxProjectConfig, KtxProjectConnectionConfig } from './context/project/config.js';
|
||||
import type { KtxProgressPort } from './context/scan/types.js';
|
||||
import type { KtxCliIo } from './index.js';
|
||||
import type { KtxIngestArgs, KtxIngestDeps, KtxIngestProgressUpdate } from './ingest.js';
|
||||
import {
|
||||
type KtxDatabaseContextDepth,
|
||||
databaseContextDepth,
|
||||
deepReadinessGaps,
|
||||
isDatabaseDriver,
|
||||
normalizeConnectionDriver,
|
||||
} from './ingest-depth.js';
|
||||
import { isDatabaseDriver, normalizeConnectionDriver } from './connection-drivers.js';
|
||||
import {
|
||||
ensureManagedPythonCommandRuntime,
|
||||
type KtxManagedPythonInstallPolicy,
|
||||
|
|
@ -29,7 +23,6 @@ profileMark('module:public-ingest');
|
|||
type KtxPublicIngestStepName = 'database-schema' | 'query-history' | 'source-ingest' | 'memory-update';
|
||||
type KtxPublicIngestStepStatus = 'done' | 'skipped' | 'failed' | 'not-run';
|
||||
type KtxPublicIngestInputMode = 'auto' | 'disabled';
|
||||
type KtxPublicIngestDepth = KtxDatabaseContextDepth;
|
||||
type KtxPublicIngestQueryHistoryFlag = 'default' | 'enabled' | 'disabled';
|
||||
type HistoricSqlDialect = 'postgres' | 'bigquery' | 'snowflake';
|
||||
|
||||
|
|
@ -41,7 +34,6 @@ export type KtxPublicIngestArgs =
|
|||
all: boolean;
|
||||
json: boolean;
|
||||
inputMode: KtxPublicIngestInputMode;
|
||||
depth?: KtxPublicIngestDepth;
|
||||
queryHistory?: KtxPublicIngestQueryHistoryFlag;
|
||||
queryHistoryWindowDays?: number;
|
||||
scanMode?: Extract<KtxScanArgs, { command: 'run' }>['mode'];
|
||||
|
|
@ -58,7 +50,6 @@ export interface KtxPublicIngestPlanTarget {
|
|||
sourceDir?: string;
|
||||
debugCommand: string;
|
||||
steps: KtxPublicIngestStepName[];
|
||||
databaseDepth?: KtxPublicIngestDepth;
|
||||
detectRelationships?: boolean;
|
||||
preflightFailure?: string;
|
||||
queryHistory?: {
|
||||
|
|
@ -67,7 +58,6 @@ export interface KtxPublicIngestPlanTarget {
|
|||
windowDays?: number;
|
||||
pullConfig?: Record<string, unknown>;
|
||||
unsupported?: boolean;
|
||||
skippedStoredByFast?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -121,7 +111,6 @@ interface KtxPublicContextBuildArgs {
|
|||
inputMode: 'auto' | 'disabled';
|
||||
targetConnectionId?: string;
|
||||
all?: boolean;
|
||||
depth?: KtxPublicIngestDepth;
|
||||
queryHistory?: KtxPublicIngestQueryHistoryFlag;
|
||||
queryHistoryWindowDays?: number;
|
||||
scanMode?: Extract<KtxScanArgs, { command: 'run' }>['mode'];
|
||||
|
|
@ -154,7 +143,6 @@ interface KtxUnsupportedQueryHistoryWarning {
|
|||
|
||||
interface KtxPublicIngestWarningAccumulator {
|
||||
warnings: string[];
|
||||
ignoredDepthForSources: string[];
|
||||
ignoredQueryHistoryForSources: string[];
|
||||
unsupportedQueryHistoryForDatabases: KtxUnsupportedQueryHistoryWarning[];
|
||||
}
|
||||
|
|
@ -162,7 +150,6 @@ interface KtxPublicIngestWarningAccumulator {
|
|||
function createWarningAccumulator(): KtxPublicIngestWarningAccumulator {
|
||||
return {
|
||||
warnings: [],
|
||||
ignoredDepthForSources: [],
|
||||
ignoredQueryHistoryForSources: [],
|
||||
unsupportedQueryHistoryForDatabases: [],
|
||||
};
|
||||
|
|
@ -233,7 +220,6 @@ function finalizeWarnings(
|
|||
accumulator: KtxPublicIngestWarningAccumulator,
|
||||
args: {
|
||||
all: boolean;
|
||||
depth?: KtxPublicIngestDepth;
|
||||
queryHistory?: KtxPublicIngestQueryHistoryFlag;
|
||||
queryHistoryWindowDays?: number;
|
||||
},
|
||||
|
|
@ -242,11 +228,6 @@ function finalizeWarnings(
|
|||
...accumulator.warnings,
|
||||
...unsupportedQueryHistoryWarnings(accumulator.unsupportedQueryHistoryForDatabases, args.all),
|
||||
];
|
||||
const depthOption = args.depth ? `--${args.depth}` : null;
|
||||
if (depthOption) {
|
||||
const warning = sourceIgnoredWarning(depthOption, accumulator.ignoredDepthForSources, args.all);
|
||||
if (warning) warnings.push(warning);
|
||||
}
|
||||
if (args.queryHistory === 'enabled' || args.queryHistoryWindowDays !== undefined) {
|
||||
const warning = sourceIgnoredWarning('--query-history', accumulator.ignoredQueryHistoryForSources, args.all);
|
||||
if (warning) warnings.push(warning);
|
||||
|
|
@ -317,13 +298,12 @@ function resolveDatabaseTargetOptions(input: {
|
|||
driver: string;
|
||||
connection: KtxProjectConnectionConfig;
|
||||
args: {
|
||||
depth?: KtxPublicIngestDepth;
|
||||
queryHistory?: KtxPublicIngestQueryHistoryFlag;
|
||||
queryHistoryWindowDays?: number;
|
||||
scanMode?: Extract<KtxScanArgs, { command: 'run' }>['mode'];
|
||||
};
|
||||
warnings: KtxPublicIngestWarningAccumulator;
|
||||
}): Pick<KtxPublicIngestPlanTarget, 'databaseDepth' | 'queryHistory' | 'steps'> {
|
||||
}): Pick<KtxPublicIngestPlanTarget, 'queryHistory' | 'steps'> {
|
||||
const storedQh = storedQueryHistory(input.connection);
|
||||
const dialect = queryHistoryDialectByDriver.get(input.driver);
|
||||
const explicitQueryHistory = input.args.queryHistory ?? 'default';
|
||||
|
|
@ -332,7 +312,6 @@ function resolveDatabaseTargetOptions(input: {
|
|||
const requestedQh =
|
||||
explicitQueryHistory === 'enabled' ||
|
||||
(explicitQueryHistory !== 'disabled' && (windowOverrideRequested || storedEnabled));
|
||||
let depth = input.args.depth ?? databaseContextDepth(input.connection) ?? 'fast';
|
||||
const queryHistory = {
|
||||
enabled: false,
|
||||
...(input.args.queryHistoryWindowDays !== undefined
|
||||
|
|
@ -350,19 +329,13 @@ function resolveDatabaseTargetOptions(input: {
|
|||
explicitQueryHistory === 'enabled' || input.args.queryHistoryWindowDays !== undefined ? 'explicit' : 'stored',
|
||||
});
|
||||
return {
|
||||
databaseDepth: depth,
|
||||
queryHistory: { ...queryHistory, unsupported: true },
|
||||
steps: ['database-schema'],
|
||||
};
|
||||
}
|
||||
|
||||
if (requestedQh && dialect) {
|
||||
if (depth === 'fast') {
|
||||
input.warnings.warnings.push(`--query-history requires deep ingest; running ${input.connectionId} with --deep.`);
|
||||
}
|
||||
depth = 'deep';
|
||||
return {
|
||||
databaseDepth: depth,
|
||||
queryHistory: {
|
||||
...queryHistory,
|
||||
enabled: true,
|
||||
|
|
@ -378,30 +351,35 @@ function resolveDatabaseTargetOptions(input: {
|
|||
};
|
||||
}
|
||||
|
||||
if (input.args.depth === 'fast' && explicitQueryHistory !== 'enabled' && storedEnabled) {
|
||||
input.warnings.warnings.push(
|
||||
`${input.connectionId} has query history enabled in ktx.yaml, but --fast skips query-history processing.`,
|
||||
);
|
||||
return {
|
||||
databaseDepth: 'fast',
|
||||
queryHistory: { ...queryHistory, skippedStoredByFast: true },
|
||||
steps: ['database-schema'],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
databaseDepth: depth,
|
||||
queryHistory,
|
||||
steps: ['database-schema'],
|
||||
};
|
||||
}
|
||||
|
||||
function enrichmentReadinessGaps(config: KtxProjectConfig): string[] {
|
||||
const gaps: string[] = [];
|
||||
if (config.llm.provider.backend === 'none' || !config.llm.models.default) {
|
||||
gaps.push('model configuration');
|
||||
}
|
||||
|
||||
if (config.scan.enrichment.mode !== 'llm') {
|
||||
gaps.push('scan enrichment mode');
|
||||
}
|
||||
|
||||
const embeddings = config.scan.enrichment.embeddings;
|
||||
if (!embeddings || embeddings.backend === 'none' || !embeddings.model || embeddings.dimensions <= 0) {
|
||||
gaps.push('scan embeddings');
|
||||
}
|
||||
|
||||
return gaps;
|
||||
}
|
||||
|
||||
function targetForConnection(
|
||||
connectionId: string,
|
||||
connection: KtxProjectConnectionConfig,
|
||||
projectConfig: KtxPublicIngestProject['config'],
|
||||
args: {
|
||||
depth?: KtxPublicIngestDepth;
|
||||
queryHistory?: KtxPublicIngestQueryHistoryFlag;
|
||||
queryHistoryWindowDays?: number;
|
||||
scanMode?: Extract<KtxScanArgs, { command: 'run' }>['mode'];
|
||||
|
|
@ -412,9 +390,6 @@ function targetForConnection(
|
|||
const adapter = sourceAdapterByDriver.get(driver);
|
||||
const sourceDir = sourceDirForConnection(connection);
|
||||
if (adapter) {
|
||||
if (args.depth) {
|
||||
warnings.ignoredDepthForSources.push(connectionId);
|
||||
}
|
||||
if (args.queryHistory === 'enabled' || args.queryHistoryWindowDays !== undefined) {
|
||||
warnings.ignoredQueryHistoryForSources.push(connectionId);
|
||||
}
|
||||
|
|
@ -431,18 +406,18 @@ function targetForConnection(
|
|||
|
||||
if (isDatabaseDriver(driver)) {
|
||||
const options = resolveDatabaseTargetOptions({ connectionId, driver, connection, args, warnings });
|
||||
const gaps = options.databaseDepth === 'deep' ? deepReadinessGaps(projectConfig) : [];
|
||||
const gaps = enrichmentReadinessGaps(projectConfig);
|
||||
return {
|
||||
connectionId,
|
||||
driver,
|
||||
operation: 'database-ingest',
|
||||
debugCommand: `ktx ingest ${connectionId} --debug`,
|
||||
detectRelationships: options.databaseDepth === 'deep' && projectConfig.scan.relationships.enabled,
|
||||
detectRelationships: projectConfig.scan.relationships.enabled,
|
||||
...(gaps.length > 0
|
||||
? {
|
||||
preflightFailure: `${connectionId} requires deep ingest readiness: ${gaps.join(
|
||||
preflightFailure: `${connectionId} cannot be ingested: enrichment is not configured (${gaps.join(
|
||||
', ',
|
||||
)}. Run ktx setup or rerun with --fast.`,
|
||||
)}). Run ktx setup to configure a model and embeddings.`,
|
||||
}
|
||||
: {}),
|
||||
...options,
|
||||
|
|
@ -458,7 +433,6 @@ export function buildPublicIngestPlan(
|
|||
projectDir: string;
|
||||
targetConnectionId?: string;
|
||||
all: boolean;
|
||||
depth?: KtxPublicIngestDepth;
|
||||
queryHistory?: KtxPublicIngestQueryHistoryFlag;
|
||||
queryHistoryWindowDays?: number;
|
||||
scanMode?: Extract<KtxScanArgs, { command: 'run' }>['mode'];
|
||||
|
|
@ -522,13 +496,12 @@ function retryCommandForTarget(
|
|||
args: Extract<KtxPublicIngestArgs, { command: 'run' }>,
|
||||
): string {
|
||||
const projectPart = ` --project-dir ${args.projectDir}`;
|
||||
const depthPart = target.databaseDepth ? ` --${target.databaseDepth}` : '';
|
||||
const queryHistoryPart = target.queryHistory?.enabled === true ? ' --query-history' : '';
|
||||
const windowPart =
|
||||
target.queryHistory?.enabled === true && target.queryHistory.windowDays !== undefined
|
||||
? ` --query-history-window-days ${target.queryHistory.windowDays}`
|
||||
: '';
|
||||
return `ktx ingest ${target.connectionId}${projectPart}${depthPart}${queryHistoryPart}${windowPart}`;
|
||||
return `ktx ingest ${target.connectionId}${projectPart}${queryHistoryPart}${windowPart}`;
|
||||
}
|
||||
|
||||
function trimTrailingPeriod(value: string): string {
|
||||
|
|
@ -830,7 +803,7 @@ export async function executePublicIngestTarget(
|
|||
command: 'run',
|
||||
projectDir: args.projectDir,
|
||||
connectionId: target.connectionId,
|
||||
mode: target.databaseDepth === 'deep' ? 'enriched' : 'structural',
|
||||
mode: 'enriched',
|
||||
detectRelationships: target.detectRelationships === true,
|
||||
dryRun: false,
|
||||
...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
|
||||
|
|
@ -979,7 +952,6 @@ export async function runKtxPublicIngest(
|
|||
all: args.all,
|
||||
entrypoint: 'ingest',
|
||||
inputMode: args.inputMode,
|
||||
...(args.depth ? { depth: args.depth } : {}),
|
||||
...(args.queryHistory ? { queryHistory: args.queryHistory } : {}),
|
||||
...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
|
||||
...(args.scanMode ? { scanMode: args.scanMode } : {}),
|
||||
|
|
|
|||
|
|
@ -7,12 +7,7 @@ import { serializeKtxProjectConfig } from './context/project/config.js';
|
|||
import type { KtxCliIo } from './cli-runtime.js';
|
||||
import { errorMessage, writePrefixedLines } from './clack.js';
|
||||
import { buildPublicIngestPlan } from './public-ingest.js';
|
||||
import {
|
||||
type KtxDatabaseContextDepth,
|
||||
databaseContextDepth,
|
||||
} from './ingest-depth.js';
|
||||
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
|
||||
import { ensureSetupDatabaseContextDepths } from './setup-database-context-depth.js';
|
||||
import {
|
||||
type ContextBuildSourceProgressUpdate,
|
||||
runContextBuild,
|
||||
|
|
@ -353,16 +348,6 @@ async function readLatestScanReport(projectDir: string, connectionId: string): P
|
|||
return reports.at(-1)?.report ?? null;
|
||||
}
|
||||
|
||||
function scanReportHasSchemaManifest(report: unknown, connectionId: string): boolean {
|
||||
if (!isRecord(report)) {
|
||||
return false;
|
||||
}
|
||||
if (report.connectionId !== connectionId || report.dryRun === true) {
|
||||
return false;
|
||||
}
|
||||
return stringArrayValue(isRecord(report.artifactPaths) ? report.artifactPaths.manifestShards : undefined).length > 0;
|
||||
}
|
||||
|
||||
function scanReportHasCompletedDeepEnrichment(
|
||||
report: unknown,
|
||||
connectionId: string,
|
||||
|
|
@ -389,18 +374,6 @@ function scanReportHasCompletedDeepEnrichment(
|
|||
);
|
||||
}
|
||||
|
||||
function scanReportSatisfiesDepth(input: {
|
||||
report: unknown;
|
||||
connectionId: string;
|
||||
depth: KtxDatabaseContextDepth;
|
||||
relationshipsRequired: boolean;
|
||||
}): boolean {
|
||||
if (input.depth === 'fast') {
|
||||
return scanReportHasSchemaManifest(input.report, input.connectionId);
|
||||
}
|
||||
return scanReportHasCompletedDeepEnrichment(input.report, input.connectionId, input.relationshipsRequired);
|
||||
}
|
||||
|
||||
async function verifyPrimarySourceScans(
|
||||
project: KtxLocalProject,
|
||||
connectionIds: string[],
|
||||
|
|
@ -408,15 +381,9 @@ async function verifyPrimarySourceScans(
|
|||
const details: string[] = [];
|
||||
const relationshipsRequired = project.config.scan.relationships.enabled;
|
||||
for (const connectionId of connectionIds) {
|
||||
const connection = project.config.connections[connectionId];
|
||||
const depth = connection ? (databaseContextDepth(connection) ?? 'fast') : 'fast';
|
||||
const report = await readLatestScanReport(project.projectDir, connectionId);
|
||||
if (!scanReportSatisfiesDepth({ report, connectionId, depth, relationshipsRequired })) {
|
||||
details.push(
|
||||
depth === 'fast'
|
||||
? `${connectionId}: schema context has not completed.`
|
||||
: `${connectionId}: deep database context has not completed.`,
|
||||
);
|
||||
if (!scanReportHasCompletedDeepEnrichment(report, connectionId, relationshipsRequired)) {
|
||||
details.push(`${connectionId}: database context has not completed.`);
|
||||
}
|
||||
}
|
||||
return { ready: details.length === 0, details };
|
||||
|
|
@ -482,7 +449,6 @@ function writeSkippedContext(projectDir: string, io: KtxCliIo): void {
|
|||
}
|
||||
|
||||
function writeSuccess(
|
||||
project: KtxLocalProject,
|
||||
readiness: KtxSetupContextReadiness,
|
||||
targets: KtxSetupContextTargets,
|
||||
io: KtxCliIo,
|
||||
|
|
@ -493,9 +459,7 @@ function writeSuccess(
|
|||
io.stdout.write(' none\n');
|
||||
} else {
|
||||
for (const connectionId of targets.primarySourceConnectionIds) {
|
||||
const connection = project.config.connections[connectionId];
|
||||
const depth = connection ? (databaseContextDepth(connection) ?? 'fast') : 'fast';
|
||||
io.stdout.write(` ${connectionId}: ${depth === 'deep' ? 'deep context complete' : 'schema context complete'}\n`);
|
||||
io.stdout.write(` ${connectionId}: database context complete\n`);
|
||||
}
|
||||
}
|
||||
io.stdout.write('\nContext sources:\n');
|
||||
|
|
@ -636,7 +600,7 @@ async function runBuild(
|
|||
failureReason: undefined,
|
||||
...(lastSourceProgress ? { sourceProgress: lastSourceProgress } : {}),
|
||||
});
|
||||
writeSuccess(project, readiness, targets, io);
|
||||
writeSuccess(readiness, targets, io);
|
||||
return { status: 'ready', projectDir: args.projectDir, runId };
|
||||
}
|
||||
|
||||
|
|
@ -678,17 +642,8 @@ export async function runKtxSetupContextStep(
|
|||
deps: KtxSetupContextDeps = {},
|
||||
): Promise<KtxSetupContextResult> {
|
||||
try {
|
||||
let project = await loadKtxProject({ projectDir: args.projectDir });
|
||||
const project = await loadKtxProject({ projectDir: args.projectDir });
|
||||
const prompts = deps.prompts ?? createPromptAdapter();
|
||||
const depthProject = await ensureSetupDatabaseContextDepths({
|
||||
project,
|
||||
args,
|
||||
prompts,
|
||||
});
|
||||
if (depthProject === 'back') {
|
||||
return { status: 'back', projectDir: args.projectDir };
|
||||
}
|
||||
project = depthProject;
|
||||
const existingState = await readKtxSetupContextState(args.projectDir);
|
||||
const completedSteps = (await readKtxSetupState(args.projectDir)).completed_steps;
|
||||
if (completedSteps.includes('context') && existingState.status === 'completed') {
|
||||
|
|
|
|||
|
|
@ -1,131 +0,0 @@
|
|||
import { writeFile } from 'node:fs/promises';
|
||||
import { type KtxLocalProject, loadKtxProject } from './context/project/project.js';
|
||||
import { type KtxProjectConnectionConfig, serializeKtxProjectConfig } from './context/project/config.js';
|
||||
import {
|
||||
type KtxDatabaseContextDepth,
|
||||
databaseContextDepth,
|
||||
deepReadinessGaps,
|
||||
isDatabaseDriver,
|
||||
normalizeConnectionDriver,
|
||||
recommendedDatabaseContextDepth,
|
||||
withDatabaseContextDepth,
|
||||
} from './ingest-depth.js';
|
||||
import type { KtxSetupPromptOption } from './setup-prompts.js';
|
||||
|
||||
export interface KtxSetupDatabaseContextDepthArgs {
|
||||
inputMode: 'auto' | 'disabled';
|
||||
}
|
||||
|
||||
export interface KtxSetupDatabaseContextDepthPromptAdapter {
|
||||
select(options: { message: string; options: KtxSetupPromptOption[] }): Promise<string>;
|
||||
}
|
||||
|
||||
function databaseConnectionsNeedingDepth(project: KtxLocalProject): string[] {
|
||||
return Object.entries(project.config.connections)
|
||||
.filter(([, connection]) => isDatabaseDriver(normalizeConnectionDriver(connection)))
|
||||
.filter(([, connection]) => databaseContextDepth(connection) === undefined)
|
||||
.map(([connectionId]) => connectionId)
|
||||
.sort((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
async function chooseSetupDatabaseContextDepth(input: {
|
||||
project: KtxLocalProject;
|
||||
args: KtxSetupDatabaseContextDepthArgs;
|
||||
prompts: KtxSetupDatabaseContextDepthPromptAdapter;
|
||||
}): Promise<KtxDatabaseContextDepth | 'back'> {
|
||||
const recommended = recommendedDatabaseContextDepth(input.project.config);
|
||||
if (input.args.inputMode === 'disabled') {
|
||||
return recommended;
|
||||
}
|
||||
|
||||
const deepReady = deepReadinessGaps(input.project.config).length === 0;
|
||||
const options =
|
||||
recommended === 'deep'
|
||||
? [
|
||||
{
|
||||
value: 'deep',
|
||||
label: 'Deep: AI descriptions, embeddings, relationships, slower',
|
||||
hint: 'recommended',
|
||||
},
|
||||
{ value: 'fast', label: 'Fast: schema only, no AI, quickest' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
]
|
||||
: [
|
||||
{ value: 'fast', label: 'Fast: schema only, no AI, quickest', hint: 'recommended' },
|
||||
{ value: 'deep', label: 'Deep: AI descriptions, embeddings, relationships, slower' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
];
|
||||
|
||||
const choice = await input.prompts.select({
|
||||
message:
|
||||
'How much database context should KTX build?\n\n' +
|
||||
(deepReady
|
||||
? 'Deep is available because model, embedding, and scan enrichment are configured.'
|
||||
: 'Fast is recommended because model, embedding, or scan enrichment is not configured.'),
|
||||
options,
|
||||
});
|
||||
if (choice === 'back') {
|
||||
return 'back';
|
||||
}
|
||||
if (choice === 'fast' || choice === 'deep') {
|
||||
return choice;
|
||||
}
|
||||
return recommended;
|
||||
}
|
||||
|
||||
async function writeDatabaseContextDepths(
|
||||
project: KtxLocalProject,
|
||||
connectionIds: string[],
|
||||
depth: KtxDatabaseContextDepth,
|
||||
): Promise<KtxLocalProject> {
|
||||
if (connectionIds.length === 0) {
|
||||
return project;
|
||||
}
|
||||
const nextConnections = { ...project.config.connections };
|
||||
for (const connectionId of connectionIds) {
|
||||
const connection = nextConnections[connectionId];
|
||||
if (connection) {
|
||||
nextConnections[connectionId] = withDatabaseContextDepth(connection, depth);
|
||||
}
|
||||
}
|
||||
const nextConfig = { ...project.config, connections: nextConnections };
|
||||
await writeFile(project.configPath, serializeKtxProjectConfig(nextConfig), 'utf-8');
|
||||
return await loadKtxProject({ projectDir: project.projectDir });
|
||||
}
|
||||
|
||||
export async function ensureSetupDatabaseContextDepths(input: {
|
||||
project: KtxLocalProject;
|
||||
args: KtxSetupDatabaseContextDepthArgs;
|
||||
prompts: KtxSetupDatabaseContextDepthPromptAdapter;
|
||||
}): Promise<KtxLocalProject | 'back'> {
|
||||
const missingDepthConnectionIds = databaseConnectionsNeedingDepth(input.project);
|
||||
if (missingDepthConnectionIds.length === 0) {
|
||||
return input.project;
|
||||
}
|
||||
|
||||
const depth = await chooseSetupDatabaseContextDepth(input);
|
||||
if (depth === 'back') {
|
||||
return 'back';
|
||||
}
|
||||
return await writeDatabaseContextDepths(input.project, missingDepthConnectionIds, depth);
|
||||
}
|
||||
|
||||
export async function applySetupDatabaseContextDepth(input: {
|
||||
project: KtxLocalProject;
|
||||
connection: KtxProjectConnectionConfig;
|
||||
args: KtxSetupDatabaseContextDepthArgs;
|
||||
prompts: KtxSetupDatabaseContextDepthPromptAdapter;
|
||||
}): Promise<KtxProjectConnectionConfig | 'back'> {
|
||||
if (
|
||||
!isDatabaseDriver(normalizeConnectionDriver(input.connection)) ||
|
||||
databaseContextDepth(input.connection) !== undefined
|
||||
) {
|
||||
return input.connection;
|
||||
}
|
||||
|
||||
const depth = await chooseSetupDatabaseContextDepth(input);
|
||||
if (depth === 'back') {
|
||||
return 'back';
|
||||
}
|
||||
return withDatabaseContextDepth(input.connection, depth);
|
||||
}
|
||||
|
|
@ -29,7 +29,6 @@ import {
|
|||
} from './database-tree-picker.js';
|
||||
import { withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js';
|
||||
import { runKtxScan } from './scan.js';
|
||||
import { applySetupDatabaseContextDepth } from './setup-database-context-depth.js';
|
||||
import { writeProjectLocalSecretReference } from './setup-secrets.js';
|
||||
import { isDemoConnection } from './telemetry/demo-detect.js';
|
||||
import { emitTelemetryEvent } from './telemetry/index.js';
|
||||
|
|
@ -1614,45 +1613,10 @@ async function applyHistoricSqlConfigToExistingConnection(input: {
|
|||
prompts: input.prompts,
|
||||
});
|
||||
if (withHistoricSql === 'back') return 'back';
|
||||
const withContextDepth = await maybeApplyContextDepthConfig({
|
||||
projectDir: input.projectDir,
|
||||
connectionId: input.connectionId,
|
||||
connection: withHistoricSql,
|
||||
args: input.args,
|
||||
prompts: input.prompts,
|
||||
});
|
||||
if (withContextDepth === 'back') return 'back';
|
||||
await writeConnectionConfig({
|
||||
projectDir: input.projectDir,
|
||||
connectionId: input.connectionId,
|
||||
connection: withContextDepth,
|
||||
});
|
||||
}
|
||||
|
||||
async function maybeApplyContextDepthConfig(input: {
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
connection: KtxProjectConnectionConfig;
|
||||
args: KtxSetupDatabasesArgs;
|
||||
prompts: KtxSetupDatabasesPromptAdapter;
|
||||
}): Promise<KtxProjectConnectionConfig | 'back'> {
|
||||
const project = await loadKtxProject({ projectDir: input.projectDir });
|
||||
return await applySetupDatabaseContextDepth({
|
||||
project: {
|
||||
...project,
|
||||
config: {
|
||||
...project.config,
|
||||
connections: {
|
||||
...project.config.connections,
|
||||
[input.connectionId]: input.connection,
|
||||
},
|
||||
},
|
||||
},
|
||||
connection: input.connection,
|
||||
args: {
|
||||
inputMode: input.args.inputMode === 'disabled' || input.args.databaseUrl ? 'disabled' : input.args.inputMode,
|
||||
},
|
||||
prompts: input.prompts,
|
||||
connection: withHistoricSql,
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -1698,7 +1662,7 @@ async function validateAndScanConnection(input: {
|
|||
deps: input.deps,
|
||||
});
|
||||
writeSetupSection(input.io, `Building schema context for ${input.connectionId}`, [
|
||||
'Running fast database ingest…',
|
||||
'Running database scan…',
|
||||
]);
|
||||
let scanIo = createBufferedCommandIo();
|
||||
let scanCode = await scanConnection(input.projectDir, input.connectionId, scanIo);
|
||||
|
|
@ -1708,7 +1672,7 @@ async function validateAndScanConnection(input: {
|
|||
writePrefixedLines(
|
||||
(chunk) => input.io.stderr.write(chunk),
|
||||
[
|
||||
`Fast database ingest failed for ${input.connectionId}.`,
|
||||
`Database scan failed for ${input.connectionId}.`,
|
||||
'Native SQLite is built for a different Node.js ABI.',
|
||||
`Detail: ${nativeSqliteDetail}`,
|
||||
'Rebuilding Native SQLite with pnpm run native:rebuild…',
|
||||
|
|
@ -1719,7 +1683,7 @@ async function validateAndScanConnection(input: {
|
|||
if (rebuildCode === 0) {
|
||||
writePrefixedLines(
|
||||
(chunk) => input.io.stderr.write(chunk),
|
||||
'Native SQLite rebuild complete. Retrying fast database ingest…',
|
||||
'Native SQLite rebuild complete. Retrying database scan…',
|
||||
);
|
||||
const retryScanIo = createBufferedCommandIo();
|
||||
scanCode = await scanConnection(input.projectDir, input.connectionId, retryScanIo);
|
||||
|
|
@ -1730,10 +1694,10 @@ async function validateAndScanConnection(input: {
|
|||
(chunk) => input.io.stderr.write(chunk),
|
||||
[
|
||||
rebuildCode === 0
|
||||
? `Fast database ingest still failed for ${input.connectionId} after rebuilding Native SQLite.`
|
||||
? `Database scan still failed for ${input.connectionId} after rebuilding Native SQLite.`
|
||||
: `Native SQLite rebuild failed for ${input.connectionId}.`,
|
||||
'Fix: pnpm run native:rebuild',
|
||||
`Retry: ktx ingest ${input.connectionId} --project-dir ${input.projectDir} --fast`,
|
||||
`Retry: ktx ingest ${input.connectionId} --project-dir ${input.projectDir}`,
|
||||
].join('\n'),
|
||||
);
|
||||
}
|
||||
|
|
@ -1742,8 +1706,8 @@ async function validateAndScanConnection(input: {
|
|||
writePrefixedLines(
|
||||
(chunk) => input.io.stderr.write(chunk),
|
||||
[
|
||||
`Fast database ingest failed for ${input.connectionId}.`,
|
||||
`Debug command: ktx ingest ${input.connectionId} --project-dir ${input.projectDir} --fast --debug`,
|
||||
`Database scan failed for ${input.connectionId}.`,
|
||||
`Debug command: ktx ingest ${input.connectionId} --project-dir ${input.projectDir} --debug`,
|
||||
].join('\n'),
|
||||
);
|
||||
}
|
||||
|
|
@ -2167,22 +2131,10 @@ export async function runKtxSetupDatabasesStep(
|
|||
returnToDriverSelection = true;
|
||||
break;
|
||||
}
|
||||
const withContextDepth = await maybeApplyContextDepthConfig({
|
||||
projectDir: args.projectDir,
|
||||
connectionId: connectionChoice.connectionId,
|
||||
connection: withHistoricSql,
|
||||
args,
|
||||
prompts,
|
||||
});
|
||||
if (withContextDepth === 'back') {
|
||||
if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir };
|
||||
returnToDriverSelection = true;
|
||||
break;
|
||||
}
|
||||
await writeConnectionConfig({
|
||||
projectDir: args.projectDir,
|
||||
connectionId: connectionChoice.connectionId,
|
||||
connection: withContextDepth,
|
||||
connection: withHistoricSql,
|
||||
io,
|
||||
});
|
||||
} else {
|
||||
|
|
@ -2193,22 +2145,10 @@ export async function runKtxSetupDatabasesStep(
|
|||
returnToDriverSelection = true;
|
||||
break;
|
||||
}
|
||||
const withContextDepth = await maybeApplyContextDepthConfig({
|
||||
projectDir: args.projectDir,
|
||||
connectionId: connectionChoice.connectionId,
|
||||
connection: withHistoricSql,
|
||||
args,
|
||||
prompts,
|
||||
});
|
||||
if (withContextDepth === 'back') {
|
||||
if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir };
|
||||
returnToDriverSelection = true;
|
||||
break;
|
||||
}
|
||||
await writeConnectionConfig({
|
||||
projectDir: args.projectDir,
|
||||
connectionId: connectionChoice.connectionId,
|
||||
connection: withContextDepth,
|
||||
connection: withHistoricSql,
|
||||
io,
|
||||
});
|
||||
}
|
||||
|
|
@ -2291,22 +2231,10 @@ export async function runKtxSetupDatabasesStep(
|
|||
returnToDriverSelection = true;
|
||||
break;
|
||||
}
|
||||
const withContextDepth = await maybeApplyContextDepthConfig({
|
||||
projectDir: args.projectDir,
|
||||
connectionId: connectionChoice.connectionId,
|
||||
connection: withHistoricSql,
|
||||
args,
|
||||
prompts,
|
||||
});
|
||||
if (withContextDepth === 'back') {
|
||||
if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir };
|
||||
returnToDriverSelection = true;
|
||||
break;
|
||||
}
|
||||
await writeConnectionConfig({
|
||||
projectDir: args.projectDir,
|
||||
connectionId: connectionChoice.connectionId,
|
||||
connection: withContextDepth,
|
||||
connection: withHistoricSql,
|
||||
io,
|
||||
});
|
||||
setupStatus = await validateAndScanConnection({
|
||||
|
|
|
|||
|
|
@ -217,6 +217,39 @@ function credentialRef(value: string | undefined, label: string): string {
|
|||
return ref;
|
||||
}
|
||||
|
||||
type SourceCredentialFlag = {
|
||||
field: 'sourceAuthTokenRef' | 'sourceApiKeyRef' | 'sourceClientSecretRef';
|
||||
flag: string;
|
||||
};
|
||||
|
||||
// Each connector reads exactly one credential ref; the flag name mirrors the
|
||||
// ktx.yaml field it writes (auth_token_ref / api_key_ref / client_secret_ref).
|
||||
const SOURCE_CREDENTIAL_FLAG: Record<KtxSetupSourceType, SourceCredentialFlag> = {
|
||||
dbt: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
|
||||
metricflow: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
|
||||
lookml: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
|
||||
notion: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
|
||||
metabase: { field: 'sourceApiKeyRef', flag: '--source-api-key-ref' },
|
||||
looker: { field: 'sourceClientSecretRef', flag: '--source-client-secret-ref' },
|
||||
};
|
||||
|
||||
const ALL_SOURCE_CREDENTIAL_FLAGS: SourceCredentialFlag[] = [
|
||||
{ field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
|
||||
{ field: 'sourceApiKeyRef', flag: '--source-api-key-ref' },
|
||||
{ field: 'sourceClientSecretRef', flag: '--source-client-secret-ref' },
|
||||
];
|
||||
|
||||
// Reject a credential ref flag the chosen source does not read, so a wrong flag
|
||||
// fails loudly instead of being silently dropped (KLO-724).
|
||||
function assertSourceCredentialFlags(source: KtxSetupSourceType, args: KtxSetupSourcesArgs): void {
|
||||
const allowed = SOURCE_CREDENTIAL_FLAG[source];
|
||||
for (const { field, flag } of ALL_SOURCE_CREDENTIAL_FLAGS) {
|
||||
if (args[field] && field !== allowed.field) {
|
||||
throw new Error(`${flag} does not apply to --source ${source}; use ${allowed.flag}.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function chooseSourceCredentialRef(input: {
|
||||
prompts: KtxSetupSourcesPromptAdapter;
|
||||
projectDir: string;
|
||||
|
|
@ -515,7 +548,7 @@ function buildNotionConnection(args: KtxSetupSourcesArgs): KtxProjectConnectionC
|
|||
}
|
||||
return {
|
||||
driver: 'notion',
|
||||
auth_token_ref: credentialRef(args.sourceApiKeyRef, 'Notion token ref'),
|
||||
auth_token_ref: credentialRef(args.sourceAuthTokenRef, 'Notion token ref'),
|
||||
crawl_mode: crawlMode,
|
||||
...(rootPageIds.length > 0 ? { root_page_ids: rootPageIds } : {}),
|
||||
root_database_ids: [],
|
||||
|
|
@ -1295,10 +1328,10 @@ async function promptForInteractiveSource(
|
|||
label: 'Notion integration token',
|
||||
envName: 'NOTION_TOKEN',
|
||||
secretFileName: `${currentState.sourceConnectionId ?? 'notion-main'}-token`,
|
||||
existingRef: currentState.sourceApiKeyRef,
|
||||
existingRef: currentState.sourceAuthTokenRef,
|
||||
});
|
||||
if (ref === 'back') return 'back';
|
||||
currentState.sourceApiKeyRef = ref;
|
||||
currentState.sourceAuthTokenRef = ref;
|
||||
return 'next';
|
||||
},
|
||||
async (currentState) => {
|
||||
|
|
@ -1326,7 +1359,7 @@ async function promptForInteractiveSource(
|
|||
connectionId,
|
||||
connection: {
|
||||
driver: 'notion',
|
||||
auth_token_ref: credentialRef(currentState.sourceApiKeyRef, 'Notion token ref'),
|
||||
auth_token_ref: credentialRef(currentState.sourceAuthTokenRef, 'Notion token ref'),
|
||||
crawl_mode: 'selected_roots',
|
||||
root_page_ids: currentState.notionRootPageIds ?? [],
|
||||
root_database_ids: [],
|
||||
|
|
@ -1516,7 +1549,7 @@ function sourceArgsFromExistingConnection(input: {
|
|||
return sourceArgs;
|
||||
}
|
||||
|
||||
sourceArgs.sourceApiKeyRef = stringField(input.connection.auth_token_ref);
|
||||
sourceArgs.sourceAuthTokenRef = stringField(input.connection.auth_token_ref);
|
||||
sourceArgs.notionCrawlMode =
|
||||
input.connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots';
|
||||
if (Array.isArray(input.connection.root_page_ids)) {
|
||||
|
|
@ -1817,6 +1850,10 @@ export async function runKtxSetupSourcesStep(
|
|||
return { status: 'skipped', projectDir: args.projectDir };
|
||||
}
|
||||
|
||||
if (args.source) {
|
||||
assertSourceCredentialFlags(args.source, args);
|
||||
}
|
||||
|
||||
const prompts = deps.prompts ?? createPromptAdapter();
|
||||
const project = await loadKtxProject({ projectDir: args.projectDir });
|
||||
if (!hasPrimarySource(project.config)) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { existsSync } from 'node:fs';
|
||||
import { basename, join, resolve } from 'node:path';
|
||||
import { getLatestLocalIngestStatus } from './context/ingest/local-ingest.js';
|
||||
import { savedMemoryCountsForReport } from './context/ingest/reports.js';
|
||||
import { ingestReportOutcome, savedMemoryCountsForReport } from './context/ingest/reports.js';
|
||||
import { ktxLocalStateDbPath } from './context/project/local-state-db.js';
|
||||
import { loadKtxProject, type KtxLocalProject } from './context/project/project.js';
|
||||
import { readKtxSetupState } from './context/project/setup-config.js';
|
||||
|
|
@ -306,7 +306,7 @@ function sourceConnections(config: Awaited<ReturnType<typeof loadKtxProject>>['c
|
|||
type LocalIngestStatusReport = NonNullable<Awaited<ReturnType<typeof getLatestLocalIngestStatus>>>;
|
||||
|
||||
function reportHasSavedContext(report: LocalIngestStatusReport): boolean {
|
||||
if (report.body.failedWorkUnits.length > 0) {
|
||||
if (ingestReportOutcome(report) === 'error') {
|
||||
return false;
|
||||
}
|
||||
const counts = savedMemoryCountsForReport(report);
|
||||
|
|
|
|||
|
|
@ -365,7 +365,6 @@
|
|||
"embeddings",
|
||||
"secrets",
|
||||
"databases",
|
||||
"database-context-depth",
|
||||
"sources",
|
||||
"context",
|
||||
"agents",
|
||||
|
|
|
|||
|
|
@ -38,7 +38,6 @@ const setupStepSchema = telemetryCommonEnvelopeSchema
|
|||
'embeddings',
|
||||
'secrets',
|
||||
'databases',
|
||||
'database-context-depth',
|
||||
'sources',
|
||||
'context',
|
||||
'agents',
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue