mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-19 08:28:06 +02:00
fix: improve ingest runtime readiness
This commit is contained in:
Andrey Avtomonov
parent
de72a10ffb
commit
564851508b
19 changed files with 927 additions and 25 deletions
|
|
@ -35,6 +35,7 @@ export function registerIngestCommands(
|
|||
.option('--query-history-window-days <days>', 'Query-history lookback window for this run', parsePositiveIntegerOption)
|
||||
.addOption(new Option('--plain', 'Print plain text output').conflicts(['json']))
|
||||
.addOption(new Option('--json', 'Print JSON output').conflicts(['plain']))
|
||||
.option('--yes', 'Install required managed runtime features without prompting')
|
||||
.option('--no-input', 'Disable interactive terminal input')
|
||||
.showHelpAfterError();
|
||||
|
||||
|
|
|
|||
|
|
@ -708,6 +708,10 @@ const INTERNAL_FAILURE_LINE_RE =
|
|||
const ACTIONABLE_FAILURE_LINE_RE =
|
||||
/^(Missing bundled Python runtime manifest|KTX Python runtime is required|KTX managed daemon|Error:|Failed\b|Could not\b|Cannot\b)/;
|
||||
|
||||
function trimErrorPrefix(line: string): string {
|
||||
return line.replace(/^Error:\s*/, '');
|
||||
}
|
||||
|
||||
function firstCapturedFailureLine(output: string | undefined): string | null {
|
||||
const lines = (output ?? '')
|
||||
.split(/\r?\n/)
|
||||
|
|
@ -715,7 +719,8 @@ function firstCapturedFailureLine(output: string | undefined): string | null {
|
|||
.filter((candidate) => candidate.length > 0)
|
||||
.filter((candidate) => !candidate.startsWith('KTX scan completed'))
|
||||
.filter((candidate) => !INTERNAL_FAILURE_LINE_RE.test(candidate));
|
||||
return lines.find((candidate) => ACTIONABLE_FAILURE_LINE_RE.test(candidate)) ?? lines.at(-1) ?? null;
|
||||
const line = lines.find((candidate) => ACTIONABLE_FAILURE_LINE_RE.test(candidate)) ?? lines.at(-1) ?? null;
|
||||
return line ? trimErrorPrefix(line) : null;
|
||||
}
|
||||
|
||||
function isGenericFailedAtDetail(target: KtxPublicIngestPlanTarget, detail: string | null | undefined): boolean {
|
||||
|
|
|
|||
|
|
@ -711,6 +711,40 @@ describe('runKtxCli', () => {
|
|||
expect(testIo.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('routes public ingest --yes as automatic runtime installation', async () => {
|
||||
const testIo = makeIo();
|
||||
const publicIngest = vi.fn().mockResolvedValue(0);
|
||||
|
||||
await expect(
|
||||
runKtxCli(['--project-dir', tempDir, 'ingest', 'warehouse', '--yes'], testIo.io, {
|
||||
publicIngest,
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(publicIngest).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
projectDir: tempDir,
|
||||
targetConnectionId: 'warehouse',
|
||||
runtimeInstallPolicy: 'auto',
|
||||
}),
|
||||
testIo.io,
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects conflicting public ingest runtime install modes', async () => {
|
||||
const testIo = makeIo();
|
||||
const publicIngest = vi.fn().mockResolvedValue(0);
|
||||
|
||||
await expect(
|
||||
runKtxCli(['--project-dir', tempDir, 'ingest', 'warehouse', '--yes', '--no-input'], testIo.io, {
|
||||
publicIngest,
|
||||
}),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(publicIngest).not.toHaveBeenCalled();
|
||||
expect(testIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input');
|
||||
});
|
||||
|
||||
it('rejects mutually exclusive public ingest depth flags before dispatch', async () => {
|
||||
const testIo = makeIo();
|
||||
const publicIngest = vi.fn().mockResolvedValue(0);
|
||||
|
|
|
|||
|
|
@ -979,6 +979,72 @@ describe('runKtxIngest', () => {
|
|||
expect(io.stdout()).toContain('Status: error\n');
|
||||
});
|
||||
|
||||
it('prints a clear first failure reason when query-history work units fail', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
const rawReason =
|
||||
'{"error":"invalid_grant","error_description":"reauth related error (invalid_rapt)","error_uri":"https://support.google.com/a/answer/9368756","error_subtype":"invalid_rapt"}';
|
||||
const runLocal = vi.fn(async (input: RunLocalIngestOptions): Promise<LocalIngestResult> => {
|
||||
const failedWorkUnit = {
|
||||
...localFakeBundleReport('query-history-failed').body.workUnits[0],
|
||||
unitKey: 'historic-sql-table-orders',
|
||||
rawFiles: ['tables/orders.json'],
|
||||
status: 'failed' as const,
|
||||
reason: rawReason,
|
||||
actions: [],
|
||||
touchedSlSources: [],
|
||||
};
|
||||
const report = localFakeBundleReport('query-history-failed', {
|
||||
id: 'report-query-history-failed',
|
||||
runId: 'run-query-history-failed',
|
||||
connectionId: input.connectionId,
|
||||
sourceKey: 'historic-sql',
|
||||
body: {
|
||||
workUnits: [failedWorkUnit],
|
||||
failedWorkUnits: [failedWorkUnit.unitKey],
|
||||
},
|
||||
});
|
||||
return {
|
||||
result: {
|
||||
jobId: 'query-history-failed',
|
||||
runId: report.runId,
|
||||
syncId: report.body.syncId,
|
||||
diffSummary: report.body.diffSummary,
|
||||
workUnitCount: report.body.workUnits.length,
|
||||
failedWorkUnits: report.body.failedWorkUnits,
|
||||
artifactsWritten: report.body.provenanceRows.length,
|
||||
commitSha: report.body.commitSha,
|
||||
},
|
||||
report,
|
||||
};
|
||||
});
|
||||
|
||||
const io = makeIo();
|
||||
await expect(
|
||||
runKtxIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir,
|
||||
connectionId: 'warehouse',
|
||||
adapter: 'historic-sql',
|
||||
outputMode: 'plain',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
runLocalIngest: runLocal,
|
||||
jobIdFactory: () => 'query-history-failed',
|
||||
},
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(io.stdout()).toContain('Status: error\n');
|
||||
expect(io.stdout()).toContain('Failed tasks: 1\n');
|
||||
expect(io.stdout()).toContain(
|
||||
'Error: Query history failed for 1 task. First failure: Google Cloud authentication failed while analyzing query history: application-default credentials expired or require reauthentication (invalid_grant / invalid_rapt). Run `gcloud auth application-default login`, then retry.',
|
||||
);
|
||||
expect(io.stdout()).not.toContain('error_uri');
|
||||
});
|
||||
|
||||
it('passes the debug LLM request file to local ingest runs', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await writeWarehouseConfig(projectDir);
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import {
|
|||
runLocalIngest,
|
||||
runLocalMetabaseIngest,
|
||||
savedMemoryCountsForReport,
|
||||
sanitizeMemoryFlowError,
|
||||
} from '@ktx/context/ingest';
|
||||
import type { KtxSqlQueryExecutorPort } from '@ktx/context/connections';
|
||||
import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project';
|
||||
|
|
@ -127,8 +128,70 @@ function reportSourceLabel(sourceKey: string): string {
|
|||
.join(' ');
|
||||
}
|
||||
|
||||
function jsonObjectFromFailureReason(reason: string): Record<string, unknown> | null {
|
||||
const trimmed = reason.trim();
|
||||
const start = trimmed.indexOf('{');
|
||||
const end = trimmed.lastIndexOf('}');
|
||||
if (start < 0 || end < start) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(trimmed.slice(start, end + 1));
|
||||
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? (parsed as Record<string, unknown>) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function stringField(record: Record<string, unknown>, key: string): string | null {
|
||||
const value = record[key];
|
||||
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
function isGoogleReauthFailure(record: Record<string, unknown>): boolean {
|
||||
const error = stringField(record, 'error')?.toLowerCase() ?? '';
|
||||
const description = stringField(record, 'error_description')?.toLowerCase() ?? '';
|
||||
const subtype = stringField(record, 'error_subtype')?.toLowerCase() ?? '';
|
||||
return error === 'invalid_grant' && (description.includes('reauth') || subtype === 'invalid_rapt');
|
||||
}
|
||||
|
||||
function formatFailureReason(sourceKey: string, reason: string): string {
|
||||
const parsed = jsonObjectFromFailureReason(reason);
|
||||
if (!parsed) {
|
||||
return sanitizeMemoryFlowError(reason);
|
||||
}
|
||||
|
||||
if (sourceKey === 'historic-sql' && isGoogleReauthFailure(parsed)) {
|
||||
return 'Google Cloud authentication failed while analyzing query history: application-default credentials expired or require reauthentication (invalid_grant / invalid_rapt). Run `gcloud auth application-default login`, then retry.';
|
||||
}
|
||||
|
||||
const error = stringField(parsed, 'error');
|
||||
const description = stringField(parsed, 'error_description');
|
||||
const subtype = stringField(parsed, 'error_subtype');
|
||||
const parts = [error, description].filter((part): part is string => Boolean(part));
|
||||
const message = parts.length > 0 ? parts.join(': ') : reason;
|
||||
return subtype ? `${message} (${subtype})` : message;
|
||||
}
|
||||
|
||||
function failedReportMessage(report: IngestReportSnapshot): string | null {
|
||||
const failedCount = report.body.failedWorkUnits.length;
|
||||
if (failedCount === 0) {
|
||||
return null;
|
||||
}
|
||||
const firstFailure = report.body.workUnits.find(
|
||||
(workUnit) => workUnit.status === 'failed' && typeof workUnit.reason === 'string' && workUnit.reason.trim(),
|
||||
);
|
||||
const sourceLabel = reportSourceLabel(report.sourceKey);
|
||||
const prefix = `${sourceLabel} failed for ${pluralize(failedCount, 'task')}.`;
|
||||
if (!firstFailure?.reason) {
|
||||
return prefix;
|
||||
}
|
||||
return `${prefix} First failure: ${formatFailureReason(report.sourceKey, firstFailure.reason)}`;
|
||||
}
|
||||
|
||||
function writeReportStatus(report: IngestReportSnapshot, io: KtxIngestIo): void {
|
||||
const counts = savedMemoryCountsForReport(report);
|
||||
const failedMessage = failedReportMessage(report);
|
||||
io.stdout.write(`Report: ${report.id}\n`);
|
||||
io.stdout.write(`Run: ${report.runId}\n`);
|
||||
io.stdout.write(`Job: ${report.jobId}\n`);
|
||||
|
|
@ -140,6 +203,12 @@ function writeReportStatus(report: IngestReportSnapshot, io: KtxIngestIo): void
|
|||
`Diff: +${report.body.diffSummary.added}/~${report.body.diffSummary.modified}/-${report.body.diffSummary.deleted}/=${report.body.diffSummary.unchanged}\n`,
|
||||
);
|
||||
io.stdout.write(`Tasks: ${report.body.workUnits.length}\n`);
|
||||
if (report.body.failedWorkUnits.length > 0) {
|
||||
io.stdout.write(`Failed tasks: ${report.body.failedWorkUnits.length}\n`);
|
||||
}
|
||||
if (failedMessage) {
|
||||
io.stdout.write(`Error: ${failedMessage}\n`);
|
||||
}
|
||||
io.stdout.write(`Saved memory: ${counts.wikiCount} wiki, ${counts.slCount} SL\n`);
|
||||
io.stdout.write(`Provenance rows: ${report.body.provenanceRows.length}\n`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import {
|
|||
type KtxPublicIngestProject,
|
||||
runKtxPublicIngest,
|
||||
} from './public-ingest.js';
|
||||
import type { ManagedPythonCommandRuntime } from './managed-python-command.js';
|
||||
|
||||
function makeIo(options: { isTTY?: boolean; interactive?: boolean } = {}) {
|
||||
let stdout = '';
|
||||
|
|
@ -750,6 +751,53 @@ describe('runKtxPublicIngest', () => {
|
|||
expect(runScan).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('preflights foreground query-history runtime before starting the context-build view', async () => {
|
||||
const io = makeIo({ isTTY: true, interactive: true });
|
||||
const calls: string[] = [];
|
||||
const project = projectWithConnections({
|
||||
warehouse: { driver: 'postgres', context: { depth: 'deep' } },
|
||||
});
|
||||
const ensureRuntime = vi.fn(async (): Promise<ManagedPythonCommandRuntime> => {
|
||||
calls.push('runtime');
|
||||
return {} as ManagedPythonCommandRuntime;
|
||||
});
|
||||
const runContextBuild = vi.fn(async () => {
|
||||
calls.push('context-build');
|
||||
return { exitCode: 0 };
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxPublicIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: '/tmp/project',
|
||||
targetConnectionId: 'warehouse',
|
||||
all: false,
|
||||
json: false,
|
||||
inputMode: 'auto',
|
||||
queryHistory: 'enabled',
|
||||
cliVersion: '0.2.0',
|
||||
runtimeInstallPolicy: 'prompt',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
loadProject: vi.fn(async () => project),
|
||||
ensureRuntime,
|
||||
runContextBuild,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(calls).toEqual(['runtime', 'context-build']);
|
||||
expect(ensureRuntime).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
cliVersion: '0.2.0',
|
||||
installPolicy: 'prompt',
|
||||
feature: 'core',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('runs all independent targets and reports partial failures', async () => {
|
||||
const io = makeIo();
|
||||
const project = projectWithConnections({
|
||||
|
|
@ -806,7 +854,12 @@ describe('runKtxPublicIngest', () => {
|
|||
warehouse: { driver: 'postgres', context: { depth: 'deep' } },
|
||||
});
|
||||
const runScan = vi.fn(async () => 0);
|
||||
const runIngest = vi.fn(async () => 1);
|
||||
const runIngest = vi.fn(async (_args, ingestIo) => {
|
||||
ingestIo.stdout.write(
|
||||
'Error: Query history failed for 60 tasks. First failure: Google Cloud authentication failed while analyzing query history: application-default credentials expired or require reauthentication (invalid_grant / invalid_rapt). Run `gcloud auth application-default login`, then retry.\n',
|
||||
);
|
||||
return 1;
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxPublicIngest(
|
||||
|
|
@ -824,7 +877,11 @@ describe('runKtxPublicIngest', () => {
|
|||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(io.stdout()).toContain('warehouse failed at query-history.');
|
||||
expect(io.stdout()).toMatch(/warehouse\s+done\s+failed\s+skipped\s+skipped/);
|
||||
expect(io.stdout()).toContain(
|
||||
'warehouse failed: Query history failed for 60 tasks. First failure: Google Cloud authentication failed while analyzing query history',
|
||||
);
|
||||
expect(io.stdout()).not.toContain('warehouse failed: Error:');
|
||||
expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --deep --query-history');
|
||||
expect(io.stdout()).not.toContain('historic-sql');
|
||||
});
|
||||
|
|
|
|||
|
|
@ -9,8 +9,14 @@ import {
|
|||
isDatabaseDriver,
|
||||
normalizeConnectionDriver,
|
||||
} from './ingest-depth.js';
|
||||
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
|
||||
import {
|
||||
ensureManagedPythonCommandRuntime,
|
||||
type KtxManagedPythonInstallPolicy,
|
||||
type ManagedPythonCommandRuntime,
|
||||
} from './managed-python-command.js';
|
||||
import type { KtxRuntimeFeature } from './managed-python-runtime.js';
|
||||
import { publicIngestOutputLine } from './public-ingest-copy.js';
|
||||
import { resolvePublicIngestRuntimeRequirements } from './runtime-requirements.js';
|
||||
import type { KtxScanArgs, KtxScanDeps } from './scan.js';
|
||||
import { profileMark } from './startup-profile.js';
|
||||
|
||||
|
|
@ -94,6 +100,13 @@ export interface KtxPublicIngestDeps {
|
|||
) => Promise<{ exitCode: number }>;
|
||||
scanProgress?: KtxProgressPort;
|
||||
ingestProgress?: (update: KtxIngestProgressUpdate) => void;
|
||||
ensureRuntime?: (options: {
|
||||
cliVersion: string;
|
||||
installPolicy: KtxManagedPythonInstallPolicy;
|
||||
io: KtxCliIo;
|
||||
feature: KtxRuntimeFeature;
|
||||
}) => Promise<ManagedPythonCommandRuntime>;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
runtimeIo?: KtxCliIo;
|
||||
onPhaseStart?: (phaseKey: KtxPublicIngestPhaseKey) => void;
|
||||
onPhaseEnd?: (phaseKey: KtxPublicIngestPhaseKey, status: 'done' | 'failed' | 'skipped', summary?: string) => void;
|
||||
|
|
@ -555,6 +568,7 @@ function markTargetResult(
|
|||
): KtxPublicIngestTargetResult {
|
||||
const selectedFailedOperation =
|
||||
failedOperation ?? (target.operation === 'database-ingest' ? 'database-schema' : 'source-ingest');
|
||||
const selectedFailedOperationIndex = target.steps.indexOf(selectedFailedOperation);
|
||||
return {
|
||||
connectionId: target.connectionId,
|
||||
driver: target.driver,
|
||||
|
|
@ -565,6 +579,10 @@ function markTargetResult(
|
|||
if (status === 'done') {
|
||||
return { ...step, status: 'done' };
|
||||
}
|
||||
const stepIndex = target.steps.indexOf(step.operation);
|
||||
if (selectedFailedOperationIndex >= 0 && stepIndex >= 0 && stepIndex < selectedFailedOperationIndex) {
|
||||
return { ...step, status: 'done' };
|
||||
}
|
||||
if (step.operation === selectedFailedOperation) {
|
||||
return {
|
||||
...step,
|
||||
|
|
@ -667,6 +685,10 @@ const ACTIONABLE_FAILURE_LINE_RE =
|
|||
/^(Missing bundled Python runtime manifest|KTX Python runtime is required|KTX managed daemon|Error:|Failed\b|Could not\b|Cannot\b)/;
|
||||
const RUNTIME_BACKED_RETRY_LINE_RE = /^Then retry the runtime-backed KTX command\.?$/;
|
||||
|
||||
function trimErrorPrefix(line: string): string {
|
||||
return line.replace(/^Error:\s*/, '');
|
||||
}
|
||||
|
||||
function capturedFailureMessage(output: string): string | undefined {
|
||||
const lines = output
|
||||
.split(/\r?\n/)
|
||||
|
|
@ -678,12 +700,13 @@ function capturedFailureMessage(output: string): string | undefined {
|
|||
|
||||
const actionableIndex = lines.findIndex((line) => ACTIONABLE_FAILURE_LINE_RE.test(line));
|
||||
if (actionableIndex < 0) {
|
||||
return lines.find((line) => line.length > 0);
|
||||
const line = lines.find((candidate) => candidate.length > 0);
|
||||
return line ? trimErrorPrefix(line) : undefined;
|
||||
}
|
||||
|
||||
const firstLine = lines[actionableIndex];
|
||||
if (!firstLine?.startsWith('Missing bundled Python runtime manifest')) {
|
||||
return firstLine;
|
||||
return trimErrorPrefix(firstLine);
|
||||
}
|
||||
|
||||
const followupLines = lines
|
||||
|
|
@ -850,6 +873,22 @@ export async function runKtxPublicIngest(
|
|||
const loadProject = deps.loadProject ?? loadKtxProject;
|
||||
const project = await loadProject({ projectDir: args.projectDir });
|
||||
if (shouldUseForegroundContextBuildView(args, io)) {
|
||||
const plan = buildPublicIngestPlan(project, args);
|
||||
const requirements = resolvePublicIngestRuntimeRequirements(plan, { env: deps.env ?? process.env });
|
||||
const ensureRuntime = deps.ensureRuntime ?? ensureManagedPythonCommandRuntime;
|
||||
for (const feature of requirements.features) {
|
||||
try {
|
||||
await ensureRuntime({
|
||||
cliVersion: args.cliVersion ?? '0.0.0-private',
|
||||
installPolicy: args.runtimeInstallPolicy ?? 'prompt',
|
||||
io,
|
||||
feature,
|
||||
});
|
||||
} catch (error) {
|
||||
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
const { runContextBuild } = await import('./context-build-view.js');
|
||||
const contextBuild = deps.runContextBuild ?? runContextBuild;
|
||||
const result = await contextBuild(
|
||||
|
|
|
|||
81
packages/cli/src/runtime-requirements.test.ts
Normal file
81
packages/cli/src/runtime-requirements.test.ts
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context';
|
||||
import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from '@ktx/context/project';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
resolveProjectRuntimeRequirements,
|
||||
resolvePublicIngestRuntimeRequirements,
|
||||
} from './runtime-requirements.js';
|
||||
|
||||
describe('runtime requirement detection', () => {
|
||||
it('requires core for agent/MCP setup', () => {
|
||||
const config = buildDefaultKtxProjectConfig();
|
||||
|
||||
expect(resolveProjectRuntimeRequirements(config, { agents: true }).features).toEqual(['core']);
|
||||
});
|
||||
|
||||
it('requires core for Looker source ingest unless an external daemon is configured', () => {
|
||||
const config: KtxProjectConfig = {
|
||||
...buildDefaultKtxProjectConfig(),
|
||||
connections: {
|
||||
looker: { driver: 'looker', base_url: 'https://looker.example.com', client_id: 'client-id' },
|
||||
},
|
||||
};
|
||||
|
||||
expect(resolveProjectRuntimeRequirements(config).features).toEqual(['core']);
|
||||
expect(resolveProjectRuntimeRequirements(config, { env: { KTX_DAEMON_URL: 'http://127.0.0.1:8765' } }).features).toEqual(
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
it('requires core for query-history ingest unless SQL analysis is externally configured', () => {
|
||||
const config: KtxProjectConfig = {
|
||||
...buildDefaultKtxProjectConfig(),
|
||||
connections: {
|
||||
warehouse: { driver: 'postgres', context: { queryHistory: { enabled: true } } },
|
||||
},
|
||||
};
|
||||
|
||||
expect(resolveProjectRuntimeRequirements(config).features).toEqual(['core']);
|
||||
expect(
|
||||
resolveProjectRuntimeRequirements(config, { env: { KTX_SQL_ANALYSIS_URL: 'http://127.0.0.1:8765' } }).features,
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it('requires local-embeddings for managed sentence-transformers embeddings', () => {
|
||||
const config: KtxProjectConfig = {
|
||||
...buildDefaultKtxProjectConfig(),
|
||||
ingest: {
|
||||
...buildDefaultKtxProjectConfig().ingest,
|
||||
embeddings: {
|
||||
backend: 'sentence-transformers' as const,
|
||||
model: 'all-MiniLM-L6-v2',
|
||||
dimensions: 384,
|
||||
sentenceTransformers: {
|
||||
base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
expect(resolveProjectRuntimeRequirements(config).features).toEqual(['local-embeddings']);
|
||||
});
|
||||
|
||||
it('detects foreground ingest runtime needs from selected query-history targets', () => {
|
||||
expect(
|
||||
resolvePublicIngestRuntimeRequirements({
|
||||
projectDir: '/tmp/project',
|
||||
warnings: [],
|
||||
targets: [
|
||||
{
|
||||
connectionId: 'warehouse',
|
||||
driver: 'postgres',
|
||||
operation: 'database-ingest',
|
||||
debugCommand: 'ktx ingest warehouse --debug',
|
||||
steps: ['database-schema', 'query-history'],
|
||||
queryHistory: { enabled: true },
|
||||
},
|
||||
],
|
||||
}).features,
|
||||
).toEqual(['core']);
|
||||
});
|
||||
});
|
||||
168
packages/cli/src/runtime-requirements.ts
Normal file
168
packages/cli/src/runtime-requirements.ts
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context';
|
||||
import type {
|
||||
KtxProjectConfig,
|
||||
KtxProjectConnectionConfig,
|
||||
KtxProjectEmbeddingConfig,
|
||||
} from '@ktx/context/project';
|
||||
import type { KtxRuntimeFeature } from './managed-python-runtime.js';
|
||||
import type { KtxPublicIngestPlan } from './public-ingest.js';
|
||||
|
||||
type KtxRuntimeRequirementReason =
|
||||
| 'agent-mcp'
|
||||
| 'query-history'
|
||||
| 'looker-source'
|
||||
| 'database-introspection'
|
||||
| 'local-embeddings';
|
||||
|
||||
interface KtxRuntimeRequirement {
|
||||
feature: KtxRuntimeFeature;
|
||||
reason: KtxRuntimeRequirementReason;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
export interface KtxRuntimeRequirements {
|
||||
features: KtxRuntimeFeature[];
|
||||
requirements: KtxRuntimeRequirement[];
|
||||
}
|
||||
|
||||
export interface KtxProjectRuntimeRequirementOptions {
|
||||
agents?: boolean;
|
||||
databaseIntrospectionFallback?: boolean;
|
||||
env?: NodeJS.ProcessEnv | Record<string, string | undefined>;
|
||||
}
|
||||
|
||||
export interface KtxPublicIngestRuntimeRequirementOptions {
|
||||
env?: NodeJS.ProcessEnv | Record<string, string | undefined>;
|
||||
}
|
||||
|
||||
function normalizeDriver(driver: unknown): string {
|
||||
return String(driver ?? '').trim().toLowerCase();
|
||||
}
|
||||
|
||||
function recordValue(value: unknown): Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null ? (value as Record<string, unknown>) : {};
|
||||
}
|
||||
|
||||
function hasEnabledQueryHistory(connection: KtxProjectConnectionConfig): boolean {
|
||||
const context = recordValue(recordValue(connection).context);
|
||||
const queryHistory = recordValue(context.queryHistory);
|
||||
return queryHistory.enabled === true;
|
||||
}
|
||||
|
||||
function hasDaemonOverride(env: NodeJS.ProcessEnv | Record<string, string | undefined>): boolean {
|
||||
return typeof env.KTX_DAEMON_URL === 'string' && env.KTX_DAEMON_URL.trim().length > 0;
|
||||
}
|
||||
|
||||
function hasSqlAnalysisOverride(env: NodeJS.ProcessEnv | Record<string, string | undefined>): boolean {
|
||||
return (
|
||||
(typeof env.KTX_SQL_ANALYSIS_URL === 'string' && env.KTX_SQL_ANALYSIS_URL.trim().length > 0) ||
|
||||
hasDaemonOverride(env)
|
||||
);
|
||||
}
|
||||
|
||||
function requiresManagedLocalEmbeddings(embeddings: KtxProjectEmbeddingConfig): boolean {
|
||||
if (embeddings.backend !== 'sentence-transformers') {
|
||||
return false;
|
||||
}
|
||||
const baseUrl = embeddings.sentenceTransformers?.base_url;
|
||||
return baseUrl === undefined || baseUrl === '' || baseUrl === MANAGED_SENTENCE_TRANSFORMERS_BASE_URL;
|
||||
}
|
||||
|
||||
function uniqueRequirements(requirements: KtxRuntimeRequirement[]): KtxRuntimeRequirements {
|
||||
const seen = new Set<string>();
|
||||
const deduped: KtxRuntimeRequirement[] = [];
|
||||
for (const requirement of requirements) {
|
||||
const key = `${requirement.feature}:${requirement.reason}:${requirement.detail}`;
|
||||
if (seen.has(key)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(key);
|
||||
deduped.push(requirement);
|
||||
}
|
||||
const features = [...new Set(deduped.map((requirement) => requirement.feature))].sort((left, right) =>
|
||||
left.localeCompare(right),
|
||||
);
|
||||
return { features, requirements: deduped };
|
||||
}
|
||||
|
||||
export function resolveProjectRuntimeRequirements(
|
||||
config: KtxProjectConfig,
|
||||
options: KtxProjectRuntimeRequirementOptions = {},
|
||||
): KtxRuntimeRequirements {
|
||||
const env = options.env ?? process.env;
|
||||
const requirements: KtxRuntimeRequirement[] = [];
|
||||
|
||||
if (options.agents === true) {
|
||||
requirements.push({
|
||||
feature: 'core',
|
||||
reason: 'agent-mcp',
|
||||
detail: 'Agent MCP setup uses semantic-layer query tools and SQL validation.',
|
||||
});
|
||||
}
|
||||
|
||||
if (options.databaseIntrospectionFallback === true && !hasDaemonOverride(env)) {
|
||||
requirements.push({
|
||||
feature: 'core',
|
||||
reason: 'database-introspection',
|
||||
detail: 'Database introspection fallback uses the Python daemon.',
|
||||
});
|
||||
}
|
||||
|
||||
for (const [connectionId, connection] of Object.entries(config.connections)) {
|
||||
const driver = normalizeDriver(connection.driver);
|
||||
if ((driver === 'looker' || driver === 'local_looker') && !hasDaemonOverride(env)) {
|
||||
requirements.push({
|
||||
feature: 'core',
|
||||
reason: 'looker-source',
|
||||
detail: `${connectionId} uses Looker identifier parsing.`,
|
||||
});
|
||||
}
|
||||
|
||||
if (hasEnabledQueryHistory(connection) && !hasSqlAnalysisOverride(env)) {
|
||||
requirements.push({
|
||||
feature: 'core',
|
||||
reason: 'query-history',
|
||||
detail: `${connectionId} has query history enabled.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (requiresManagedLocalEmbeddings(config.ingest.embeddings)) {
|
||||
requirements.push({
|
||||
feature: 'local-embeddings',
|
||||
reason: 'local-embeddings',
|
||||
detail: 'Local sentence-transformers embeddings use the managed Python runtime.',
|
||||
});
|
||||
}
|
||||
|
||||
return uniqueRequirements(requirements);
|
||||
}
|
||||
|
||||
export function resolvePublicIngestRuntimeRequirements(
|
||||
plan: KtxPublicIngestPlan,
|
||||
options: KtxPublicIngestRuntimeRequirementOptions = {},
|
||||
): KtxRuntimeRequirements {
|
||||
const env = options.env ?? process.env;
|
||||
const requirements: KtxRuntimeRequirement[] = [];
|
||||
|
||||
for (const target of plan.targets) {
|
||||
const driver = normalizeDriver(target.driver);
|
||||
const adapter = normalizeDriver(target.adapter);
|
||||
if (target.queryHistory?.enabled === true && !hasSqlAnalysisOverride(env)) {
|
||||
requirements.push({
|
||||
feature: 'core',
|
||||
reason: 'query-history',
|
||||
detail: `${target.connectionId} query-history ingest uses SQL analysis.`,
|
||||
});
|
||||
}
|
||||
if ((driver === 'looker' || driver === 'local_looker' || adapter === 'looker') && !hasDaemonOverride(env)) {
|
||||
requirements.push({
|
||||
feature: 'core',
|
||||
reason: 'looker-source',
|
||||
detail: `${target.connectionId} uses Looker identifier parsing.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return uniqueRequirements(requirements);
|
||||
}
|
||||
|
|
@ -8,6 +8,7 @@ const readyStatus: KtxSetupStatus = {
|
|||
embeddings: { backend: 'openai', ready: true, model: 'text-embedding-3-small', dimensions: 1536 },
|
||||
databases: [{ connectionId: 'warehouse', ready: true }],
|
||||
sources: [],
|
||||
runtime: { required: false, ready: true, features: [] },
|
||||
context: { ready: true, status: 'completed' },
|
||||
agents: [{ target: 'codex', scope: 'project', ready: true }],
|
||||
};
|
||||
|
|
@ -16,6 +17,7 @@ describe('setup ready menu', () => {
|
|||
it('recognizes a ready setup only when required sections are ready', () => {
|
||||
expect(isKtxSetupReady(readyStatus)).toBe(true);
|
||||
expect(isKtxSetupReady({ ...readyStatus, embeddings: { ready: false } })).toBe(false);
|
||||
expect(isKtxSetupReady({ ...readyStatus, runtime: { required: true, ready: false, features: ['core'] } })).toBe(false);
|
||||
expect(isKtxSetupReady({ ...readyStatus, context: { ready: false, status: 'not_started' } })).toBe(false);
|
||||
expect(isKtxSetupReady({ ...readyStatus, agents: [] })).toBe(false);
|
||||
});
|
||||
|
|
@ -24,6 +26,9 @@ describe('setup ready menu', () => {
|
|||
expect(isKtxPreAgentSetupReady(readyStatus)).toBe(true);
|
||||
expect(isKtxPreAgentSetupReady({ ...readyStatus, agents: [] })).toBe(true);
|
||||
expect(isKtxPreAgentSetupReady({ ...readyStatus, embeddings: { ready: false } })).toBe(false);
|
||||
expect(isKtxPreAgentSetupReady({ ...readyStatus, runtime: { required: true, ready: false, features: ['core'] } })).toBe(
|
||||
false,
|
||||
);
|
||||
expect(isKtxPreAgentSetupReady({ ...readyStatus, context: { ready: false, status: 'not_started' } })).toBe(false);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,15 @@ import {
|
|||
} from './setup-prompts.js';
|
||||
import type { KtxSetupStatus } from './setup.js';
|
||||
|
||||
export type KtxSetupReadyAction = 'models' | 'embeddings' | 'databases' | 'sources' | 'context' | 'agents' | 'exit';
|
||||
export type KtxSetupReadyAction =
|
||||
| 'models'
|
||||
| 'embeddings'
|
||||
| 'databases'
|
||||
| 'sources'
|
||||
| 'runtime'
|
||||
| 'context'
|
||||
| 'agents'
|
||||
| 'exit';
|
||||
|
||||
export interface KtxSetupReadyMenuPromptAdapter {
|
||||
select(options: { message: string; options: KtxSetupPromptOption[] }): Promise<string>;
|
||||
|
|
@ -22,6 +30,7 @@ export function isKtxPreAgentSetupReady(status: KtxSetupStatus): boolean {
|
|||
status.embeddings.ready &&
|
||||
status.databases.every((database) => database.ready) &&
|
||||
status.sources.every((source) => source.ready) &&
|
||||
status.runtime.ready &&
|
||||
status.context.ready
|
||||
);
|
||||
}
|
||||
|
|
@ -46,6 +55,7 @@ export async function runKtxSetupReadyChangeMenu(
|
|||
{ value: 'embeddings', label: 'Embeddings' },
|
||||
{ value: 'databases', label: 'Databases' },
|
||||
{ value: 'sources', label: 'Context sources' },
|
||||
...(status.runtime.required ? [{ value: 'runtime', label: 'Runtime' }] : []),
|
||||
{ value: 'context', label: 'Rebuild KTX context' },
|
||||
{ value: 'agents', label: 'Agent integration' },
|
||||
{ value: 'exit', label: 'Exit' },
|
||||
|
|
|
|||
153
packages/cli/src/setup-runtime.test.ts
Normal file
153
packages/cli/src/setup-runtime.test.ts
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
import { mkdtemp, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context';
|
||||
import { buildDefaultKtxProjectConfig, readKtxSetupState, type KtxProjectConfig } from '@ktx/context/project';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { ManagedPythonCommandRuntime } from './managed-python-command.js';
|
||||
import { runKtxSetupRuntimeStep } from './setup-runtime.js';
|
||||
|
||||
function makeIo() {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: {
|
||||
write: (chunk: string) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write: (chunk: string) => {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
function projectConfig(config: KtxProjectConfig) {
|
||||
return vi.fn(async () => ({ config }));
|
||||
}
|
||||
|
||||
describe('runKtxSetupRuntimeStep', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-setup-runtime-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('ensures core runtime for agent setup and records the runtime step', async () => {
|
||||
const io = makeIo();
|
||||
const ensureRuntime = vi.fn(async (): Promise<ManagedPythonCommandRuntime> => ({} as ManagedPythonCommandRuntime));
|
||||
|
||||
await expect(
|
||||
runKtxSetupRuntimeStep(
|
||||
{
|
||||
projectDir: tempDir,
|
||||
inputMode: 'auto',
|
||||
cliVersion: '0.2.0',
|
||||
runtimeInstallPolicy: 'prompt',
|
||||
agents: true,
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
loadProject: projectConfig(buildDefaultKtxProjectConfig()),
|
||||
ensureRuntime,
|
||||
env: {},
|
||||
},
|
||||
),
|
||||
).resolves.toMatchObject({ status: 'ready' });
|
||||
|
||||
expect(ensureRuntime).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
cliVersion: '0.2.0',
|
||||
installPolicy: 'prompt',
|
||||
feature: 'core',
|
||||
}),
|
||||
);
|
||||
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('runtime');
|
||||
expect(io.stdout()).toContain('Runtime ready: yes (core)');
|
||||
});
|
||||
|
||||
it('fails fast when required runtime features cannot be installed in no-input mode', async () => {
|
||||
const io = makeIo();
|
||||
const ensureRuntime = vi.fn(async () => {
|
||||
throw new Error('KTX Python runtime is required for this command. Run: ktx dev runtime install --yes');
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetupRuntimeStep(
|
||||
{
|
||||
projectDir: tempDir,
|
||||
inputMode: 'disabled',
|
||||
cliVersion: '0.2.0',
|
||||
runtimeInstallPolicy: 'never',
|
||||
agents: true,
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
loadProject: projectConfig(buildDefaultKtxProjectConfig()),
|
||||
ensureRuntime,
|
||||
env: {},
|
||||
},
|
||||
),
|
||||
).resolves.toMatchObject({ status: 'failed' });
|
||||
|
||||
expect(ensureRuntime).toHaveBeenCalledWith(expect.objectContaining({ installPolicy: 'never' }));
|
||||
expect((await readKtxSetupState(tempDir)).completed_steps).not.toContain('runtime');
|
||||
expect(io.stderr()).toContain('ktx dev runtime install --yes');
|
||||
});
|
||||
|
||||
it('starts the managed local embeddings daemon for configured sentence-transformers embeddings', async () => {
|
||||
const io = makeIo();
|
||||
const ensureLocalEmbeddings = vi.fn(async () => ({
|
||||
baseUrl: 'http://127.0.0.1:61234',
|
||||
env: { KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL: 'http://127.0.0.1:61234' },
|
||||
}));
|
||||
const config: KtxProjectConfig = {
|
||||
...buildDefaultKtxProjectConfig(),
|
||||
ingest: {
|
||||
...buildDefaultKtxProjectConfig().ingest,
|
||||
embeddings: {
|
||||
backend: 'sentence-transformers',
|
||||
model: 'all-MiniLM-L6-v2',
|
||||
dimensions: 384,
|
||||
sentenceTransformers: { base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
await expect(
|
||||
runKtxSetupRuntimeStep(
|
||||
{
|
||||
projectDir: tempDir,
|
||||
inputMode: 'auto',
|
||||
cliVersion: '0.2.0',
|
||||
runtimeInstallPolicy: 'auto',
|
||||
agents: false,
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
loadProject: projectConfig(config),
|
||||
ensureLocalEmbeddings,
|
||||
env: {},
|
||||
},
|
||||
),
|
||||
).resolves.toMatchObject({ status: 'ready' });
|
||||
|
||||
expect(ensureLocalEmbeddings).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
projectDir: tempDir,
|
||||
installPolicy: 'auto',
|
||||
}),
|
||||
);
|
||||
expect(io.stdout()).toContain('Runtime ready: yes (local embeddings)');
|
||||
});
|
||||
});
|
||||
103
packages/cli/src/setup-runtime.ts
Normal file
103
packages/cli/src/setup-runtime.ts
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
import {
|
||||
loadKtxProject,
|
||||
markKtxSetupStateStepComplete,
|
||||
type KtxLocalProject,
|
||||
} from '@ktx/context/project';
|
||||
import type { KtxCliIo } from './cli-runtime.js';
|
||||
import {
|
||||
ensureManagedLocalEmbeddingsDaemon,
|
||||
type ManagedLocalEmbeddingsDaemon,
|
||||
} from './managed-local-embeddings.js';
|
||||
import {
|
||||
ensureManagedPythonCommandRuntime,
|
||||
type KtxManagedPythonInstallPolicy,
|
||||
type ManagedPythonCommandRuntime,
|
||||
} from './managed-python-command.js';
|
||||
import type { KtxRuntimeFeature } from './managed-python-runtime.js';
|
||||
import {
|
||||
resolveProjectRuntimeRequirements,
|
||||
type KtxRuntimeRequirements,
|
||||
} from './runtime-requirements.js';
|
||||
|
||||
export interface KtxSetupRuntimeArgs {
|
||||
projectDir: string;
|
||||
inputMode: 'auto' | 'disabled';
|
||||
cliVersion: string;
|
||||
runtimeInstallPolicy: KtxManagedPythonInstallPolicy;
|
||||
agents: boolean;
|
||||
databaseIntrospectionFallback?: boolean;
|
||||
}
|
||||
|
||||
export type KtxSetupRuntimeResult =
|
||||
| { status: 'ready'; projectDir: string; requirements: KtxRuntimeRequirements }
|
||||
| { status: 'skipped'; projectDir: string; requirements: KtxRuntimeRequirements }
|
||||
| { status: 'failed'; projectDir: string; requirements: KtxRuntimeRequirements };
|
||||
|
||||
export interface KtxSetupRuntimeDeps {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
loadProject?: (options: { projectDir: string }) => Promise<Pick<KtxLocalProject, 'config'>>;
|
||||
ensureRuntime?: (options: {
|
||||
cliVersion: string;
|
||||
installPolicy: KtxManagedPythonInstallPolicy;
|
||||
io: KtxCliIo;
|
||||
feature: KtxRuntimeFeature;
|
||||
}) => Promise<ManagedPythonCommandRuntime>;
|
||||
ensureLocalEmbeddings?: (options: {
|
||||
cliVersion: string;
|
||||
projectDir: string;
|
||||
installPolicy: KtxManagedPythonInstallPolicy;
|
||||
io: KtxCliIo;
|
||||
}) => Promise<ManagedLocalEmbeddingsDaemon>;
|
||||
}
|
||||
|
||||
function formatRuntimeFeature(feature: KtxRuntimeFeature): string {
|
||||
return feature === 'local-embeddings' ? 'local embeddings' : 'core';
|
||||
}
|
||||
|
||||
export async function runKtxSetupRuntimeStep(
|
||||
args: KtxSetupRuntimeArgs,
|
||||
io: KtxCliIo,
|
||||
deps: KtxSetupRuntimeDeps = {},
|
||||
): Promise<KtxSetupRuntimeResult> {
|
||||
const loadProjectForRuntime = deps.loadProject ?? loadKtxProject;
|
||||
const project = await loadProjectForRuntime({ projectDir: args.projectDir });
|
||||
const requirements = resolveProjectRuntimeRequirements(project.config, {
|
||||
agents: args.agents,
|
||||
databaseIntrospectionFallback: args.databaseIntrospectionFallback,
|
||||
env: deps.env ?? process.env,
|
||||
});
|
||||
|
||||
if (requirements.features.length === 0) {
|
||||
io.stdout.write('│ Runtime setup skipped.\n');
|
||||
return { status: 'skipped', projectDir: args.projectDir, requirements };
|
||||
}
|
||||
|
||||
const ensureRuntime = deps.ensureRuntime ?? ensureManagedPythonCommandRuntime;
|
||||
const ensureLocalEmbeddings = deps.ensureLocalEmbeddings ?? ensureManagedLocalEmbeddingsDaemon;
|
||||
try {
|
||||
for (const feature of requirements.features) {
|
||||
if (feature === 'local-embeddings') {
|
||||
await ensureLocalEmbeddings({
|
||||
cliVersion: args.cliVersion,
|
||||
projectDir: args.projectDir,
|
||||
installPolicy: args.runtimeInstallPolicy,
|
||||
io,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
await ensureRuntime({
|
||||
cliVersion: args.cliVersion,
|
||||
installPolicy: args.runtimeInstallPolicy,
|
||||
io,
|
||||
feature,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
return { status: 'failed', projectDir: args.projectDir, requirements };
|
||||
}
|
||||
|
||||
await markKtxSetupStateStepComplete(args.projectDir, 'runtime');
|
||||
io.stdout.write(`│ Runtime ready: yes (${requirements.features.map(formatRuntimeFeature).join(', ')})\n`);
|
||||
return { status: 'ready', projectDir: args.projectDir, requirements };
|
||||
}
|
||||
|
|
@ -1054,7 +1054,7 @@ describe('setup status', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('auto-installs the managed runtime by default during setup', async () => {
|
||||
it('prompts before installing the managed runtime by default during setup', async () => {
|
||||
const io = makeIo();
|
||||
const embeddings = vi.fn(async () => ({ status: 'ready' as const, projectDir: tempDir }));
|
||||
const context = vi.fn(async () => ({ status: 'failed' as const, projectDir: tempDir }));
|
||||
|
|
@ -1088,14 +1088,14 @@ describe('setup status', () => {
|
|||
expect(embeddings).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
cliVersion: '0.2.0',
|
||||
runtimeInstallPolicy: 'auto',
|
||||
runtimeInstallPolicy: 'prompt',
|
||||
}),
|
||||
io.io,
|
||||
);
|
||||
expect(context).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
cliVersion: '0.2.0',
|
||||
runtimeInstallPolicy: 'auto',
|
||||
runtimeInstallPolicy: 'prompt',
|
||||
}),
|
||||
io.io,
|
||||
);
|
||||
|
|
@ -1508,6 +1508,10 @@ describe('setup status', () => {
|
|||
calls.push('sources');
|
||||
return { status: 'skipped', projectDir: tempDir };
|
||||
},
|
||||
runtime: async () => {
|
||||
calls.push('runtime');
|
||||
return { status: 'ready', projectDir: tempDir, requirements: { features: ['core'], requirements: [] } };
|
||||
},
|
||||
context: async () => {
|
||||
calls.push('context');
|
||||
return { status: 'ready', projectDir: tempDir, runId: 'setup-context-local-test' };
|
||||
|
|
@ -1524,7 +1528,7 @@ describe('setup status', () => {
|
|||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(calls).toEqual(['model', 'embeddings', 'databases', 'sources', 'context', 'agents']);
|
||||
expect(calls).toEqual(['model', 'embeddings', 'databases', 'sources', 'runtime', 'context', 'agents']);
|
||||
});
|
||||
|
||||
it('commits setup config changes written by later setup steps', async () => {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,9 @@ import {
|
|||
} from '@ktx/context/project';
|
||||
import type { KtxCliIo } from './cli-runtime.js';
|
||||
import { formatSetupNextStepLines } from './next-steps.js';
|
||||
import { runtimeInstallPolicyFromFlags } from './managed-python-command.js';
|
||||
import { readManagedPythonRuntimeStatus } from './managed-python-runtime.js';
|
||||
import { resolveProjectRuntimeRequirements } from './runtime-requirements.js';
|
||||
import { isKtxSetupExitError } from './setup-interrupt.js';
|
||||
import {
|
||||
type KtxAgentScope,
|
||||
|
|
@ -37,6 +40,11 @@ import {
|
|||
runKtxSetupReadyChangeMenu,
|
||||
} from './setup-ready-menu.js';
|
||||
import { type KtxSetupSourcesDeps, type KtxSetupSourceType, runKtxSetupSourcesStep } from './setup-sources.js';
|
||||
import {
|
||||
type KtxSetupRuntimeDeps,
|
||||
type KtxSetupRuntimeResult,
|
||||
runKtxSetupRuntimeStep,
|
||||
} from './setup-runtime.js';
|
||||
import {
|
||||
createKtxSetupPromptAdapter,
|
||||
createKtxSetupUiAdapter,
|
||||
|
|
@ -58,6 +66,7 @@ export interface KtxSetupStatus {
|
|||
embeddings: { backend?: string; ready: boolean; model?: string; dimensions?: number };
|
||||
databases: Array<{ connectionId: string; ready: boolean }>;
|
||||
sources: Array<{ connectionId: string; type: string; ready: boolean }>;
|
||||
runtime: { required: boolean; ready: boolean; features: string[]; detail?: string };
|
||||
context: KtxSetupContextStatusSummary;
|
||||
agents: Array<{ target: string; scope: string; ready: boolean }>;
|
||||
}
|
||||
|
|
@ -143,6 +152,8 @@ export interface KtxSetupDeps {
|
|||
io: KtxCliIo,
|
||||
) => Promise<Awaited<ReturnType<typeof runKtxSetupSourcesStep>>>;
|
||||
sourcesDeps?: KtxSetupSourcesDeps;
|
||||
runtime?: (args: Parameters<typeof runKtxSetupRuntimeStep>[0], io: KtxCliIo) => Promise<KtxSetupRuntimeResult>;
|
||||
runtimeDeps?: KtxSetupRuntimeDeps;
|
||||
agents?: (
|
||||
args: Parameters<typeof runKtxSetupAgentsStep>[0],
|
||||
io: KtxCliIo,
|
||||
|
|
@ -158,7 +169,7 @@ export interface KtxSetupDeps {
|
|||
const SOURCE_DRIVERS = new Set(['dbt', 'metricflow', 'metabase', 'looker', 'lookml', 'notion']);
|
||||
|
||||
type KtxSetupEntryAction = 'setup' | 'new-project' | 'agents' | 'status' | 'demo' | 'exit';
|
||||
type KtxSetupFlowStep = 'models' | 'embeddings' | 'databases' | 'sources' | 'context' | 'agents';
|
||||
type KtxSetupFlowStep = 'models' | 'embeddings' | 'databases' | 'sources' | 'runtime' | 'context' | 'agents';
|
||||
type KtxSetupFlowStatus =
|
||||
| 'ready'
|
||||
| 'skipped'
|
||||
|
|
@ -269,7 +280,16 @@ async function readIngestContextStatus(project: KtxLocalProject): Promise<KtxSet
|
|||
};
|
||||
}
|
||||
|
||||
export async function readKtxSetupStatus(projectDir: string): Promise<KtxSetupStatus> {
|
||||
export interface ReadKtxSetupStatusOptions {
|
||||
cliVersion?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
readRuntimeStatus?: typeof readManagedPythonRuntimeStatus;
|
||||
}
|
||||
|
||||
export async function readKtxSetupStatus(
|
||||
projectDir: string,
|
||||
options: ReadKtxSetupStatusOptions = {},
|
||||
): Promise<KtxSetupStatus> {
|
||||
const resolvedProjectDir = resolve(projectDir);
|
||||
if (!existsSync(join(resolvedProjectDir, 'ktx.yaml'))) {
|
||||
return {
|
||||
|
|
@ -278,6 +298,7 @@ export async function readKtxSetupStatus(projectDir: string): Promise<KtxSetupSt
|
|||
embeddings: { ready: false },
|
||||
databases: [],
|
||||
sources: [],
|
||||
runtime: { required: false, ready: true, features: [] },
|
||||
context: setupContextStatusFromState(await readKtxSetupContextState(resolvedProjectDir)),
|
||||
agents: [],
|
||||
};
|
||||
|
|
@ -316,6 +337,21 @@ export async function readKtxSetupStatus(projectDir: string): Promise<KtxSetupSt
|
|||
});
|
||||
}
|
||||
const agents = [...agentMap.values()];
|
||||
const runtimeRequirements = resolveProjectRuntimeRequirements(project.config, {
|
||||
agents: agents.length > 0,
|
||||
env: options.env ?? process.env,
|
||||
});
|
||||
let runtimeReady = runtimeRequirements.features.length === 0 || completedSteps.includes('runtime');
|
||||
let runtimeDetail: string | undefined;
|
||||
if (runtimeRequirements.features.length > 0 && options.cliVersion) {
|
||||
const readRuntimeStatus = options.readRuntimeStatus ?? readManagedPythonRuntimeStatus;
|
||||
const runtimeStatus = await readRuntimeStatus({ cliVersion: options.cliVersion, env: options.env ?? process.env });
|
||||
runtimeDetail = runtimeStatus.detail;
|
||||
runtimeReady =
|
||||
runtimeStatus.kind === 'ready' &&
|
||||
runtimeStatus.manifest !== undefined &&
|
||||
runtimeRequirements.features.every((feature) => runtimeStatus.manifest?.features.includes(feature));
|
||||
}
|
||||
|
||||
return {
|
||||
project: { path: resolvedProjectDir, ready: true, name: basename(project.projectDir) || project.projectDir },
|
||||
|
|
@ -329,6 +365,12 @@ export async function readKtxSetupStatus(projectDir: string): Promise<KtxSetupSt
|
|||
...source,
|
||||
ready: completedSteps.includes('sources'),
|
||||
})),
|
||||
runtime: {
|
||||
required: runtimeRequirements.features.length > 0,
|
||||
ready: runtimeReady,
|
||||
features: runtimeRequirements.features,
|
||||
...(runtimeDetail ? { detail: runtimeDetail } : {}),
|
||||
},
|
||||
context: ingestContextStatus ?? setupContextStatus,
|
||||
agents,
|
||||
};
|
||||
|
|
@ -374,6 +416,13 @@ export function formatKtxSetupStatus(status: KtxSetupStatus): string {
|
|||
}`,
|
||||
`Databases configured: ${formatConnectionList(status.databases.map((database) => database.connectionId))}`,
|
||||
`Context sources configured: ${formatConnectionList(status.sources.map((source) => source.connectionId))}`,
|
||||
...(status.runtime.required
|
||||
? [
|
||||
`Runtime ready: ${formatReady(status.runtime.ready)}${
|
||||
status.runtime.features.length > 0 ? ` (${status.runtime.features.join(', ')})` : ''
|
||||
}`,
|
||||
]
|
||||
: []),
|
||||
`KTX context built: ${formatContextBuilt(status.context)}`,
|
||||
`Agent integration ready: ${formatReady(status.agents.some((agent) => agent.ready))}${
|
||||
status.agents.length > 0 ? ` (${status.agents.map((agent) => `${agent.target}:${agent.scope}`).join(', ')})` : ''
|
||||
|
|
@ -397,7 +446,8 @@ function setupStatusReady(status: KtxSetupStatus): boolean {
|
|||
status.llm.ready &&
|
||||
embeddingsReady(status.embeddings) &&
|
||||
status.databases.every((database) => database.ready) &&
|
||||
status.sources.every((source) => source.ready)
|
||||
status.sources.every((source) => source.ready) &&
|
||||
status.runtime.ready
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -416,7 +466,10 @@ function writeContextNotReadyForAgents(projectDir: string, io: KtxCliIo): void {
|
|||
}
|
||||
|
||||
function setupRuntimeInstallPolicy(args: Extract<KtxSetupArgs, { command: 'run' }>): 'prompt' | 'auto' | 'never' {
|
||||
return args.inputMode === 'disabled' && !args.yes ? 'never' : 'auto';
|
||||
if (args.yes) {
|
||||
return 'auto';
|
||||
}
|
||||
return runtimeInstallPolicyFromFlags({ input: args.inputMode === 'disabled' ? false : true });
|
||||
}
|
||||
|
||||
async function commitSetupConfigChanges(projectDir: string): Promise<void> {
|
||||
|
|
@ -449,7 +502,7 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup
|
|||
setupLoop: while (true) {
|
||||
entryAction = undefined;
|
||||
if (canShowEntryMenu) {
|
||||
const status = await readKtxSetupStatus(args.projectDir);
|
||||
const status = await readKtxSetupStatus(args.projectDir, { cliVersion: args.cliVersion });
|
||||
entryAction = (await runKtxSetupEntryMenu(status, deps.entryMenuDeps)).action;
|
||||
if (entryAction === 'exit') {
|
||||
(deps.entryMenuDeps?.prompts ?? createEntryMenuPromptAdapter()).cancel('Setup cancelled.');
|
||||
|
|
@ -486,7 +539,7 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup
|
|||
}
|
||||
|
||||
const agentsRequested = args.agents || entryAction === 'agents';
|
||||
const currentStatus = await readKtxSetupStatus(projectResult.projectDir);
|
||||
const currentStatus = await readKtxSetupStatus(projectResult.projectDir, { cliVersion: args.cliVersion });
|
||||
let readyAction: string | undefined;
|
||||
|
||||
if (args.inputMode !== 'disabled' && !agentsRequested) {
|
||||
|
|
@ -503,13 +556,15 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup
|
|||
const shouldRunEmbeddings = !runOnly || runOnly === 'embeddings';
|
||||
const shouldRunDatabases = !runOnly || runOnly === 'databases';
|
||||
const shouldRunSources = !runOnly || runOnly === 'sources';
|
||||
const shouldRunRuntime =
|
||||
agentsRequested || !runOnly || runOnly === 'runtime' || runOnly === 'context' || runOnly === 'agents';
|
||||
const shouldRunContext = agentsRequested || !runOnly || runOnly === 'context';
|
||||
const shouldRunAgents = agentsRequested || !runOnly || runOnly === 'agents';
|
||||
const showPromptInstructions = projectResult.confirmedCreation !== true;
|
||||
|
||||
const setupSteps: KtxSetupFlowStep[] = agentsRequested
|
||||
? ['context']
|
||||
: ['models', 'embeddings', 'databases', 'sources', 'context'];
|
||||
? ['runtime', 'context']
|
||||
: ['models', 'embeddings', 'databases', 'sources', 'runtime', 'context'];
|
||||
if (shouldRunAgents && args.skipAgents !== true) {
|
||||
setupSteps.push('agents');
|
||||
}
|
||||
|
|
@ -520,6 +575,7 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup
|
|||
if (step === 'embeddings') return !args.skipEmbeddings && shouldRunEmbeddings;
|
||||
if (step === 'databases') return !args.skipDatabases && shouldRunDatabases;
|
||||
if (step === 'sources') return args.skipSources !== true && shouldRunSources;
|
||||
if (step === 'runtime') return shouldRunRuntime;
|
||||
if (step === 'context') return shouldRunContext;
|
||||
return shouldRunAgents && args.skipAgents !== true;
|
||||
};
|
||||
|
|
@ -636,6 +692,20 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup
|
|||
},
|
||||
io,
|
||||
);
|
||||
} else if (step === 'runtime') {
|
||||
const runtimeRunner =
|
||||
deps.runtime ??
|
||||
((runtimeArgs, runtimeIo) => runKtxSetupRuntimeStep(runtimeArgs, runtimeIo, deps.runtimeDeps));
|
||||
stepResult = await runtimeRunner(
|
||||
{
|
||||
projectDir: projectResult.projectDir,
|
||||
inputMode: args.inputMode,
|
||||
cliVersion: args.cliVersion,
|
||||
runtimeInstallPolicy: setupRuntimeInstallPolicy(args),
|
||||
agents: shouldRunAgents && args.skipAgents !== true,
|
||||
},
|
||||
io,
|
||||
);
|
||||
} else if (step === 'context') {
|
||||
const contextRunner =
|
||||
deps.context ??
|
||||
|
|
@ -706,7 +776,7 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup
|
|||
|
||||
await commitSetupConfigChanges(projectResult.projectDir);
|
||||
|
||||
const status = await readKtxSetupStatus(projectResult.projectDir);
|
||||
const status = await readKtxSetupStatus(projectResult.projectDir, { cliVersion: args.cliVersion });
|
||||
const focusedOnAgents = args.agents || entryAction === 'agents';
|
||||
if (!focusedOnAgents) {
|
||||
setupUi.note(formatKtxSetupStatus(status).trimEnd(), 'Project status', io, {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue