apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-07 07:55:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge scan into ingest flow

Browse source
This commit is contained in:
Andrey Avtomonov 2026-05-13 23:53:45 +02:00
parent 75e04cfa56
commit ecebc018b9
31 changed files with 747 additions and 216 deletions
Download patch file Download diff file Expand all files Collapse all files

35
docs/superpowers/specs/2026-05-13-unified-ingest-ux-design.md
View file

@ -87,33 +87,28 @@ The command dispatches by connection driver:
- `--all` runs database ingest targets first, then source ingest targets.
The old `ktx ingest run --connection-id <id> --adapter <adapter>` command is
not the primary public interface. The implementation plan can either remove it
or move it under an advanced/debug namespace. Normal users configure and ingest
removed from the public interface. Normal users configure and ingest
connections, not adapters.
`ktx scan` is no longer a documented public command. Database schema scanning
continues as an internal phase of database ingest.
Stored report inspection is separate from live context-build control.
`ktx ingest status [runId]`, `ktx ingest replay <runId>`, and `--report-file`
remain valid report-viewing surfaces unless the implementation plan replaces
them with an equivalent status command. `ktx ingest watch` is no longer a normal
public verb because `watch` conflicts with the foreground-only model. If a
stored-report visual replay remains useful, expose it as `replay` or hide it
under an advanced/debug namespace.
Stored report inspection is separate from live context-build control. The
public `ktx ingest` namespace has no subcommands, so `run`, `status`, `watch`,
and `replay` are ordinary connection IDs:
Any surviving `ktx ingest` subcommand reserves its command name as a connection
id. In v1, `status` and `replay` are reserved when those report-viewing
subcommands remain. `run` is also reserved while the old adapter-backed command
exists anywhere under `ktx ingest`, even if it is hidden or advanced. `watch`
is reserved until the live-watch command is removed or moved out of
`ktx ingest`. Setup and config validation must reject a connection id that
matches a reserved ingest subcommand with a clear message, such as:
```text
"status" is reserved for ktx ingest status; choose a different connection id.
```bash
ktx ingest run
ktx ingest status
ktx ingest watch
ktx ingest replay
```
No setup or config validation rejects those names. Old adapter-backed command
shapes such as `ktx ingest run --connection-id warehouse --adapter
live-database` fail through normal option parsing because `--connection-id` and
`--adapter` are not public `ktx ingest` options.
## Database ingest depth
Database ingest always includes a schema baseline. The depth controls how much
@ -593,8 +588,6 @@ The implementation plan must decide these lower-level details:
- Whether old `ktx scan` exits with an error, is hidden, or remains as a
temporary undocumented debug command.
- Whether old `ktx ingest run --connection-id ... --adapter ...` is removed,
hidden, or moved to `ktx dev ingest run`.
- Whether internal artifact paths keep `raw-sources/<connection>/live-database`
for the first implementation.
- Whether setup needs a headless `--context-depth fast|deep` flag for CI.

4
packages/cli/src/cli-program.ts
View file

@ -55,10 +55,6 @@ type CommandPathNode = CommandWithGlobalOptions & {
const PROJECT_AWARE_ROOT_COMMANDS = new Set(['setup', 'connection', 'ingest', 'wiki', 'sl', 'status']);
const REMOVED_COMMAND_PATHS = new Set([
'scan',
'ingest run',
'ingest status',
'ingest watch',
'ingest replay',
'wiki read',
'wiki write',
]);

3
packages/cli/src/commands/ingest-commands.ts
View file

@ -4,6 +4,7 @@ import {
parsePositiveIntegerOption,
resolveCommandProjectDir,
} from '../cli-program.js';
import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js';
import type { KtxPublicIngestArgs } from '../public-ingest.js';
import { profileMark } from '../startup-profile.js';
@ -41,6 +42,8 @@ export function registerIngestCommands(program: Command, context: KtxCliCommandC
...(options.deep === true ? { depth: 'deep' as const } : {}),
queryHistory,
...(options.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: options.queryHistoryWindowDays } : {}),
cliVersion: context.packageInfo.version,
runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options),
};
context.setExitCode(await (context.deps.publicIngest ?? runKtxPublicIngest)(args, context.io));
});

5
packages/cli/src/commands/setup-commands.ts
View file

@ -1,5 +1,4 @@
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
import { reservedKtxIngestConnectionIdMessage } from '@ktx/context/project';
import type { KtxCliCommandContext } from '../cli-program.js';
import { resolveCommandProjectDir } from '../cli-program.js';
import type { KtxSetupDatabaseDriver } from '../setup-databases.js';
@ -269,10 +268,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
}
const reservedMessage = reservedKtxIngestConnectionIdMessage(value);
if (reservedMessage) {
throw new InvalidArgumentError(reservedMessage);
}
return value;
})
.option('--database-url <url>', 'URL, env:NAME, or file:/path for one new URL-style database connection')

78
packages/cli/src/context-build-view.test.ts
View file

@ -552,6 +552,84 @@ describe('runContextBuild', () => {
expect(io.stdout()).not.toContain('BoundPool');
});
it('renders localhost SQL analysis refusal as a runtime failure during query history', async () => {
const io = makeIo();
const project = projectWithConnections({
warehouse: { driver: 'postgres', context: { depth: 'deep', queryHistory: { enabled: true } } },
});
const executeTarget = vi.fn(async (target, _args, targetIo) => {
targetIo.stderr.write('connect ECONNREFUSED 127.0.0.1:8765\n');
return {
connectionId: target.connectionId,
driver: target.driver,
steps: [
{ operation: 'database-schema', status: 'done' },
{ operation: 'query-history', status: 'failed', detail: 'warehouse failed at query-history.' },
{ operation: 'source-ingest', status: 'skipped' },
{ operation: 'memory-update', status: 'skipped' },
],
} satisfies KtxPublicIngestTargetResult;
});
const result = await runContextBuild(
project,
{ projectDir: '/tmp/project', inputMode: 'disabled' },
io.io,
{ executeTarget, now: () => 1000 },
);
expect(result).toEqual({ exitCode: 1 });
expect(io.stdout()).toContain(
'KTX could not reach the local SQL analysis runtime while processing query history for warehouse.',
);
expect(io.stdout()).toContain('connection refused (ECONNREFUSED)');
expect(io.stdout()).toContain('Retry: ktx setup --project-dir /tmp/project');
expect(io.stdout()).not.toContain('KTX lost its connection to PostgreSQL');
});
it('uses captured query-history stderr instead of generic failed-at detail', async () => {
const io = makeIo();
const project = projectWithConnections({
warehouse: { driver: 'postgres', context: { depth: 'deep', queryHistory: { enabled: true } } },
});
const executeTarget = vi.fn(async (target, _args, targetIo) => {
targetIo.stdout.write('KTX scan completed\n');
targetIo.stdout.write('Mode: enriched\n');
targetIo.stderr.write('Missing bundled Python runtime manifest: /tmp/assets/python/manifest.json\n');
targetIo.stderr.write('In a source checkout, build the local runtime assets with: pnpm run artifacts:build\n');
targetIo.stderr.write('Then retry the runtime-backed KTX command.\n');
return {
connectionId: target.connectionId,
driver: target.driver,
steps: [
{ operation: 'database-schema', status: 'done' },
{
operation: 'query-history',
status: 'failed',
detail:
'warehouse failed at query-history. Retry: ktx ingest warehouse --project-dir /tmp/project --deep --query-history',
},
{ operation: 'source-ingest', status: 'skipped' },
{ operation: 'memory-update', status: 'skipped' },
],
} satisfies KtxPublicIngestTargetResult;
});
const result = await runContextBuild(
project,
{ projectDir: '/tmp/project', inputMode: 'disabled', entrypoint: 'ingest' },
io.io,
{ executeTarget, now: () => 1000 },
);
expect(result).toEqual({ exitCode: 1 });
expect(io.stdout()).toContain('Missing bundled Python runtime manifest: /tmp/assets/python/manifest.json.');
expect(io.stdout()).toContain('Retry: ktx ingest warehouse --project-dir /tmp/project --deep --query-history');
expect(io.stdout()).not.toContain('Then retry the runtime-backed KTX command');
expect(io.stdout()).not.toContain('warehouse failed at query-history');
expect(io.stdout().match(/Retry: /g)).toHaveLength(1);
});
it('renders a friendly network failure when target execution throws', async () => {
const io = makeIo();
const project = projectWithConnections({

88
packages/cli/src/context-build-view.ts
View file

@ -1,6 +1,7 @@
import type { KtxProgressPort, KtxProgressUpdateOptions } from '@ktx/context/scan';
import type { KtxCliIo } from './index.js';
import type { KtxIngestProgressUpdate } from './ingest.js';
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
import { publicDatabaseIngestMessage, publicQueryHistoryMessage } from './public-ingest-copy.js';
import type {
KtxPublicIngestArgs,
@ -48,6 +49,8 @@ export interface ContextBuildArgs {
queryHistoryWindowDays?: number;
scanMode?: Extract<KtxPublicIngestArgs, { command: 'run' }>['scanMode'];
detectRelationships?: boolean;
cliVersion?: string;
runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
}
export interface ContextBuildResult {
@ -210,11 +213,18 @@ function retryCommand(input: {
entrypoint?: 'setup' | 'ingest';
connectionId?: string;
depth?: 'fast' | 'deep';
queryHistory?: boolean;
queryHistoryWindowDays?: number;
}): string {
const projectPart = input.projectDir ? ` --project-dir ${input.projectDir}` : '';
if (input.entrypoint === 'ingest' && input.connectionId) {
const depthPart = input.depth ? ` --${input.depth}` : '';
return `ktx ingest ${input.connectionId}${projectPart}${depthPart}`;
const queryHistoryPart = input.queryHistory ? ' --query-history' : '';
const windowPart =
input.queryHistory && input.queryHistoryWindowDays !== undefined
? ` --query-history-window-days ${input.queryHistoryWindowDays}`
: '';
return `ktx ingest ${input.connectionId}${projectPart}${depthPart}${queryHistoryPart}${windowPart}`;
}
return input.projectDir ? `ktx setup --project-dir ${input.projectDir}` : 'ktx setup';
}
@ -518,6 +528,11 @@ function networkErrorCode(error: unknown, capturedOutput = ''): string | null {
return networkErrorCodeFromText(`${unknownErrorMessage(error)}\n${capturedOutput}`);
}
function isLocalSqlAnalysisConnectionRefused(input: { capturedOutput?: string; fallback?: string | null }): boolean {
const text = `${input.capturedOutput ?? ''}\n${input.fallback ?? ''}`;
return /\bECONNREFUSED\b/.test(text) && /\b(?:127\.0\.0\.1|localhost):8765\b/.test(text);
}
function friendlyDriverName(driver: string): string {
const normalized = driver.toLowerCase();
if (normalized === 'postgres' || normalized === 'postgresql') return 'PostgreSQL';
@ -534,6 +549,45 @@ function failedStepDetail(result: KtxPublicIngestTargetResult): string | null {
return result.steps.find((step) => step.status === 'failed')?.detail ?? null;
}
const INTERNAL_FAILURE_LINE_RE =
/^(Report|Run|Job|Status|Adapter|Connection|Sync|Mode|Dry run|Diff|Work units|Saved memory|Provenance rows):\s*/;
const ACTIONABLE_FAILURE_LINE_RE =
/^(Missing bundled Python runtime manifest|KTX Python runtime is required|KTX managed daemon|Error:|Failed\b|Could not\b|Cannot\b)/;
function firstCapturedFailureLine(output: string | undefined): string | null {
const lines = (output ?? '')
.split(/\r?\n/)
.map((candidate) => candidate.trim())
.filter((candidate) => candidate.length > 0)
.filter((candidate) => !candidate.startsWith('KTX scan completed'))
.filter((candidate) => !INTERNAL_FAILURE_LINE_RE.test(candidate));
return lines.find((candidate) => ACTIONABLE_FAILURE_LINE_RE.test(candidate)) ?? lines.at(-1) ?? null;
}
function isGenericFailedAtDetail(target: KtxPublicIngestPlanTarget, detail: string | null | undefined): boolean {
return new RegExp(`^${target.connectionId} failed at [a-z-]+\\.?(?: Retry: .*)?$`).test(detail ?? '');
}
function appendRetryIfNeeded(input: {
message: string;
target: KtxPublicIngestPlanTarget;
projectDir: string;
entrypoint?: 'setup' | 'ingest';
}): string {
const base = input.message.trim().replace(/\.+$/, '');
if (/\bRetry:\s/.test(base)) {
return base;
}
return `${base}. Retry: ${retryCommand({
projectDir: input.projectDir,
entrypoint: input.entrypoint,
connectionId: input.target.connectionId,
depth: input.target.databaseDepth,
queryHistory: input.target.queryHistory?.enabled === true,
queryHistoryWindowDays: input.target.queryHistory?.windowDays,
})}`;
}
function failureTextForTarget(input: {
target: KtxPublicIngestPlanTarget;
projectDir: string;
@ -543,6 +597,20 @@ function failureTextForTarget(input: {
fallback?: string | null;
}): string {
const code = networkErrorCode(input.error, input.capturedOutput);
if (code && isLocalSqlAnalysisConnectionRefused({ capturedOutput: input.capturedOutput, fallback: input.fallback })) {
return [
`KTX could not reach the local SQL analysis runtime while processing query history for ${input.target.connectionId}.`,
`Reason: ${NETWORK_ERROR_REASONS[code]} (${code}).`,
`Retry: ${retryCommand({
projectDir: input.projectDir,
entrypoint: input.entrypoint,
connectionId: input.target.connectionId,
depth: input.target.databaseDepth,
queryHistory: input.target.queryHistory?.enabled === true,
queryHistoryWindowDays: input.target.queryHistory?.windowDays,
})}`,
].join(' ');
}
if (code) {
const operation = input.target.operation === 'database-ingest' ? 'reading schema for' : 'ingesting';
return [
@ -553,17 +621,23 @@ function failureTextForTarget(input: {
entrypoint: input.entrypoint,
connectionId: input.target.connectionId,
depth: input.target.databaseDepth,
queryHistory: input.target.queryHistory?.enabled === true,
queryHistoryWindowDays: input.target.queryHistory?.windowDays,
})}`,
].join(' ');
}
const fallback = input.fallback ?? `${input.target.connectionId} failed.`;
const capturedFailure = firstCapturedFailureLine(input.capturedOutput);
const fallback =
capturedFailure && isGenericFailedAtDetail(input.target, input.fallback)
? capturedFailure
: (input.fallback ?? capturedFailure ?? `${input.target.connectionId} failed.`);
if (input.entrypoint === 'ingest') {
return `${fallback} Retry: ${retryCommand({
return appendRetryIfNeeded({
message: fallback,
target: input.target,
projectDir: input.projectDir,
entrypoint: input.entrypoint,
connectionId: input.target.connectionId,
depth: input.target.databaseDepth,
})}`;
});
}
return fallback;
}
@ -697,6 +771,8 @@ export async function runContextBuild(
...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
...(args.scanMode ? { scanMode: args.scanMode } : {}),
...(args.detectRelationships !== undefined ? { detectRelationships: args.detectRelationships } : {}),
...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
};
let hasFailure = false;

8
packages/cli/src/dev.test.ts
View file

@ -146,7 +146,7 @@ describe('dev Commander tree', () => {
expect(doctor).not.toHaveBeenCalled();
});
it('rejects removed adapter-backed ingest run and keeps it out of ingest help', async () => {
it('rejects old adapter-backed ingest flags through public option parsing and keeps run out of ingest help', async () => {
const helpIo = makeIo();
const runIo = makeIo();
const publicIngest = vi.fn(async () => 0);
@ -161,7 +161,7 @@ describe('dev Commander tree', () => {
).resolves.toBe(1);
expect(helpIo.stdout()).not.toMatch(/^ run\s/m);
expect(runIo.stderr()).toMatch(/unknown command|error:/);
expect(runIo.stderr()).toMatch(/unknown option '--connection-id'|error:/);
expect(publicIngest).not.toHaveBeenCalled();
});
@ -181,7 +181,7 @@ describe('dev Commander tree', () => {
expect(io.stderr()).toMatch(/unknown command|error:/);
});
it('rejects top-level ingest run through the removed low-level ingest registration', async () => {
it('rejects old adapter-backed top-level ingest flags without low-level ingest registration', async () => {
const io = makeIo();
const publicIngest = vi.fn(async () => 0);
@ -204,6 +204,6 @@ describe('dev Commander tree', () => {
).resolves.toBe(1);
expect(publicIngest).not.toHaveBeenCalled();
expect(io.stderr()).toMatch(/unknown command|error:/);
expect(io.stderr()).toMatch(/unknown option '--connection-id'|error:/);
});
});

69
packages/cli/src/index.test.ts
View file

@ -628,6 +628,8 @@ describe('runKtxCli', () => {
inputMode: 'disabled',
depth: 'fast',
queryHistory: 'default',
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'never',
},
testIo.io,
);
@ -653,6 +655,8 @@ describe('runKtxCli', () => {
inputMode: 'auto',
depth: 'deep',
queryHistory: 'default',
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'prompt',
},
testIo.io,
);
@ -673,25 +677,34 @@ describe('runKtxCli', () => {
expect(testIo.stderr()).toMatch(/option '--(deep|fast)' cannot be used with option '--(fast|deep)'/);
});
it.each([
{ argv: ['ingest', 'run', '--connection-id', 'warehouse', '--adapter', 'metabase'] },
{ argv: ['ingest', 'run', '--help'] },
{ argv: ['ingest', 'status'] },
{ argv: ['ingest', 'status', 'run-1', '--json', '--no-input'] },
{ argv: ['ingest', 'watch'] },
{ argv: ['ingest', 'watch', '--help'] },
{ argv: ['ingest', 'replay', 'run-1'] },
{ argv: ['ingest', 'replay', '--help'] },
{ argv: ['--project-dir', '/tmp/project', 'ingest', 'status', 'run-1'] },
])('rejects removed ingest subcommand $argv', async ({ argv }) => {
const testIo = makeIo();
const publicIngest = vi.fn(async () => 0);
it.each(['run', 'status', 'watch', 'replay'])(
'routes former ingest subcommand name "%s" as a connection id',
async (connectionId) => {
const testIo = makeIo();
const publicIngest = vi.fn(async () => 0);
await expect(runKtxCli(argv, testIo.io, { publicIngest })).resolves.toBe(1);
await expect(
runKtxCli(['--project-dir', '/tmp/project', 'ingest', connectionId, '--no-input'], testIo.io, {
publicIngest,
}),
).resolves.toBe(0);
expect(testIo.stderr()).toMatch(/unknown command|error:/);
expect(publicIngest).not.toHaveBeenCalled();
});
expect(publicIngest).toHaveBeenCalledWith(
{
command: 'run',
projectDir: '/tmp/project',
targetConnectionId: connectionId,
all: false,
json: false,
inputMode: 'disabled',
queryHistory: 'default',
cliVersion: '0.0.0-private',
runtimeInstallPolicy: 'never',
},
testIo.io,
);
},
);
it('rejects standalone demo commands', async () => {
const testIo = makeIo();
@ -746,7 +759,7 @@ describe('runKtxCli', () => {
expect(publicIngest).not.toHaveBeenCalled();
});
it('rejects removed ingest run at the top level and under dev', async () => {
it('rejects old adapter-backed ingest flags at the top level and under dev', async () => {
const rootRunIo = makeIo();
const devRunIo = makeIo();
const publicIngest = vi.fn(async () => 0);
@ -762,7 +775,7 @@ describe('runKtxCli', () => {
}),
).resolves.toBe(1);
expect(publicIngest).not.toHaveBeenCalled();
expect(rootRunIo.stderr()).toMatch(/unknown command|error:/);
expect(rootRunIo.stderr()).toMatch(/unknown option '--connection-id'|error:/);
expect(devRunIo.stderr()).toMatch(/unknown command|error:/);
});
@ -770,7 +783,6 @@ describe('runKtxCli', () => {
const doctor = vi.fn(async () => 0);
const doctorIo = makeIo();
const ingestRunIo = makeIo();
const ingestReplayHelpIo = makeIo();
await expect(runKtxCli(['dev', 'doctor', 'setup', '--json', '--no-input'], doctorIo.io, { doctor })).resolves.toBe(1);
await expect(
@ -795,12 +807,10 @@ describe('runKtxCli', () => {
{},
),
).resolves.toBe(1);
await expect(runKtxCli(['ingest', 'replay', '--help'], ingestReplayHelpIo.io)).resolves.toBe(1);
expect(doctor).not.toHaveBeenCalled();
expect(doctorIo.stderr()).toMatch(/unknown command|error:/);
expect(ingestRunIo.stderr()).toMatch(/unknown command|error:/);
expect(ingestReplayHelpIo.stderr()).toMatch(/unknown command|error:/);
expect(ingestRunIo.stderr()).toMatch(/unknown option '--connection-id'|error:/);
});
it('dispatches public connection through the existing connection implementation', async () => {
@ -1055,17 +1065,20 @@ describe('runKtxCli', () => {
);
});
it('rejects reserved setup database connection ids before dispatch', async () => {
it('dispatches setup database connection ids that match former ingest subcommand names', async () => {
const testIo = makeIo();
const setup = vi.fn(async () => 0);
await expect(
runKtxCli(['setup', '--new-database-connection-id', 'status', '--no-input'], testIo.io, { setup }),
).resolves.toBe(1);
).resolves.toBe(0);
expect(setup).not.toHaveBeenCalled();
expect(testIo.stderr()).toContain(
'"status" is reserved for the KTX ingest command namespace; choose a different connection id.',
expect(setup).toHaveBeenCalledWith(
expect.objectContaining({
command: 'run',
databaseConnectionId: 'status',
}),
testIo.io,
);
});

6
packages/cli/src/public-ingest.test.ts
View file

@ -436,6 +436,8 @@ describe('runKtxPublicIngest', () => {
all: false,
json: false,
inputMode: 'disabled',
cliVersion: '0.0.0-test',
runtimeInstallPolicy: 'never',
queryHistory: 'enabled',
queryHistoryWindowDays: 30,
},
@ -454,6 +456,8 @@ describe('runKtxPublicIngest', () => {
connectionId: 'warehouse',
adapter: 'historic-sql',
allowImplicitAdapter: true,
cliVersion: '0.0.0-test',
runtimeInstallPolicy: 'never',
historicSqlPullConfigOverride: expect.objectContaining({ dialect: 'postgres', windowDays: 30 }),
}),
expect.anything(),
@ -465,6 +469,7 @@ describe('runKtxPublicIngest', () => {
const project = deepReadyProject({
warehouse: {
driver: 'postgres',
enabled_tables: ['orbit_analytics.int_active_contract_arr'],
context: {
queryHistory: {
enabled: true,
@ -524,6 +529,7 @@ describe('runKtxPublicIngest', () => {
dropFailedBelow: { errorRate: 0.5, executions: 3 },
},
redactionPatterns: ['(?i)secret'],
enabledTables: ['orbit_analytics.int_active_contract_arr'],
},
});
expect(ingestArgs?.historicSqlPullConfigOverride).not.toHaveProperty('enabled');

25
packages/cli/src/public-ingest.ts
View file

@ -9,6 +9,7 @@ import {
isDatabaseDriver,
normalizeConnectionDriver,
} from './ingest-depth.js';
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
import { publicIngestOutputLine } from './public-ingest-copy.js';
import type { KtxScanArgs, KtxScanDeps } from './scan.js';
import { profileMark } from './startup-profile.js';
@ -35,6 +36,8 @@ export type KtxPublicIngestArgs =
queryHistoryWindowDays?: number;
scanMode?: Extract<KtxScanArgs, { command: 'run' }>['mode'];
detectRelationships?: boolean;
cliVersion?: string;
runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
};
export interface KtxPublicIngestPlanTarget {
@ -101,6 +104,8 @@ interface KtxPublicContextBuildArgs {
queryHistoryWindowDays?: number;
scanMode?: Extract<KtxScanArgs, { command: 'run' }>['mode'];
detectRelationships?: boolean;
cliVersion?: string;
runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
}
const sourceAdapterByDriver = new Map<string, string>([
@ -258,15 +263,26 @@ function positiveInteger(value: unknown): number | undefined {
return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : undefined;
}
function enabledTablesForConnection(connection: KtxProjectConnectionConfig): string[] | undefined {
const raw = connection.enabled_tables;
if (!Array.isArray(raw)) {
return undefined;
}
const tables = raw.filter((value): value is string => typeof value === 'string' && value.trim().length > 0);
return tables.length > 0 ? tables : undefined;
}
function queryHistoryPullConfig(input: {
stored: Record<string, unknown>;
dialect: HistoricSqlDialect;
windowDays?: number;
enabledTables?: string[];
}): Record<string, unknown> {
const { enabled: _enabled, dialect: _dialect, ...storedConfig } = input.stored;
return {
...storedConfig,
dialect: input.dialect,
...(input.enabledTables ? { enabledTables: input.enabledTables } : {}),
...(input.windowDays !== undefined ? { windowDays: input.windowDays } : {}),
};
}
@ -342,6 +358,7 @@ function resolveDatabaseTargetOptions(input: {
stored: storedQh,
dialect,
windowDays: queryHistory.windowDays,
enabledTables: enabledTablesForConnection(input.connection),
}),
},
steps: ['database-schema', 'query-history'],
@ -684,6 +701,8 @@ export async function executePublicIngestTarget(
mode: target.databaseDepth === 'deep' ? 'enriched' : 'structural',
detectRelationships: target.detectRelationships === true,
dryRun: false,
...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
};
const runScan = deps.runScan ?? runKtxScan;
const capturedScanIo = deps.scanProgress ? null : createCapturedPublicIngestIo();
@ -711,6 +730,8 @@ export async function executePublicIngestTarget(
adapter: 'historic-sql',
outputMode: sourceIngestOutputMode(args, io),
inputMode: args.inputMode,
...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
allowImplicitAdapter: true,
historicSqlPullConfigOverride:
target.queryHistory.pullConfig ?? {
@ -746,6 +767,8 @@ export async function executePublicIngestTarget(
...(target.sourceDir ? { sourceDir: target.sourceDir } : {}),
outputMode: sourceIngestOutputMode(args, io),
inputMode: args.inputMode,
...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
allowImplicitAdapter: true,
};
const runIngest = deps.runIngest ?? runKtxIngest;
@ -786,6 +809,8 @@ export async function runKtxPublicIngest(
...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
...(args.scanMode ? { scanMode: args.scanMode } : {}),
...(args.detectRelationships !== undefined ? { detectRelationships: args.detectRelationships } : {}),
...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
},
io,
);

67
packages/cli/src/runtime.test.ts
View file

@ -368,4 +368,71 @@ describe('runKtxRuntime', () => {
expect(io.stdout()).toContain('PASS Managed Python runtime: Runtime ready at /runtime/0.2.0');
expect(io.stderr()).toBe('');
});
it('returns success when the installed runtime is ready but source assets are missing', async () => {
const io = makeIo();
const deps: KtxRuntimeDeps = {
readStatus: vi.fn(async (): Promise<ManagedPythonRuntimeStatus> => ({
kind: 'ready',
detail: 'Runtime ready at /runtime/0.2.0',
layout: {
cliVersion: '0.2.0',
runtimeRoot: '/runtime',
versionDir: '/runtime/0.2.0',
venvDir: '/runtime/0.2.0/.venv',
manifestPath: '/runtime/0.2.0/manifest.json',
installLogPath: '/runtime/0.2.0/install.log',
assetDir: '/assets/python',
assetManifestPath: '/assets/python/manifest.json',
pythonPath: '/runtime/0.2.0/.venv/bin/python',
daemonPath: '/runtime/0.2.0/.venv/bin/ktx-daemon',
daemonStatePath: '/runtime/0.2.0/daemon.json',
daemonStdoutPath: '/runtime/0.2.0/daemon.stdout.log',
daemonStderrPath: '/runtime/0.2.0/daemon.stderr.log',
},
manifest: {
schemaVersion: 1,
cliVersion: '0.2.0',
installedAt: '2026-05-11T00:00:00.000Z',
asset: {
schemaVersion: 1,
distributionName: 'kaelio-ktx',
normalizedName: 'kaelio_ktx',
version: '0.1.0',
wheel: {
file: 'kaelio_ktx-0.1.0-py3-none-any.whl',
sha256: 'a'.repeat(64),
bytes: 10,
},
},
features: ['core'],
python: {
executable: '/runtime/0.2.0/.venv/bin/python',
daemonExecutable: '/runtime/0.2.0/.venv/bin/ktx-daemon',
},
installLog: '/runtime/0.2.0/install.log',
},
})),
doctorRuntime: vi.fn(async (): Promise<ManagedPythonRuntimeDoctorCheck[]> => [
{ id: 'uv', label: 'uv', status: 'pass', detail: 'uv 0.9.5' },
{
id: 'asset',
label: 'Bundled Python wheel',
status: 'fail',
detail: 'Missing bundled Python runtime manifest: /assets/python/manifest.json',
fix: 'Run: pnpm run artifacts:check',
},
{ id: 'runtime', label: 'Managed Python runtime', status: 'pass', detail: 'Runtime ready at /runtime/0.2.0' },
]),
};
await expect(runKtxRuntime({ command: 'status', cliVersion: '0.2.0', json: false }, io.io, deps)).resolves.toBe(
0,
);
expect(io.stdout()).toContain('status: ready');
expect(io.stdout()).toContain('FAIL Bundled Python wheel: Missing bundled Python runtime manifest');
expect(io.stdout()).toContain('PASS Managed Python runtime: Runtime ready at /runtime/0.2.0');
expect(io.stderr()).toBe('');
});
});

6
packages/cli/src/runtime.ts
View file

@ -150,8 +150,8 @@ function writeRuntimeChecks(io: KtxCliIo, checks: ManagedPythonRuntimeDoctorChec
}
}
function hasRuntimeCheckFailures(checks: ManagedPythonRuntimeDoctorCheck[]): boolean {
return checks.some((check) => check.status === 'fail');
function hasRuntimeStatusFailure(status: ManagedPythonRuntimeStatus): boolean {
return status.kind !== 'ready';
}
export async function runKtxRuntime(
@ -203,7 +203,7 @@ export async function runKtxRuntime(
writeStatus(io, status);
writeRuntimeChecks(io, checks);
}
return hasRuntimeCheckFailures(checks) ? 1 : 0;
return hasRuntimeStatusFailure(status) ? 1 : 0;
}
const _exhaustive: never = args;
return _exhaustive;

82
packages/cli/src/setup-context.ts
View file

@ -13,12 +13,9 @@ import { buildPublicIngestPlan } from './public-ingest.js';
import {
type KtxDatabaseContextDepth,
databaseContextDepth,
deepReadinessGaps,
isDatabaseDriver,
normalizeConnectionDriver,
recommendedDatabaseContextDepth,
withDatabaseContextDepth,
} from './ingest-depth.js';
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
import { ensureSetupDatabaseContextDepths } from './setup-database-context-depth.js';
import {
type ContextBuildSourceProgressUpdate,
runContextBuild,
@ -88,6 +85,8 @@ export interface KtxSetupContextStepArgs {
allowEmpty?: boolean;
prompt?: boolean;
autoWatch?: boolean;
cliVersion?: string;
runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
}
export interface KtxSetupContextPromptAdapter {
@ -295,77 +294,6 @@ function listContextTargets(project: KtxLocalProject): KtxSetupContextTargets {
};
}
function databaseConnectionsNeedingDepth(project: KtxLocalProject): string[] {
return Object.entries(project.config.connections)
.filter(([, connection]) => isDatabaseDriver(normalizeConnectionDriver(connection)))
.filter(([, connection]) => databaseContextDepth(connection) === undefined)
.map(([connectionId]) => connectionId)
.sort((left, right) => left.localeCompare(right));
}
async function writeDatabaseContextDepths(
project: KtxLocalProject,
connectionIds: string[],
depth: KtxDatabaseContextDepth,
): Promise<KtxLocalProject> {
if (connectionIds.length === 0) {
return project;
}
const nextConnections = { ...project.config.connections };
for (const connectionId of connectionIds) {
const connection = nextConnections[connectionId];
if (connection) {
nextConnections[connectionId] = withDatabaseContextDepth(connection, depth);
}
}
const nextConfig = { ...project.config, connections: nextConnections };
await writeFile(project.configPath, serializeKtxProjectConfig(nextConfig), 'utf-8');
return await loadKtxProject({ projectDir: project.projectDir });
}
async function ensureSetupDatabaseContextDepths(input: {
project: KtxLocalProject;
args: KtxSetupContextStepArgs;
prompts: KtxSetupContextPromptAdapter;
}): Promise<KtxLocalProject | 'back'> {
const missingDepthConnectionIds = databaseConnectionsNeedingDepth(input.project);
if (missingDepthConnectionIds.length === 0) {
return input.project;
}
const recommended = recommendedDatabaseContextDepth(input.project.config);
if (input.args.inputMode === 'disabled') {
return await writeDatabaseContextDepths(input.project, missingDepthConnectionIds, recommended);
}
const deepReady = deepReadinessGaps(input.project.config).length === 0;
const options =
recommended === 'deep'
? [
{ value: 'deep', label: 'Deep: AI descriptions, embeddings, relationships, slower' },
{ value: 'fast', label: 'Fast: schema only, no AI, quickest' },
{ value: 'back', label: 'Back' },
]
: [
{ value: 'fast', label: 'Fast: schema only, no AI, quickest' },
{ value: 'deep', label: 'Deep: AI descriptions, embeddings, relationships, slower' },
{ value: 'back', label: 'Back' },
];
const choice = await input.prompts.select({
message:
'How much database context should KTX build?\n\n' +
(deepReady
? 'Deep is available because model, embedding, and scan enrichment are configured.'
: 'Fast is recommended because model, embedding, or scan enrichment is not configured.'),
options,
});
if (choice === 'back') {
return 'back';
}
return await writeDatabaseContextDepths(input.project, missingDepthConnectionIds, choice as KtxDatabaseContextDepth);
}
async function hasFileWithExtension(
root: string,
extensions: Set<string>,
@ -645,6 +573,8 @@ async function runBuild(
{
projectDir: args.projectDir,
inputMode: args.inputMode,
...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
},
io,
{

131
packages/cli/src/setup-database-context-depth.ts Normal file
View file

@ -0,0 +1,131 @@
import { writeFile } from 'node:fs/promises';
import {
type KtxLocalProject,
type KtxProjectConnectionConfig,
loadKtxProject,
serializeKtxProjectConfig,
} from '@ktx/context/project';
import {
type KtxDatabaseContextDepth,
databaseContextDepth,
deepReadinessGaps,
isDatabaseDriver,
normalizeConnectionDriver,
recommendedDatabaseContextDepth,
withDatabaseContextDepth,
} from './ingest-depth.js';
import type { KtxSetupPromptOption } from './setup-prompts.js';
export interface KtxSetupDatabaseContextDepthArgs {
inputMode: 'auto' | 'disabled';
}
export interface KtxSetupDatabaseContextDepthPromptAdapter {
select(options: { message: string; options: KtxSetupPromptOption[] }): Promise<string>;
}
function databaseConnectionsNeedingDepth(project: KtxLocalProject): string[] {
return Object.entries(project.config.connections)
.filter(([, connection]) => isDatabaseDriver(normalizeConnectionDriver(connection)))
.filter(([, connection]) => databaseContextDepth(connection) === undefined)
.map(([connectionId]) => connectionId)
.sort((left, right) => left.localeCompare(right));
}
async function chooseSetupDatabaseContextDepth(input: {
project: KtxLocalProject;
args: KtxSetupDatabaseContextDepthArgs;
prompts: KtxSetupDatabaseContextDepthPromptAdapter;
}): Promise<KtxDatabaseContextDepth | 'back'> {
const recommended = recommendedDatabaseContextDepth(input.project.config);
if (input.args.inputMode === 'disabled') {
return recommended;
}
const deepReady = deepReadinessGaps(input.project.config).length === 0;
const options =
recommended === 'deep'
? [
{ value: 'deep', label: 'Deep: AI descriptions, embeddings, relationships, slower' },
{ value: 'fast', label: 'Fast: schema only, no AI, quickest' },
{ value: 'back', label: 'Back' },
]
: [
{ value: 'fast', label: 'Fast: schema only, no AI, quickest' },
{ value: 'deep', label: 'Deep: AI descriptions, embeddings, relationships, slower' },
{ value: 'back', label: 'Back' },
];
const choice = await input.prompts.select({
message:
'How much database context should KTX build?\n\n' +
(deepReady
? 'Deep is available because model, embedding, and scan enrichment are configured.'
: 'Fast is recommended because model, embedding, or scan enrichment is not configured.'),
options,
});
if (choice === 'back') {
return 'back';
}
if (choice === 'fast' || choice === 'deep') {
return choice;
}
return recommended;
}
async function writeDatabaseContextDepths(
project: KtxLocalProject,
connectionIds: string[],
depth: KtxDatabaseContextDepth,
): Promise<KtxLocalProject> {
if (connectionIds.length === 0) {
return project;
}
const nextConnections = { ...project.config.connections };
for (const connectionId of connectionIds) {
const connection = nextConnections[connectionId];
if (connection) {
nextConnections[connectionId] = withDatabaseContextDepth(connection, depth);
}
}
const nextConfig = { ...project.config, connections: nextConnections };
await writeFile(project.configPath, serializeKtxProjectConfig(nextConfig), 'utf-8');
return await loadKtxProject({ projectDir: project.projectDir });
}
export async function ensureSetupDatabaseContextDepths(input: {
project: KtxLocalProject;
args: KtxSetupDatabaseContextDepthArgs;
prompts: KtxSetupDatabaseContextDepthPromptAdapter;
}): Promise<KtxLocalProject | 'back'> {
const missingDepthConnectionIds = databaseConnectionsNeedingDepth(input.project);
if (missingDepthConnectionIds.length === 0) {
return input.project;
}
const depth = await chooseSetupDatabaseContextDepth(input);
if (depth === 'back') {
return 'back';
}
return await writeDatabaseContextDepths(input.project, missingDepthConnectionIds, depth);
}
export async function applySetupDatabaseContextDepth(input: {
project: KtxLocalProject;
connection: KtxProjectConnectionConfig;
args: KtxSetupDatabaseContextDepthArgs;
prompts: KtxSetupDatabaseContextDepthPromptAdapter;
}): Promise<KtxProjectConnectionConfig | 'back'> {
if (
!isDatabaseDriver(normalizeConnectionDriver(input.connection)) ||
databaseContextDepth(input.connection) !== undefined
) {
return input.connection;
}
const depth = await chooseSetupDatabaseContextDepth(input);
if (depth === 'back') {
return 'back';
}
return withDatabaseContextDepth(input.connection, depth);
}

58
packages/cli/src/setup-databases.test.ts
View file

@ -44,7 +44,15 @@ function makePromptAdapter(options: {
const passwordValues = [...(options.passwordValues ?? [])];
return {
multiselect: vi.fn(async () => multiselectValues.shift() ?? ['postgres']),
select: vi.fn(async () => selectValues.shift() ?? 'finish'),
select: vi.fn(async ({ message }) => {
if (message.includes('How much database context should KTX build?')) {
const nextValue = selectValues[0];
return nextValue === 'fast' || nextValue === 'deep' || nextValue === 'back'
? (selectValues.shift() ?? 'fast')
: 'fast';
}
return selectValues.shift() ?? 'finish';
}),
text: vi.fn(async () => (textValues.length > 0 ? textValues.shift() : '')),
password: vi.fn(async () => (passwordValues.length > 0 ? passwordValues.shift() : '')),
cancel: vi.fn(),
@ -283,6 +291,7 @@ describe('setup databases step', () => {
driver: 'postgres',
url: 'env:DATABASE_URL',
readonly: true,
context: { depth: 'fast' },
});
});
@ -714,7 +723,7 @@ describe('setup databases step', () => {
});
expect(prompts.multiselect).toHaveBeenCalledTimes(2);
expect(io.stdout()).not.toContain('KTX cannot work without at least one database');
expect(prompts.select).toHaveBeenNthCalledWith(2, {
expect(prompts.select).toHaveBeenNthCalledWith(3, {
message: 'Databases already configured: postgres-warehouse\nWhat would you like to do?',
options: [
{ value: 'continue', label: 'Continue to context sources' },
@ -786,9 +795,8 @@ describe('setup databases step', () => {
);
expect(result.status).toBe('ready');
expect(prompts.select).toHaveBeenCalledTimes(2);
expect(vi.mocked(prompts.select).mock.calls[0]?.[0].message).toBe('How do you want to connect to PostgreSQL?');
expect(vi.mocked(prompts.select).mock.calls[1]?.[0].message).toBe('How do you want to connect to PostgreSQL?');
const selectMessages = vi.mocked(prompts.select).mock.calls.map(([options]) => options.message);
expect(selectMessages.filter((message) => message === 'How do you want to connect to PostgreSQL?')).toHaveLength(2);
expect(testConnection).toHaveBeenCalledWith(tempDir, 'postgres-warehouse', expect.anything());
});
@ -1149,7 +1157,7 @@ describe('setup databases step', () => {
driver: 'postgres',
url: 'env:DATABASE_URL',
schemas: ['public'],
context: { queryHistory: { enabled: false } },
context: { queryHistory: { enabled: false }, depth: 'fast' },
readonly: true,
});
expect(config.setup).toEqual({
@ -1190,6 +1198,7 @@ describe('setup databases step', () => {
driver: 'sqlite',
path: './warehouse.sqlite',
readonly: true,
context: { depth: 'fast' },
});
expect(config.setup).toEqual({
database_connection_ids: ['warehouse'],
@ -1502,12 +1511,24 @@ describe('setup databases step', () => {
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
'llm:',
' provider:',
' backend: anthropic',
' models:',
' default: claude-sonnet-4-6',
'scan:',
' enrichment:',
' mode: llm',
' embeddings:',
' backend: openai',
' model: text-embedding-3-small',
' dimensions: 1536',
'',
].join('\n'),
'utf-8',
);
const io = makeIo();
const prompts = makePromptAdapter({ selectValues: ['yes'] });
const prompts = makePromptAdapter({ selectValues: ['yes', 'deep'] });
const historicSqlProbe = vi.fn(async () => ({ ok: true, lines: [] }));
const result = await runKtxSetupDatabasesStep(
@ -1536,6 +1557,12 @@ describe('setup databases step', () => {
{ value: 'back', label: 'Back' },
],
});
expect(prompts.select).toHaveBeenNthCalledWith(
2,
expect.objectContaining({
message: expect.stringContaining('How much database context should KTX build?'),
}),
);
expect(historicSqlProbe).toHaveBeenCalledWith({
projectDir: tempDir,
connectionId: 'warehouse',
@ -1549,6 +1576,7 @@ describe('setup databases step', () => {
minExecutions: 5,
filters: { dropTrivialProbes: true },
},
depth: 'deep',
},
});
});
@ -1822,7 +1850,7 @@ describe('setup databases step', () => {
expect(io.stderr()).toContain('Missing database connection id');
});
it('rejects reserved non-interactive database connection ids', async () => {
it('accepts former ingest subcommand names as non-interactive database connection ids', async () => {
const io = makeIo();
const result = await runKtxSetupDatabasesStep(
@ -1836,12 +1864,18 @@ describe('setup databases step', () => {
skipDatabases: false,
},
io.io,
{
testConnection: vi.fn(async () => 0),
scanConnection: vi.fn(async () => 0),
},
);
expect(result.status).toBe('failed');
expect(io.stderr()).toContain(
'"replay" is reserved for the KTX ingest command namespace; choose a different connection id.',
);
expect(result.status).toBe('ready');
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections.replay).toMatchObject({
driver: 'postgres',
url: 'env:DATABASE_URL',
});
});
it('leaves setup incomplete when databases are skipped', async () => {

82
packages/cli/src/setup-databases.ts
View file

@ -5,7 +5,6 @@ import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';
import type { HistoricSqlDialect } from '@ktx/context/ingest';
import {
assertKtxConnectionIdIsNotReserved,
type KtxProjectConnectionConfig,
loadKtxProject,
markKtxSetupStateStepComplete,
@ -17,6 +16,7 @@ import type { KtxCliIo } from './cli-runtime.js';
import { runKtxConnection } from './connection.js';
import { withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js';
import { runKtxScan } from './scan.js';
import { applySetupDatabaseContextDepth } from './setup-database-context-depth.js';
import { writeProjectLocalSecretReference } from './setup-secrets.js';
import {
createKtxSetupPromptAdapter,
@ -232,7 +232,6 @@ function assertSafeDatabaseConnectionId(connectionId: string): void {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) {
throw new Error(`Unsafe connection id: ${connectionId}`);
}
assertKtxConnectionIdIsNotReserved(connectionId);
}
function historicSqlConfigRecord(connection: KtxProjectConnectionConfig | undefined): Record<string, unknown> | null {
@ -1498,10 +1497,45 @@ async function applyHistoricSqlConfigToExistingConnection(input: {
prompts: input.prompts,
});
if (withHistoricSql === 'back') return 'back';
await writeConnectionConfig({
const withContextDepth = await maybeApplyContextDepthConfig({
projectDir: input.projectDir,
connectionId: input.connectionId,
connection: withHistoricSql,
args: input.args,
prompts: input.prompts,
});
if (withContextDepth === 'back') return 'back';
await writeConnectionConfig({
projectDir: input.projectDir,
connectionId: input.connectionId,
connection: withContextDepth,
});
}
async function maybeApplyContextDepthConfig(input: {
projectDir: string;
connectionId: string;
connection: KtxProjectConnectionConfig;
args: KtxSetupDatabasesArgs;
prompts: KtxSetupDatabasesPromptAdapter;
}): Promise<KtxProjectConnectionConfig | 'back'> {
const project = await loadKtxProject({ projectDir: input.projectDir });
return await applySetupDatabaseContextDepth({
project: {
...project,
config: {
...project.config,
connections: {
...project.config.connections,
[input.connectionId]: input.connection,
},
},
},
connection: input.connection,
args: {
inputMode: input.args.inputMode === 'disabled' || input.args.databaseUrl ? 'disabled' : input.args.inputMode,
},
prompts: input.prompts,
});
}
@ -1850,10 +1884,22 @@ export async function runKtxSetupDatabasesStep(
returnToDriverSelection = true;
break;
}
await writeConnectionConfig({
const withContextDepth = await maybeApplyContextDepthConfig({
projectDir: args.projectDir,
connectionId: connectionChoice.connectionId,
connection: withHistoricSql,
args,
prompts,
});
if (withContextDepth === 'back') {
if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir };
returnToDriverSelection = true;
break;
}
await writeConnectionConfig({
projectDir: args.projectDir,
connectionId: connectionChoice.connectionId,
connection: withContextDepth,
});
} else {
const existing = project.config.connections[connectionChoice.connectionId];
@ -1863,10 +1909,22 @@ export async function runKtxSetupDatabasesStep(
returnToDriverSelection = true;
break;
}
await writeConnectionConfig({
const withContextDepth = await maybeApplyContextDepthConfig({
projectDir: args.projectDir,
connectionId: connectionChoice.connectionId,
connection: withHistoricSql,
args,
prompts,
});
if (withContextDepth === 'back') {
if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir };
returnToDriverSelection = true;
break;
}
await writeConnectionConfig({
projectDir: args.projectDir,
connectionId: connectionChoice.connectionId,
connection: withContextDepth,
});
}
@ -1919,10 +1977,22 @@ export async function runKtxSetupDatabasesStep(
returnToDriverSelection = true;
break;
}
await writeConnectionConfig({
const withContextDepth = await maybeApplyContextDepthConfig({
projectDir: args.projectDir,
connectionId: connectionChoice.connectionId,
connection: withHistoricSql,
args,
prompts,
});
if (withContextDepth === 'back') {
if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir };
returnToDriverSelection = true;
break;
}
await writeConnectionConfig({
projectDir: args.projectDir,
connectionId: connectionChoice.connectionId,
connection: withContextDepth,
});
}
}

14
packages/cli/src/setup-sources.test.ts
View file

@ -254,9 +254,10 @@ describe('setup sources step', () => {
});
});
it('rejects reserved interactive source connection ids', async () => {
it('accepts former ingest subcommand names as interactive source connection ids', async () => {
await addPrimarySource();
const io = makeIo();
const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'workspace=ok' }));
const result = await runKtxSetupSourcesStep(
{
@ -272,13 +273,16 @@ describe('setup sources step', () => {
text: ['status', 'env:NOTION_TOKEN'],
select: ['env', 'all_accessible'],
}),
validateNotion,
},
);
expect(result.status).toBe('failed');
expect(io.stderr()).toContain(
'"status" is reserved for the KTX ingest command namespace; choose a different connection id.',
);
expect(result.status).toBe('ready');
const config = await readConfig();
expect(config.connections.status).toMatchObject({
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
});
});
it('uses selected Notion roots when root page ids are provided even if crawl mode says all accessible', async () => {

2
packages/cli/src/setup-sources.ts
View file

@ -19,7 +19,6 @@ import {
testRepoConnection,
} from '@ktx/context/ingest';
import {
assertKtxConnectionIdIsNotReserved,
type KtxProjectConfig,
type KtxProjectConnectionConfig,
loadKtxProject,
@ -202,7 +201,6 @@ function assertSafeConnectionId(connectionId: string): void {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) {
throw new Error(`Unsafe connection id: ${connectionId}`);
}
assertKtxConnectionIdIsNotReserved(connectionId);
}
function credentialRef(value: string | undefined, label: string): string {

2
packages/cli/src/setup.ts
View file

@ -680,6 +680,8 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup
inputMode: args.inputMode,
forcePrompt: forcePromptSteps.has('context') || runOnly === 'context',
allowEmpty: true,
cliVersion: args.cliVersion,
runtimeInstallPolicy: setupRuntimeInstallPolicy(args),
},
io,
);

4
packages/cli/src/standalone-smoke.test.ts
View file

@ -135,7 +135,7 @@ describe('standalone built ktx CLI smoke', () => {
await rm(tempDir, { recursive: true, force: true });
});
it('rejects removed low-level ingest run through the built binary', async () => {
it('rejects old low-level ingest flags through the built binary', async () => {
const projectDir = join(tempDir, 'project');
const init = await runSetupNewProject(projectDir);
@ -151,7 +151,7 @@ describe('standalone built ktx CLI smoke', () => {
'fake',
]);
expect(run).toMatchObject({ code: 1, stdout: '' });
expect(run.stderr).toContain("unknown command 'run'");
expect(run.stderr).toContain("unknown option '--connection-id'");
});
it('rejects the removed agent command through the built binary', async () => {

13
packages/context/src/agent/agent-runner.service.test.ts
View file

@ -50,11 +50,8 @@ describe('AgentRunnerService.runLoop', () => {
telemetryTags: { source: 'test' },
});
const call = (generateText as any).mock.calls[0][0];
expect(call.messages).toEqual([
{ role: 'system', content: 'SYS' },
{ role: 'user', content: 'USR' },
]);
expect(call.system).toBeUndefined();
expect(call.system).toEqual({ role: 'system', content: 'SYS' });
expect(call.messages).toEqual([{ role: 'user', content: 'USR' }]);
expect(call.prompt).toBeUndefined();
expect(call.tools).toEqual(tools);
expect(call.stopWhen).toBe(17);
@ -77,10 +74,8 @@ describe('AgentRunnerService.runLoop', () => {
expect(llmProvider.getModel).toHaveBeenCalledWith('candidateExtraction');
expect(generateText).toHaveBeenCalledWith(
expect.objectContaining({
messages: [
{ role: 'system', content: 'system' },
{ role: 'user', content: 'user' },
],
system: { role: 'system', content: 'system' },
messages: [{ role: 'user', content: 'user' }],
}),
);
});

12
packages/context/src/agent/agent-runner.service.ts
View file

@ -36,6 +36,14 @@ export interface AgentRunnerServiceDeps {
logger?: KtxLogger;
}
function splitSystemPromptMessages(messages: ReturnType<KtxMessageBuilder['wrapSimple']>['messages']) {
const systemMessages = messages.filter((message) => message.role === 'system');
return {
system: systemMessages.length === 0 ? undefined : systemMessages.length === 1 ? systemMessages[0] : systemMessages,
messages: messages.filter((message) => message.role !== 'system'),
};
}
export class AgentRunnerService {
private readonly logger: KtxLogger;
@ -54,6 +62,7 @@ export class AgentRunnerService {
tools: params.toolSet,
model,
});
const promptMessages = splitSystemPromptMessages(built.messages);
await this.deps.debugRequestRecorder?.record(
summarizeKtxLlmDebugRequest({
@ -73,7 +82,8 @@ export class AgentRunnerService {
temperature: 0,
stopWhen: stepCountIs(params.stepBudget),
experimental_telemetry: this.deps.telemetry?.createTelemetry(params.telemetryTags),
messages: built.messages,
...(promptMessages.system ? { system: promptMessages.system } : {}),
messages: promptMessages.messages,
tools: built.tools as Record<string, Tool>,
onStepFinish: async () => {
stepIndex += 1;

3
packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts
View file

@ -103,7 +103,7 @@ describe('BigQueryHistoricSqlQueryHistoryReader', () => {
for await (const row of reader.fetchAggregated(
client,
{ start: new Date('2026-02-10T00:00:00.000Z'), end: new Date('2026-05-11T00:00:00.000Z') },
{ dialect: 'bigquery', minExecutions: 5, windowDays: 90, concurrency: 12, filters: { dropTrivialProbes: true }, redactionPatterns: [], staleArchiveAfterDays: 90 },
{ dialect: 'bigquery', minExecutions: 5, windowDays: 90, concurrency: 12, enabledTables: [], filters: { dropTrivialProbes: true }, redactionPatterns: [], staleArchiveAfterDays: 90 },
)) {
rows.push(row);
}
@ -137,6 +137,7 @@ describe('BigQueryHistoricSqlQueryHistoryReader', () => {
minExecutions: 5,
windowDays: 90,
concurrency: 12,
enabledTables: [],
filters: { dropTrivialProbes: true },
redactionPatterns: [],
staleArchiveAfterDays: 90,

2
packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts
View file

@ -215,7 +215,7 @@ describe('PostgresPgssReader aggregate path', () => {
for await (const row of reader.fetchAggregated(
{ executeQuery },
{ start: new Date('2026-02-10T00:00:00.000Z'), end: new Date('2026-05-11T00:00:00.000Z') },
{ dialect: 'postgres', minExecutions: 5, windowDays: 90, concurrency: 12, filters: { dropTrivialProbes: true }, redactionPatterns: [], staleArchiveAfterDays: 90 },
{ dialect: 'postgres', minExecutions: 5, windowDays: 90, concurrency: 12, enabledTables: [], filters: { dropTrivialProbes: true }, redactionPatterns: [], staleArchiveAfterDays: 90 },
)) {
rows.push(row);
}

3
packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts
View file

@ -102,7 +102,7 @@ describe('SnowflakeHistoricSqlQueryHistoryReader', () => {
for await (const row of reader.fetchAggregated(
client,
{ start: new Date('2026-02-10T00:00:00.000Z'), end: new Date('2026-05-11T00:00:00.000Z') },
{ dialect: 'snowflake', minExecutions: 5, windowDays: 90, concurrency: 12, filters: { dropTrivialProbes: true }, redactionPatterns: [], staleArchiveAfterDays: 90 },
{ dialect: 'snowflake', minExecutions: 5, windowDays: 90, concurrency: 12, enabledTables: [], filters: { dropTrivialProbes: true }, redactionPatterns: [], staleArchiveAfterDays: 90 },
)) {
rows.push(row);
}
@ -136,6 +136,7 @@ describe('SnowflakeHistoricSqlQueryHistoryReader', () => {
minExecutions: 5,
windowDays: 90,
concurrency: 12,
enabledTables: [],
filters: { dropTrivialProbes: true },
redactionPatterns: [],
staleArchiveAfterDays: 90,

74
packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts
View file

@ -237,6 +237,80 @@ describe('stageHistoricSqlAggregatedSnapshot', () => {
expect(patternsJson).toContain('[REDACTED]');
});
it('limits staged table artifacts to configured enabled tables', async () => {
const stagedDir = await tempDir();
const reader: HistoricSqlReader = {
async probe() {
return { warnings: [], info: [] };
},
async *fetchAggregated() {
yield aggregate({
templateId: 'selected-qualified',
canonicalSql: 'select count(*) from orbit_analytics.int_active_contract_arr',
});
yield aggregate({
templateId: 'selected-unqualified',
canonicalSql: 'select count(*) from int_customer_health_signals',
});
yield aggregate({
templateId: 'unselected',
canonicalSql: 'select count(*) from orbit_raw.accounts',
});
},
};
const sqlAnalysis: SqlAnalysisPort = {
analyzeForFingerprint: vi.fn(),
analyzeBatch: vi.fn(async () => new Map([
[
'selected-qualified',
{
tablesTouched: ['orbit_analytics.int_active_contract_arr'],
columnsByClause: { select: [], where: [], join: [], groupBy: [] },
},
],
[
'selected-unqualified',
{
tablesTouched: ['int_customer_health_signals'],
columnsByClause: { select: [], where: [], join: [], groupBy: [] },
},
],
[
'unselected',
{
tablesTouched: ['orbit_raw.accounts'],
columnsByClause: { select: [], where: [], join: [], groupBy: [] },
},
],
])),
};
await stageHistoricSqlAggregatedSnapshot({
stagedDir,
connectionId: 'warehouse',
queryClient: {},
reader,
sqlAnalysis,
pullConfig: {
dialect: 'postgres',
enabledTables: [
'orbit_analytics.int_active_contract_arr',
'orbit_analytics.int_customer_health_signals',
],
},
now: new Date('2026-05-11T12:00:00.000Z'),
});
expect(await readdir(join(stagedDir, 'tables'))).toEqual([
'int_customer_health_signals.json',
'orbit_analytics.int_active_contract_arr.json',
]);
const manifest = await readJson<Record<string, any>>(stagedDir, 'manifest.json');
expect(manifest.touchedTableCount).toBe(2);
const patterns = await readJson<Record<string, any>>(stagedDir, 'patterns-input.json');
expect(patterns.templates.map((entry: any) => entry.id)).toEqual(['selected-qualified', 'selected-unqualified']);
});
it('preserves full patterns audit input and writes bounded cross-table pattern shards', async () => {
const stagedDir = await tempDir();
const largeSql = `select * from public.orders o join public.customers c on c.id = o.customer_id where payload = '${'x'.repeat(8000)}'`;

52
packages/context/src/ingest/adapters/historic-sql/stage-unified.ts
View file

@ -39,9 +39,15 @@ interface StageHistoricSqlAggregatedSnapshotInput {
interface ParsedTemplate {
template: AggregatedTemplate;
tablesTouched: string[];
includedTables: string[];
columnsByClause: Record<string, string[]>;
}
interface EnabledTableFilter {
exact: Set<string>;
uniqueUnqualified: Set<string>;
}
interface TableAccumulator {
table: string;
executions: number;
@ -103,6 +109,45 @@ function shouldDropTemplate(template: AggregatedTemplate, config: HistoricSqlUni
return false;
}
function normalizeTableIdentifier(value: string): string {
return value.trim().toLowerCase();
}
function unqualifiedTableIdentifier(value: string): string {
const parts = normalizeTableIdentifier(value).split('.').filter(Boolean);
return parts.at(-1) ?? '';
}
function buildEnabledTableFilter(enabledTables: string[]): EnabledTableFilter | null {
if (enabledTables.length === 0) {
return null;
}
const exact = new Set(enabledTables.map(normalizeTableIdentifier).filter((value) => value.length > 0));
const unqualifiedCounts = new Map<string, number>();
for (const table of exact) {
const unqualified = unqualifiedTableIdentifier(table);
if (unqualified.length > 0) {
unqualifiedCounts.set(unqualified, (unqualifiedCounts.get(unqualified) ?? 0) + 1);
}
}
return {
exact,
uniqueUnqualified: new Set(
[...unqualifiedCounts.entries()]
.filter(([, count]) => count === 1)
.map(([table]) => table),
),
};
}
function isEnabledTable(table: string, filter: EnabledTableFilter | null): boolean {
if (!filter) {
return true;
}
const normalized = normalizeTableIdentifier(table);
return filter.exact.has(normalized) || filter.uniqueUnqualified.has(unqualifiedTableIdentifier(normalized));
}
function redactTemplateSql(
template: AggregatedTemplate,
redactors: readonly HistoricSqlRedactionPattern[],
@ -231,6 +276,7 @@ function toPatternsInput(parsedTemplates: ParsedTemplate[]): StagedPatternsInput
export async function stageHistoricSqlAggregatedSnapshot(input: StageHistoricSqlAggregatedSnapshotInput): Promise<void> {
const config = historicSqlUnifiedPullConfigSchema.parse(input.pullConfig);
const enabledTableFilter = buildEnabledTableFilter(config.enabledTables);
const redactors = compileHistoricSqlRedactionPatterns(config.redactionPatterns);
const now = input.now ?? new Date();
const windowStart = new Date(now.getTime() - config.windowDays * 24 * 60 * 60 * 1000);
@ -259,12 +305,14 @@ export async function stageHistoricSqlAggregatedSnapshot(input: StageHistoricSql
continue;
}
const tablesTouched = [...new Set(parsed.tablesTouched)].filter((table) => table.length > 0).sort();
if (tablesTouched.length === 0) {
const includedTables = tablesTouched.filter((table) => isEnabledTable(table, enabledTableFilter));
if (includedTables.length === 0) {
continue;
}
parsedTemplates.push({
template: redactTemplateSql(template, redactors),
tablesTouched,
includedTables,
columnsByClause: Object.fromEntries(
Object.entries(parsed.columnsByClause).map(([clause, columns]) => [clause, [...new Set(columns)].sort()]),
),
@ -273,7 +321,7 @@ export async function stageHistoricSqlAggregatedSnapshot(input: StageHistoricSql
const byTable = new Map<string, TableAccumulator>();
for (const parsed of parsedTemplates) {
for (const table of parsed.tablesTouched) {
for (const table of parsed.includedTables) {
const acc = byTable.get(table) ?? accumulatorFor(table);
addTemplate(acc, parsed);
byTable.set(table, acc);

1
packages/context/src/ingest/adapters/historic-sql/types.ts
View file

@ -13,6 +13,7 @@ export const historicSqlUnifiedPullConfigSchema = z.object({
windowDays: z.number().int().positive().default(90),
minExecutions: z.number().int().nonnegative().default(5),
concurrency: z.number().int().positive().default(12),
enabledTables: z.array(z.string().min(1)).default([]),
filters: z.object({
serviceAccounts: z.object({
patterns: z.array(z.string()).default([]),

10
packages/context/src/project/config.test.ts
View file

@ -2,15 +2,19 @@ import { describe, expect, it } from 'vitest';
import { buildDefaultKtxProjectConfig, parseKtxProjectConfig, serializeKtxProjectConfig } from './config.js';
describe('KTX project config', () => {
it.each(['status', 'replay', 'run', 'watch'])('rejects reserved ingest connection id "%s"', (connectionId) => {
expect(() =>
it.each(['status', 'replay', 'run', 'watch'])('accepts former ingest subcommand name "%s" as a connection id', (connectionId) => {
expect(
parseKtxProjectConfig(`
project: reserved-test
connections:
${connectionId}:
driver: postgres
`),
).toThrow(`"${connectionId}" is reserved for the KTX ingest command namespace`);
).toMatchObject({
connections: {
[connectionId]: { driver: 'postgres' },
},
});
});
it('builds the default standalone project config', () => {

22
packages/context/src/project/config.ts
View file

@ -112,25 +112,6 @@ function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
const RESERVED_INGEST_CONNECTION_IDS = new Map([
['status', 'the KTX ingest command namespace'],
['replay', 'the KTX ingest command namespace'],
['run', 'the KTX ingest command namespace'],
['watch', 'the KTX ingest command namespace'],
]);
export function reservedKtxIngestConnectionIdMessage(connectionId: string): string | null {
const command = RESERVED_INGEST_CONNECTION_IDS.get(connectionId);
return command ? `"${connectionId}" is reserved for ${command}; choose a different connection id.` : null;
}
export function assertKtxConnectionIdIsNotReserved(connectionId: string): void {
const message = reservedKtxIngestConnectionIdMessage(connectionId);
if (message) {
throw new Error(message);
}
}
function stringArray(value: unknown, fallback: string[]): string[] {
if (!Array.isArray(value)) {
return fallback;
@ -507,9 +488,6 @@ export function parseKtxProjectConfig(raw: string): KtxProjectConfig {
const parsedConnections = isRecord(parsed.connections)
? (parsed.connections as Record<string, KtxProjectConnectionConfig>)
: defaults.connections;
for (const connectionId of Object.keys(parsedConnections)) {
assertKtxConnectionIdIsNotReserved(connectionId);
}
return {
project: project.trim(),

2
packages/context/src/project/index.ts
View file

@ -7,10 +7,8 @@ export type {
KtxStorageState,
} from './config.js';
export {
assertKtxConnectionIdIsNotReserved,
buildDefaultKtxProjectConfig,
parseKtxProjectConfig,
reservedKtxIngestConnectionIdMessage,
serializeKtxProjectConfig,
} from './config.js';
export type { LocalGitFileStoreDeps } from './local-git-file-store.js';