mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-16 08:25:14 +02:00
feat: merge ingest and scan
* docs: add CLI component reuse guidance * docs: add unified ingest ux design * Refine unified ingest UX design after adversarial review iteration 1 * Refine unified ingest UX design after adversarial review iteration 2 * Refine unified ingest UX design after adversarial review iteration 3 * feat(cli): route public connection ingest command * feat(cli): hide standalone scan from public help * feat(cli): plan public ingest depth and query history * feat(cli): execute public database ingest facets * feat(ingest): read connection query history config * fix(cli): use public ingest wording * fix(config): stop generating ingest adapter allow lists * docs: document public ingest command * test: align ingest surface expectations * docs: add unified ingest public CLI surface plan * feat(cli): preflight deep public ingest readiness * feat(setup): store query history in connection context * feat(setup): store database context depth * feat(setup): verify context readiness by database depth * fix(setup): keep context build foreground only * fix(config): reject reserved ingest connection ids * test: close unified ingest v1 expectations * docs: add unified ingest v1 closure plan * fix(ingest): bypass adapter allow-list for public source ingest * fix(ingest): honor query history window intent * fix(ingest): hide scan internals from public database ingest * feat(ingest): use foreground view for interactive public ingest * fix(setup): use schema context and query history wording * test(cli): verify unified ingest public output * docs: add unified ingest v1 public output closure plan * fix(setup): forward query history flags * fix(setup): prompt for postgres query history * fix(status): report query history readiness * fix(ingest): remove legacy public guidance * fix(ingest): polish foreground retry copy * docs(examples): use unified query history wording * chore(ingest): finish public query history cleanup * docs: add unified ingest v1 query history status cleanup plan * test(docs): cover unified ingest public docs * docs: align ingest CLI reference with unified UX * docs: update context build guides for unified ingest * docs: update setup and primary source ingest wording * docs: stop advertising adapter-backed example ingest * docs: close unified ingest public docs gaps * docs: add unified ingest v1 docs site closure plan * fix: render unified ingest foreground warnings * fix: explain query history schema order * fix: add public ingest retry guidance * fix: align setup next steps with unified ingest * fix: remove scan wording from demo progress * test: verify unified ingest ux closure * docs: add unified ingest v1 foreground and retry closure plan * fix(cli): preserve query-history pull config in public ingest * fix(cli): omit hidden commands from docs command tree * test(cli): close unified ingest final public surface checks * docs: add unified ingest v1 final public surface closure plan * fix(cli): use public source labels in ingest reports * fix(cli): suppress low-level public ingest output * test(cli): verify unified ingest public plain output * docs: add unified ingest v1 public plain output closure plan * fix(cli): add public ingest copy sanitizers * fix(cli): sanitize public ingest progress copy * fix(cli): rename setup schema scope prompt * docs(plan): add progress copy closure; test: align setup back-nav fixture Adds the iter9 plan and updates the setup back-navigation test fixture to pass disableQueryHistory plus listSchemas/listTables stubs that the unified ingest setup step now requires. * docs(plan): add final ux labels plan with narrowed label scans * fix(cli): aggregate unsupported query-history warnings * fix(cli): align setup database labels * test(cli): fix setup database test type-check * fix(cli): remove primary-source wording from setup output * test(cli): verify unified ingest setup closure * docs(plan): add unified ingest v1 verification copy closure plan * fix(cli): remove top-level scan command * fix(cli): remove legacy ingest and wiki commands * Merge scan into ingest flow * feat(cli): split ingest progress into per-phase rows, rename work units to tasks Each database target in the unified ingest dashboard now renders one row per real subprocess (Schema, then Query history when enabled) instead of a single combined bar. Each phase has its own monotonic 0-100% bar so the progress never snaps back to zero when historic-sql starts after scan completes. Completed phases keep their final bar, summary, and elapsed time visible as an inline audit trail; queued and skipped phases are shown explicitly. Also rename user-facing "work units" / "Failed work units" to "tasks" / "Failed tasks" in ingest output and parseIngestSummary. The parser still accepts the legacy "Work units:" wording in captured output for backward compat. Internal memory-flow event names and type fields are left alone. * Fix test harness failures * Fix CI smoke checks --------- Co-authored-by: Andrey Avtomonov <7889985+andreybavt@users.noreply.github.com>
This commit is contained in:
Andrey Avtomonov
GitHub
parent
1a472cf3ed
commit
b00c1a11a9
118 changed files with 16890 additions and 2992 deletions
131
packages/cli/src/setup-database-context-depth.ts
Normal file
131
packages/cli/src/setup-database-context-depth.ts
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
import { writeFile } from 'node:fs/promises';
|
||||
import {
|
||||
type KtxLocalProject,
|
||||
type KtxProjectConnectionConfig,
|
||||
loadKtxProject,
|
||||
serializeKtxProjectConfig,
|
||||
} from '@ktx/context/project';
|
||||
import {
|
||||
type KtxDatabaseContextDepth,
|
||||
databaseContextDepth,
|
||||
deepReadinessGaps,
|
||||
isDatabaseDriver,
|
||||
normalizeConnectionDriver,
|
||||
recommendedDatabaseContextDepth,
|
||||
withDatabaseContextDepth,
|
||||
} from './ingest-depth.js';
|
||||
import type { KtxSetupPromptOption } from './setup-prompts.js';
|
||||
|
||||
export interface KtxSetupDatabaseContextDepthArgs {
|
||||
inputMode: 'auto' | 'disabled';
|
||||
}
|
||||
|
||||
export interface KtxSetupDatabaseContextDepthPromptAdapter {
|
||||
select(options: { message: string; options: KtxSetupPromptOption[] }): Promise<string>;
|
||||
}
|
||||
|
||||
function databaseConnectionsNeedingDepth(project: KtxLocalProject): string[] {
|
||||
return Object.entries(project.config.connections)
|
||||
.filter(([, connection]) => isDatabaseDriver(normalizeConnectionDriver(connection)))
|
||||
.filter(([, connection]) => databaseContextDepth(connection) === undefined)
|
||||
.map(([connectionId]) => connectionId)
|
||||
.sort((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
async function chooseSetupDatabaseContextDepth(input: {
|
||||
project: KtxLocalProject;
|
||||
args: KtxSetupDatabaseContextDepthArgs;
|
||||
prompts: KtxSetupDatabaseContextDepthPromptAdapter;
|
||||
}): Promise<KtxDatabaseContextDepth | 'back'> {
|
||||
const recommended = recommendedDatabaseContextDepth(input.project.config);
|
||||
if (input.args.inputMode === 'disabled') {
|
||||
return recommended;
|
||||
}
|
||||
|
||||
const deepReady = deepReadinessGaps(input.project.config).length === 0;
|
||||
const options =
|
||||
recommended === 'deep'
|
||||
? [
|
||||
{ value: 'deep', label: 'Deep: AI descriptions, embeddings, relationships, slower' },
|
||||
{ value: 'fast', label: 'Fast: schema only, no AI, quickest' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
]
|
||||
: [
|
||||
{ value: 'fast', label: 'Fast: schema only, no AI, quickest' },
|
||||
{ value: 'deep', label: 'Deep: AI descriptions, embeddings, relationships, slower' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
];
|
||||
|
||||
const choice = await input.prompts.select({
|
||||
message:
|
||||
'How much database context should KTX build?\n\n' +
|
||||
(deepReady
|
||||
? 'Deep is available because model, embedding, and scan enrichment are configured.'
|
||||
: 'Fast is recommended because model, embedding, or scan enrichment is not configured.'),
|
||||
options,
|
||||
});
|
||||
if (choice === 'back') {
|
||||
return 'back';
|
||||
}
|
||||
if (choice === 'fast' || choice === 'deep') {
|
||||
return choice;
|
||||
}
|
||||
return recommended;
|
||||
}
|
||||
|
||||
async function writeDatabaseContextDepths(
|
||||
project: KtxLocalProject,
|
||||
connectionIds: string[],
|
||||
depth: KtxDatabaseContextDepth,
|
||||
): Promise<KtxLocalProject> {
|
||||
if (connectionIds.length === 0) {
|
||||
return project;
|
||||
}
|
||||
const nextConnections = { ...project.config.connections };
|
||||
for (const connectionId of connectionIds) {
|
||||
const connection = nextConnections[connectionId];
|
||||
if (connection) {
|
||||
nextConnections[connectionId] = withDatabaseContextDepth(connection, depth);
|
||||
}
|
||||
}
|
||||
const nextConfig = { ...project.config, connections: nextConnections };
|
||||
await writeFile(project.configPath, serializeKtxProjectConfig(nextConfig), 'utf-8');
|
||||
return await loadKtxProject({ projectDir: project.projectDir });
|
||||
}
|
||||
|
||||
export async function ensureSetupDatabaseContextDepths(input: {
|
||||
project: KtxLocalProject;
|
||||
args: KtxSetupDatabaseContextDepthArgs;
|
||||
prompts: KtxSetupDatabaseContextDepthPromptAdapter;
|
||||
}): Promise<KtxLocalProject | 'back'> {
|
||||
const missingDepthConnectionIds = databaseConnectionsNeedingDepth(input.project);
|
||||
if (missingDepthConnectionIds.length === 0) {
|
||||
return input.project;
|
||||
}
|
||||
|
||||
const depth = await chooseSetupDatabaseContextDepth(input);
|
||||
if (depth === 'back') {
|
||||
return 'back';
|
||||
}
|
||||
return await writeDatabaseContextDepths(input.project, missingDepthConnectionIds, depth);
|
||||
}
|
||||
|
||||
export async function applySetupDatabaseContextDepth(input: {
|
||||
project: KtxLocalProject;
|
||||
connection: KtxProjectConnectionConfig;
|
||||
args: KtxSetupDatabaseContextDepthArgs;
|
||||
prompts: KtxSetupDatabaseContextDepthPromptAdapter;
|
||||
}): Promise<KtxProjectConnectionConfig | 'back'> {
|
||||
if (
|
||||
!isDatabaseDriver(normalizeConnectionDriver(input.connection)) ||
|
||||
databaseContextDepth(input.connection) !== undefined
|
||||
) {
|
||||
return input.connection;
|
||||
}
|
||||
|
||||
const depth = await chooseSetupDatabaseContextDepth(input);
|
||||
if (depth === 'back') {
|
||||
return 'back';
|
||||
}
|
||||
return withDatabaseContextDepth(input.connection, depth);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue