feat: merge ingest and scan

* docs: add CLI component reuse guidance

* docs: add unified ingest ux design

* Refine unified ingest UX design after adversarial review iteration 1

* Refine unified ingest UX design after adversarial review iteration 2

* Refine unified ingest UX design after adversarial review iteration 3

* feat(cli): route public connection ingest command

* feat(cli): hide standalone scan from public help

* feat(cli): plan public ingest depth and query history

* feat(cli): execute public database ingest facets

* feat(ingest): read connection query history config

* fix(cli): use public ingest wording

* fix(config): stop generating ingest adapter allow lists

* docs: document public ingest command

* test: align ingest surface expectations

* docs: add unified ingest public CLI surface plan

* feat(cli): preflight deep public ingest readiness

* feat(setup): store query history in connection context

* feat(setup): store database context depth

* feat(setup): verify context readiness by database depth

* fix(setup): keep context build foreground only

* fix(config): reject reserved ingest connection ids

* test: close unified ingest v1 expectations

* docs: add unified ingest v1 closure plan

* fix(ingest): bypass adapter allow-list for public source ingest

* fix(ingest): honor query history window intent

* fix(ingest): hide scan internals from public database ingest

* feat(ingest): use foreground view for interactive public ingest

* fix(setup): use schema context and query history wording

* test(cli): verify unified ingest public output

* docs: add unified ingest v1 public output closure plan

* fix(setup): forward query history flags

* fix(setup): prompt for postgres query history

* fix(status): report query history readiness

* fix(ingest): remove legacy public guidance

* fix(ingest): polish foreground retry copy

* docs(examples): use unified query history wording

* chore(ingest): finish public query history cleanup

* docs: add unified ingest v1 query history status cleanup plan

* test(docs): cover unified ingest public docs

* docs: align ingest CLI reference with unified UX

* docs: update context build guides for unified ingest

* docs: update setup and primary source ingest wording

* docs: stop advertising adapter-backed example ingest

* docs: close unified ingest public docs gaps

* docs: add unified ingest v1 docs site closure plan

* fix: render unified ingest foreground warnings

* fix: explain query history schema order

* fix: add public ingest retry guidance

* fix: align setup next steps with unified ingest

* fix: remove scan wording from demo progress

* test: verify unified ingest ux closure

* docs: add unified ingest v1 foreground and retry closure plan

* fix(cli): preserve query-history pull config in public ingest

* fix(cli): omit hidden commands from docs command tree

* test(cli): close unified ingest final public surface checks

* docs: add unified ingest v1 final public surface closure plan

* fix(cli): use public source labels in ingest reports

* fix(cli): suppress low-level public ingest output

* test(cli): verify unified ingest public plain output

* docs: add unified ingest v1 public plain output closure plan

* fix(cli): add public ingest copy sanitizers

* fix(cli): sanitize public ingest progress copy

* fix(cli): rename setup schema scope prompt

* docs(plan): add progress copy closure; test: align setup back-nav fixture

Adds the iter9 plan and updates the setup back-navigation test fixture
to pass disableQueryHistory plus listSchemas/listTables stubs that the
unified ingest setup step now requires.

* docs(plan): add final ux labels plan with narrowed label scans

* fix(cli): aggregate unsupported query-history warnings

* fix(cli): align setup database labels

* test(cli): fix setup database test type-check

* fix(cli): remove primary-source wording from setup output

* test(cli): verify unified ingest setup closure

* docs(plan): add unified ingest v1 verification copy closure plan

* fix(cli): remove top-level scan command

* fix(cli): remove legacy ingest and wiki commands

* Merge scan into ingest flow

* feat(cli): split ingest progress into per-phase rows, rename work units to tasks

Each database target in the unified ingest dashboard now renders one row per
real subprocess (Schema, then Query history when enabled) instead of a single
combined bar. Each phase has its own monotonic 0-100% bar so the progress
never snaps back to zero when historic-sql starts after scan completes.
Completed phases keep their final bar, summary, and elapsed time visible as
an inline audit trail; queued and skipped phases are shown explicitly.

Also rename user-facing "work units" / "Failed work units" to "tasks" /
"Failed tasks" in ingest output and parseIngestSummary. The parser still
accepts the legacy "Work units:" wording in captured output for backward
compat. Internal memory-flow event names and type fields are left alone.

* Fix test harness failures

* Fix CI smoke checks

---------

Co-authored-by: Andrey Avtomonov <7889985+andreybavt@users.noreply.github.com>
This commit is contained in:
Andrey Avtomonov 2026-05-14 01:43:06 +02:00 committed by GitHub
parent 1a472cf3ed
commit b00c1a11a9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
118 changed files with 16890 additions and 2992 deletions

View file

@ -41,8 +41,8 @@ export function defaultOrbitVerificationProjectDir() {
return defaultProjectDir;
}
function shellCommand(argv) {
return ['pnpm', 'run', 'ktx', '--', ...argv].join(' ');
function internalScanCommand(input) {
return `internal runKtxScan connection=${input.connectionId} mode=relationships projectDir=${input.projectDir}`;
}
function firstNonEmptyLine(...values) {
@ -55,7 +55,7 @@ function firstNonEmptyLine(...values) {
return line;
}
}
return 'Orbit scan command failed before producing diagnostic output';
return 'Orbit relationship scan failed before producing diagnostic output';
}
function parseArgs(argv) {
@ -88,8 +88,15 @@ function parseArgs(argv) {
return options;
}
export function buildOrbitScanArgv(input) {
return ['scan', input.connectionId, '--mode', 'relationships', '--project-dir', input.projectDir];
export function buildOrbitScanArgs(input) {
return {
command: 'run',
projectDir: input.projectDir,
connectionId: input.connectionId,
mode: 'relationships',
detectRelationships: true,
dryRun: false,
};
}
export function extractRunId(stdout) {
@ -171,7 +178,7 @@ function formatBlocked(result) {
'',
'## Evidence',
'',
'- Orbit verification was not executed because the current local Orbit scan command failed.',
'- Orbit verification was not executed because the current local Orbit relationship scan failed.',
'- Re-run with `--report-path` to write verification evidence to a custom location.',
'',
'Scan stdout:',
@ -228,6 +235,36 @@ async function runBufferedWorkspaceKtx(runner, argv, rootDir, execFile) {
};
}
function cliScanModulePath(rootDir) {
return resolve(rootDir, 'packages/cli/dist/scan.js');
}
async function loadRunKtxScan(rootDir) {
const module = await import(pathToFileURL(cliScanModulePath(rootDir)).href);
return module.runKtxScan;
}
async function runBufferedInternalScan(input) {
const stdout = new BufferWriter();
const stderr = new BufferWriter();
let runKtxScan = input.runKtxScan;
if (!runKtxScan) {
const build = await runBufferedWorkspaceKtx(input.runner, ['--version'], input.rootDir, input.execFile);
if (build.exitCode !== 0) {
return build;
}
runKtxScan = await loadRunKtxScan(input.rootDir);
}
const exitCode = await runKtxScan(input.scanArgs, { stdout, stderr });
return {
exitCode,
stdout: stdout.text(),
stderr: stderr.text(),
};
}
function orbitVerificationEnv(projectDir) {
if (projectDir !== defaultProjectDir) {
return process.env;
@ -253,8 +290,15 @@ export async function runOrbitVerification(options = {}) {
const env = options.env ?? orbitVerificationEnv(projectDir);
const runWithEnv = (argv, runnerOptions) => runner(argv, { ...runnerOptions, env });
const scanArgv = buildOrbitScanArgv({ connectionId, projectDir });
const scan = await runBufferedWorkspaceKtx(runWithEnv, scanArgv, rootDir, execFile);
const scanArgs = buildOrbitScanArgs({ connectionId, projectDir });
const scanCommand = internalScanCommand({ connectionId, projectDir });
const scan = await runBufferedInternalScan({
scanArgs,
rootDir,
execFile,
runner: runWithEnv,
runKtxScan: options.runKtxScan,
});
let result;
if (scan.exitCode !== 0) {
@ -263,7 +307,7 @@ export async function runOrbitVerification(options = {}) {
date,
connectionId,
projectDir,
scanCommand: shellCommand(scanArgv),
scanCommand,
scanExitCode: scan.exitCode,
blocker: firstNonEmptyLine(scan.stderr, scan.stdout),
scanStdout: scan.stdout,
@ -277,7 +321,7 @@ export async function runOrbitVerification(options = {}) {
date,
connectionId,
projectDir,
scanCommand: shellCommand(scanArgv),
scanCommand,
scanExitCode: scan.exitCode,
blocker: 'KTX scan completed without printing a Run id',
scanStdout: scan.stdout,
@ -291,7 +335,7 @@ export async function runOrbitVerification(options = {}) {
date,
connectionId,
projectDir,
scanCommand: shellCommand(scanArgv),
scanCommand,
scanExitCode: scan.exitCode,
blocker: 'KTX scan completed without printing a report artifact path',
scanStdout: scan.stdout,
@ -304,7 +348,7 @@ export async function runOrbitVerification(options = {}) {
date,
connectionId,
projectDir,
scanCommand: shellCommand(scanArgv),
scanCommand,
reportPath: fullScanReportPath,
scanExitCode: scan.exitCode,
scanStdout: scan.stdout,