mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-04 10:52:13 +02:00
feat: merge ingest and scan
* docs: add CLI component reuse guidance * docs: add unified ingest ux design * Refine unified ingest UX design after adversarial review iteration 1 * Refine unified ingest UX design after adversarial review iteration 2 * Refine unified ingest UX design after adversarial review iteration 3 * feat(cli): route public connection ingest command * feat(cli): hide standalone scan from public help * feat(cli): plan public ingest depth and query history * feat(cli): execute public database ingest facets * feat(ingest): read connection query history config * fix(cli): use public ingest wording * fix(config): stop generating ingest adapter allow lists * docs: document public ingest command * test: align ingest surface expectations * docs: add unified ingest public CLI surface plan * feat(cli): preflight deep public ingest readiness * feat(setup): store query history in connection context * feat(setup): store database context depth * feat(setup): verify context readiness by database depth * fix(setup): keep context build foreground only * fix(config): reject reserved ingest connection ids * test: close unified ingest v1 expectations * docs: add unified ingest v1 closure plan * fix(ingest): bypass adapter allow-list for public source ingest * fix(ingest): honor query history window intent * fix(ingest): hide scan internals from public database ingest * feat(ingest): use foreground view for interactive public ingest * fix(setup): use schema context and query history wording * test(cli): verify unified ingest public output * docs: add unified ingest v1 public output closure plan * fix(setup): forward query history flags * fix(setup): prompt for postgres query history * fix(status): report query history readiness * fix(ingest): remove legacy public guidance * fix(ingest): polish foreground retry copy * docs(examples): use unified query history wording * chore(ingest): finish public query history cleanup * docs: add unified ingest v1 query history status cleanup plan * test(docs): cover unified ingest public docs * docs: align ingest CLI reference with unified UX * docs: update context build guides for unified ingest * docs: update setup and primary source ingest wording * docs: stop advertising adapter-backed example ingest * docs: close unified ingest public docs gaps * docs: add unified ingest v1 docs site closure plan * fix: render unified ingest foreground warnings * fix: explain query history schema order * fix: add public ingest retry guidance * fix: align setup next steps with unified ingest * fix: remove scan wording from demo progress * test: verify unified ingest ux closure * docs: add unified ingest v1 foreground and retry closure plan * fix(cli): preserve query-history pull config in public ingest * fix(cli): omit hidden commands from docs command tree * test(cli): close unified ingest final public surface checks * docs: add unified ingest v1 final public surface closure plan * fix(cli): use public source labels in ingest reports * fix(cli): suppress low-level public ingest output * test(cli): verify unified ingest public plain output * docs: add unified ingest v1 public plain output closure plan * fix(cli): add public ingest copy sanitizers * fix(cli): sanitize public ingest progress copy * fix(cli): rename setup schema scope prompt * docs(plan): add progress copy closure; test: align setup back-nav fixture Adds the iter9 plan and updates the setup back-navigation test fixture to pass disableQueryHistory plus listSchemas/listTables stubs that the unified ingest setup step now requires. * docs(plan): add final ux labels plan with narrowed label scans * fix(cli): aggregate unsupported query-history warnings * fix(cli): align setup database labels * test(cli): fix setup database test type-check * fix(cli): remove primary-source wording from setup output * test(cli): verify unified ingest setup closure * docs(plan): add unified ingest v1 verification copy closure plan * fix(cli): remove top-level scan command * fix(cli): remove legacy ingest and wiki commands * Merge scan into ingest flow * feat(cli): split ingest progress into per-phase rows, rename work units to tasks Each database target in the unified ingest dashboard now renders one row per real subprocess (Schema, then Query history when enabled) instead of a single combined bar. Each phase has its own monotonic 0-100% bar so the progress never snaps back to zero when historic-sql starts after scan completes. Completed phases keep their final bar, summary, and elapsed time visible as an inline audit trail; queued and skipped phases are shown explicitly. Also rename user-facing "work units" / "Failed work units" to "tasks" / "Failed tasks" in ingest output and parseIngestSummary. The parser still accepts the legacy "Work units:" wording in captured output for backward compat. Internal memory-flow event names and type fields are left alone. * Fix test harness failures * Fix CI smoke checks --------- Co-authored-by: Andrey Avtomonov <7889985+andreybavt@users.noreply.github.com>
This commit is contained in:
Andrey Avtomonov
GitHub
parent
1a472cf3ed
commit
b00c1a11a9
118 changed files with 16890 additions and 2992 deletions
|
|
@ -1,9 +1,8 @@
|
|||
import assert from 'node:assert/strict';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { dirname } from 'node:path';
|
||||
import { describe, it } from 'node:test';
|
||||
import {
|
||||
buildOrbitScanArgv,
|
||||
buildOrbitScanArgs,
|
||||
defaultOrbitVerificationProjectDir,
|
||||
extractReportPath,
|
||||
extractRunId,
|
||||
|
|
@ -49,6 +48,14 @@ function successReportJson() {
|
|||
});
|
||||
}
|
||||
|
||||
function successfulRunKtxScan(calls = []) {
|
||||
return async (args, io) => {
|
||||
calls.push(args);
|
||||
io.stdout.write('KTX scan completed\nRun: scan-orbit-1\nConnection: orbit\n Report: reports/scan-report.json\n');
|
||||
return 0;
|
||||
};
|
||||
}
|
||||
|
||||
describe('relationship Orbit verification helper', () => {
|
||||
it('exposes the Orbit verification command from the KTX workspace package', async () => {
|
||||
const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8'));
|
||||
|
|
@ -59,20 +66,19 @@ describe('relationship Orbit verification helper', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('builds the current KTX launcher arguments for scan commands', () => {
|
||||
assert.deepEqual(buildOrbitScanArgv({ connectionId: 'orbit', projectDir: '/tmp/orbit-project' }), [
|
||||
'scan',
|
||||
'orbit',
|
||||
'--mode',
|
||||
'relationships',
|
||||
'--project-dir',
|
||||
'/tmp/orbit-project',
|
||||
]);
|
||||
it('builds the internal relationship scan arguments', () => {
|
||||
assert.deepEqual(buildOrbitScanArgs({ connectionId: 'orbit', projectDir: '/tmp/orbit-project' }), {
|
||||
command: 'run',
|
||||
projectDir: '/tmp/orbit-project',
|
||||
connectionId: 'orbit',
|
||||
mode: 'relationships',
|
||||
detectRelationships: true,
|
||||
dryRun: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('uses the checked-in Orbit verification project by default', async () => {
|
||||
const calls = [];
|
||||
const envs = [];
|
||||
const scanCalls = [];
|
||||
const writes = [];
|
||||
const defaultProjectDir = defaultOrbitVerificationProjectDir();
|
||||
|
||||
|
|
@ -83,27 +89,28 @@ describe('relationship Orbit verification helper', () => {
|
|||
writeFile: async (path, content) => {
|
||||
writes.push({ path, content });
|
||||
},
|
||||
runWorkspaceKtx: async (argv, options) => {
|
||||
calls.push(argv);
|
||||
envs.push(options.env);
|
||||
options.stdout.write('KTX scan completed\nRun: scan-orbit-1\nConnection: orbit\n Report: reports/scan-report.json\n');
|
||||
return 0;
|
||||
},
|
||||
runKtxScan: successfulRunKtxScan(scanCalls),
|
||||
readFile: async () => successReportJson(),
|
||||
});
|
||||
|
||||
assert.equal(result.status, 'success');
|
||||
assert.deepEqual(calls, [
|
||||
['scan', 'orbit', '--mode', 'relationships', '--project-dir', defaultProjectDir],
|
||||
assert.deepEqual(scanCalls, [
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: defaultProjectDir,
|
||||
connectionId: 'orbit',
|
||||
mode: 'relationships',
|
||||
detectRelationships: true,
|
||||
dryRun: false,
|
||||
},
|
||||
]);
|
||||
assert.equal(envs[0].GIT_CEILING_DIRECTORIES, dirname(defaultProjectDir));
|
||||
assert.equal(writes.length, 1);
|
||||
assert.match(writes[0].content, new RegExp(defaultProjectDir.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')));
|
||||
});
|
||||
|
||||
it('uses KTX_PROJECT_DIR for the Orbit verification project override', async () => {
|
||||
const previousProjectDir = process.env.KTX_PROJECT_DIR;
|
||||
const calls = [];
|
||||
const scanCalls = [];
|
||||
|
||||
try {
|
||||
process.env.KTX_PROJECT_DIR = '/tmp/orbit-project-from-env';
|
||||
|
|
@ -113,17 +120,20 @@ describe('relationship Orbit verification helper', () => {
|
|||
now: () => new Date('2026-05-07T10:00:00.000Z'),
|
||||
mkdir: async () => {},
|
||||
writeFile: async () => {},
|
||||
runWorkspaceKtx: async (argv, options) => {
|
||||
calls.push(argv);
|
||||
options.stdout.write('KTX scan completed\nRun: scan-orbit-1\nConnection: orbit\n Report: reports/scan-report.json\n');
|
||||
return 0;
|
||||
},
|
||||
runKtxScan: successfulRunKtxScan(scanCalls),
|
||||
readFile: async () => successReportJson(),
|
||||
});
|
||||
|
||||
assert.equal(result.projectDir, '/tmp/orbit-project-from-env');
|
||||
assert.deepEqual(calls, [
|
||||
['scan', 'orbit', '--mode', 'relationships', '--project-dir', '/tmp/orbit-project-from-env'],
|
||||
assert.deepEqual(scanCalls, [
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: '/tmp/orbit-project-from-env',
|
||||
connectionId: 'orbit',
|
||||
mode: 'relationships',
|
||||
detectRelationships: true,
|
||||
dryRun: false,
|
||||
},
|
||||
]);
|
||||
} finally {
|
||||
if (previousProjectDir === undefined) {
|
||||
|
|
@ -146,7 +156,7 @@ describe('relationship Orbit verification helper', () => {
|
|||
date: '2026-05-07',
|
||||
connectionId: 'orbit',
|
||||
projectDir: '/tmp/orbit-project',
|
||||
scanCommand: 'pnpm run ktx -- scan orbit --mode relationships --project-dir /tmp/orbit-project',
|
||||
scanCommand: 'internal runKtxScan connection=orbit mode=relationships projectDir=/tmp/orbit-project',
|
||||
reportPath: '/tmp/orbit-project/reports/scan-report.json',
|
||||
scanExitCode: 0,
|
||||
scanStdout: 'KTX scan completed\nRun: scan-orbit-1\n',
|
||||
|
|
@ -171,7 +181,7 @@ describe('relationship Orbit verification helper', () => {
|
|||
date: '2026-05-07',
|
||||
connectionId: 'orbit',
|
||||
projectDir: '/tmp/orbit-project',
|
||||
scanCommand: 'pnpm run ktx -- scan orbit --mode relationships --project-dir /tmp/orbit-project',
|
||||
scanCommand: 'internal runKtxScan connection=orbit mode=relationships projectDir=/tmp/orbit-project',
|
||||
scanExitCode: 1,
|
||||
blocker: 'Connection "orbit" was not found',
|
||||
scanStdout: '',
|
||||
|
|
@ -180,12 +190,12 @@ describe('relationship Orbit verification helper', () => {
|
|||
|
||||
assert.match(markdown, /Exit code: 1/);
|
||||
assert.match(markdown, /Connection "orbit" was not found/);
|
||||
assert.match(markdown, /Orbit verification was not executed because the current local Orbit scan command failed/);
|
||||
assert.match(markdown, /Orbit verification was not executed because the current local Orbit relationship scan failed/);
|
||||
assert.doesNotMatch(markdown, /scan\.enrichment\.mode is required/);
|
||||
});
|
||||
|
||||
it('runs scan then reads the report artifact and writes success Markdown', async () => {
|
||||
const calls = [];
|
||||
const scanCalls = [];
|
||||
const writes = [];
|
||||
const result = await runOrbitVerification({
|
||||
connectionId: 'orbit',
|
||||
|
|
@ -196,24 +206,27 @@ describe('relationship Orbit verification helper', () => {
|
|||
writeFile: async (path, content) => {
|
||||
writes.push({ path, content });
|
||||
},
|
||||
runWorkspaceKtx: async (argv, options) => {
|
||||
calls.push(argv);
|
||||
options.stdout.write('KTX scan completed\nRun: scan-orbit-1\nConnection: orbit\n Report: reports/scan-report.json\n');
|
||||
return 0;
|
||||
},
|
||||
runKtxScan: successfulRunKtxScan(scanCalls),
|
||||
readFile: async () => successReportJson(),
|
||||
});
|
||||
|
||||
assert.equal(result.status, 'success');
|
||||
assert.deepEqual(calls, [
|
||||
['scan', 'orbit', '--mode', 'relationships', '--project-dir', '/tmp/orbit-project'],
|
||||
assert.deepEqual(scanCalls, [
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: '/tmp/orbit-project',
|
||||
connectionId: 'orbit',
|
||||
mode: 'relationships',
|
||||
detectRelationships: true,
|
||||
dryRun: false,
|
||||
},
|
||||
]);
|
||||
assert.equal(writes.length, 1);
|
||||
assert.equal(writes[0].path, '/tmp/orbit-report.md');
|
||||
assert.match(writes[0].content, /Accepted: 14/);
|
||||
});
|
||||
|
||||
it('writes blocked Markdown when the scan command fails before a run id exists', async () => {
|
||||
it('writes blocked Markdown when the internal scan fails before a run id exists', async () => {
|
||||
const writes = [];
|
||||
const result = await runOrbitVerification({
|
||||
connectionId: 'orbit',
|
||||
|
|
@ -224,8 +237,8 @@ describe('relationship Orbit verification helper', () => {
|
|||
writeFile: async (path, content) => {
|
||||
writes.push({ path, content });
|
||||
},
|
||||
runWorkspaceKtx: async (_argv, options) => {
|
||||
options.stderr.write('Connection "orbit" was not found\n');
|
||||
runKtxScan: async (_args, io) => {
|
||||
io.stderr.write('Connection "orbit" was not found\n');
|
||||
return 1;
|
||||
},
|
||||
});
|
||||
|
|
@ -236,7 +249,7 @@ describe('relationship Orbit verification helper', () => {
|
|||
assert.match(writes[0].content, /Connection "orbit" was not found/);
|
||||
});
|
||||
|
||||
it('runs the workspace launcher in buffered mode so real scan errors are captured', async () => {
|
||||
it('runs the workspace launcher in buffered mode when preparing the internal scan module', async () => {
|
||||
let sawExecFile = false;
|
||||
const result = await runOrbitVerification({
|
||||
connectionId: 'orbit',
|
||||
|
|
@ -246,7 +259,8 @@ describe('relationship Orbit verification helper', () => {
|
|||
mkdir: async () => {},
|
||||
writeFile: async () => {},
|
||||
execFile: async () => ({ stdout: '', stderr: '' }),
|
||||
runWorkspaceKtx: async (_argv, options) => {
|
||||
runWorkspaceKtx: async (argv, options) => {
|
||||
assert.deepEqual(argv, ['--version']);
|
||||
sawExecFile = typeof options.execFile === 'function';
|
||||
options.stderr.write('ENOENT: no such file or directory, open \'/tmp/orbit-project/ktx.yaml\'\n');
|
||||
return 1;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue