mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-25 08:48:08 +02:00
feat: merge ingest and scan
* docs: add CLI component reuse guidance * docs: add unified ingest ux design * Refine unified ingest UX design after adversarial review iteration 1 * Refine unified ingest UX design after adversarial review iteration 2 * Refine unified ingest UX design after adversarial review iteration 3 * feat(cli): route public connection ingest command * feat(cli): hide standalone scan from public help * feat(cli): plan public ingest depth and query history * feat(cli): execute public database ingest facets * feat(ingest): read connection query history config * fix(cli): use public ingest wording * fix(config): stop generating ingest adapter allow lists * docs: document public ingest command * test: align ingest surface expectations * docs: add unified ingest public CLI surface plan * feat(cli): preflight deep public ingest readiness * feat(setup): store query history in connection context * feat(setup): store database context depth * feat(setup): verify context readiness by database depth * fix(setup): keep context build foreground only * fix(config): reject reserved ingest connection ids * test: close unified ingest v1 expectations * docs: add unified ingest v1 closure plan * fix(ingest): bypass adapter allow-list for public source ingest * fix(ingest): honor query history window intent * fix(ingest): hide scan internals from public database ingest * feat(ingest): use foreground view for interactive public ingest * fix(setup): use schema context and query history wording * test(cli): verify unified ingest public output * docs: add unified ingest v1 public output closure plan * fix(setup): forward query history flags * fix(setup): prompt for postgres query history * fix(status): report query history readiness * fix(ingest): remove legacy public guidance * fix(ingest): polish foreground retry copy * docs(examples): use unified query history wording * chore(ingest): finish public query history cleanup * docs: add unified ingest v1 query history status cleanup plan * test(docs): cover unified ingest public docs * docs: align ingest CLI reference with unified UX * docs: update context build guides for unified ingest * docs: update setup and primary source ingest wording * docs: stop advertising adapter-backed example ingest * docs: close unified ingest public docs gaps * docs: add unified ingest v1 docs site closure plan * fix: render unified ingest foreground warnings * fix: explain query history schema order * fix: add public ingest retry guidance * fix: align setup next steps with unified ingest * fix: remove scan wording from demo progress * test: verify unified ingest ux closure * docs: add unified ingest v1 foreground and retry closure plan * fix(cli): preserve query-history pull config in public ingest * fix(cli): omit hidden commands from docs command tree * test(cli): close unified ingest final public surface checks * docs: add unified ingest v1 final public surface closure plan * fix(cli): use public source labels in ingest reports * fix(cli): suppress low-level public ingest output * test(cli): verify unified ingest public plain output * docs: add unified ingest v1 public plain output closure plan * fix(cli): add public ingest copy sanitizers * fix(cli): sanitize public ingest progress copy * fix(cli): rename setup schema scope prompt * docs(plan): add progress copy closure; test: align setup back-nav fixture Adds the iter9 plan and updates the setup back-navigation test fixture to pass disableQueryHistory plus listSchemas/listTables stubs that the unified ingest setup step now requires. * docs(plan): add final ux labels plan with narrowed label scans * fix(cli): aggregate unsupported query-history warnings * fix(cli): align setup database labels * test(cli): fix setup database test type-check * fix(cli): remove primary-source wording from setup output * test(cli): verify unified ingest setup closure * docs(plan): add unified ingest v1 verification copy closure plan * fix(cli): remove top-level scan command * fix(cli): remove legacy ingest and wiki commands * Merge scan into ingest flow * feat(cli): split ingest progress into per-phase rows, rename work units to tasks Each database target in the unified ingest dashboard now renders one row per real subprocess (Schema, then Query history when enabled) instead of a single combined bar. Each phase has its own monotonic 0-100% bar so the progress never snaps back to zero when historic-sql starts after scan completes. Completed phases keep their final bar, summary, and elapsed time visible as an inline audit trail; queued and skipped phases are shown explicitly. Also rename user-facing "work units" / "Failed work units" to "tasks" / "Failed tasks" in ingest output and parseIngestSummary. The parser still accepts the legacy "Work units:" wording in captured output for backward compat. Internal memory-flow event names and type fields are left alone. * Fix test harness failures * Fix CI smoke checks --------- Co-authored-by: Andrey Avtomonov <7889985+andreybavt@users.noreply.github.com>
This commit is contained in:
parent
1a472cf3ed
commit
b00c1a11a9
118 changed files with 16890 additions and 2992 deletions
|
|
@ -255,6 +255,37 @@ describe('setup sources step', () => {
|
|||
expect((await readConfig()).connections['notion-main']?.last_successful_cursor).toBeUndefined();
|
||||
});
|
||||
|
||||
it('accepts former ingest subcommand names as interactive source connection ids', async () => {
|
||||
await addPrimarySource();
|
||||
const io = makeIo();
|
||||
const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'workspace=ok' }));
|
||||
|
||||
const result = await runKtxSetupSourcesStep(
|
||||
{
|
||||
projectDir,
|
||||
inputMode: 'auto',
|
||||
runInitialSourceIngest: false,
|
||||
skipSources: false,
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
prompts: prompts({
|
||||
multiselect: [['notion']],
|
||||
text: ['status', 'env:NOTION_TOKEN'],
|
||||
select: ['env', 'all_accessible'],
|
||||
}),
|
||||
validateNotion,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.status).toBe('ready');
|
||||
const config = await readConfig();
|
||||
expect(config.connections.status).toMatchObject({
|
||||
driver: 'notion',
|
||||
auth_token_ref: 'env:NOTION_TOKEN',
|
||||
});
|
||||
});
|
||||
|
||||
it('uses selected Notion roots when root page ids are provided even if crawl mode says all accessible', async () => {
|
||||
await addPrimarySource();
|
||||
const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'roots=1' }));
|
||||
|
|
@ -756,7 +787,7 @@ describe('setup sources step', () => {
|
|||
expect(testPrompts.text).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
|
||||
it('enables the dbt adapter when adding a dbt source connection', async () => {
|
||||
it('adds a dbt source connection and enables its adapter', async () => {
|
||||
await addPrimarySource();
|
||||
const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' }));
|
||||
|
||||
|
|
@ -776,7 +807,10 @@ describe('setup sources step', () => {
|
|||
),
|
||||
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] });
|
||||
|
||||
expect((await readConfig()).ingest.adapters).toContain('dbt');
|
||||
const configText = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
|
||||
expect(configText).not.toContain('live-database');
|
||||
expect(configText).not.toContain('historic-sql');
|
||||
expect((await readConfig()).ingest.adapters).toEqual(['dbt']);
|
||||
});
|
||||
|
||||
it('lets interactive setup retry or continue after initial source ingest fails', async () => {
|
||||
|
|
@ -805,7 +839,9 @@ describe('setup sources step', () => {
|
|||
expect(runInitialIngest).toHaveBeenCalledTimes(1);
|
||||
expect((await readConfig()).connections['dbt-main']).toMatchObject({ driver: 'dbt', source_dir: '/repo/dbt' });
|
||||
expect(io.stdout()).toContain('Context source saved without a completed context build for dbt-main.');
|
||||
expect(io.stdout()).toContain('Run later: ktx ingest run --connection-id dbt-main --adapter <adapter>');
|
||||
expect(io.stdout()).toContain('Run later: ktx ingest dbt-main');
|
||||
expect(io.stdout()).not.toContain('ktx ingest run --connection-id');
|
||||
expect(io.stdout()).not.toContain('--adapter');
|
||||
});
|
||||
|
||||
it('retries initial source ingest from the failure menu', async () => {
|
||||
|
|
@ -1472,7 +1508,7 @@ describe('setup sources step', () => {
|
|||
}
|
||||
});
|
||||
|
||||
it('does not offer context sources until a primary source exists', async () => {
|
||||
it('does not offer context sources until a database exists', async () => {
|
||||
const io = makeIo();
|
||||
const testPrompts = prompts({ multiselect: [['notion']] });
|
||||
|
||||
|
|
@ -1485,7 +1521,7 @@ describe('setup sources step', () => {
|
|||
).resolves.toEqual({ status: 'skipped', projectDir });
|
||||
|
||||
expect(testPrompts.multiselect).not.toHaveBeenCalled();
|
||||
expect(io.stdout()).toContain('Connect a primary source before adding context sources.');
|
||||
expect(io.stdout()).toContain('Connect a database before adding context sources.');
|
||||
expect(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue