mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-22 08:38:08 +02:00
feat: merge ingest and scan
* docs: add CLI component reuse guidance * docs: add unified ingest ux design * Refine unified ingest UX design after adversarial review iteration 1 * Refine unified ingest UX design after adversarial review iteration 2 * Refine unified ingest UX design after adversarial review iteration 3 * feat(cli): route public connection ingest command * feat(cli): hide standalone scan from public help * feat(cli): plan public ingest depth and query history * feat(cli): execute public database ingest facets * feat(ingest): read connection query history config * fix(cli): use public ingest wording * fix(config): stop generating ingest adapter allow lists * docs: document public ingest command * test: align ingest surface expectations * docs: add unified ingest public CLI surface plan * feat(cli): preflight deep public ingest readiness * feat(setup): store query history in connection context * feat(setup): store database context depth * feat(setup): verify context readiness by database depth * fix(setup): keep context build foreground only * fix(config): reject reserved ingest connection ids * test: close unified ingest v1 expectations * docs: add unified ingest v1 closure plan * fix(ingest): bypass adapter allow-list for public source ingest * fix(ingest): honor query history window intent * fix(ingest): hide scan internals from public database ingest * feat(ingest): use foreground view for interactive public ingest * fix(setup): use schema context and query history wording * test(cli): verify unified ingest public output * docs: add unified ingest v1 public output closure plan * fix(setup): forward query history flags * fix(setup): prompt for postgres query history * fix(status): report query history readiness * fix(ingest): remove legacy public guidance * fix(ingest): polish foreground retry copy * docs(examples): use unified query history wording * chore(ingest): finish public query history cleanup * docs: add unified ingest v1 query history status cleanup plan * test(docs): cover unified ingest public docs * docs: align ingest CLI reference with unified UX * docs: update context build guides for unified ingest * docs: update setup and primary source ingest wording * docs: stop advertising adapter-backed example ingest * docs: close unified ingest public docs gaps * docs: add unified ingest v1 docs site closure plan * fix: render unified ingest foreground warnings * fix: explain query history schema order * fix: add public ingest retry guidance * fix: align setup next steps with unified ingest * fix: remove scan wording from demo progress * test: verify unified ingest ux closure * docs: add unified ingest v1 foreground and retry closure plan * fix(cli): preserve query-history pull config in public ingest * fix(cli): omit hidden commands from docs command tree * test(cli): close unified ingest final public surface checks * docs: add unified ingest v1 final public surface closure plan * fix(cli): use public source labels in ingest reports * fix(cli): suppress low-level public ingest output * test(cli): verify unified ingest public plain output * docs: add unified ingest v1 public plain output closure plan * fix(cli): add public ingest copy sanitizers * fix(cli): sanitize public ingest progress copy * fix(cli): rename setup schema scope prompt * docs(plan): add progress copy closure; test: align setup back-nav fixture Adds the iter9 plan and updates the setup back-navigation test fixture to pass disableQueryHistory plus listSchemas/listTables stubs that the unified ingest setup step now requires. * docs(plan): add final ux labels plan with narrowed label scans * fix(cli): aggregate unsupported query-history warnings * fix(cli): align setup database labels * test(cli): fix setup database test type-check * fix(cli): remove primary-source wording from setup output * test(cli): verify unified ingest setup closure * docs(plan): add unified ingest v1 verification copy closure plan * fix(cli): remove top-level scan command * fix(cli): remove legacy ingest and wiki commands * Merge scan into ingest flow * feat(cli): split ingest progress into per-phase rows, rename work units to tasks Each database target in the unified ingest dashboard now renders one row per real subprocess (Schema, then Query history when enabled) instead of a single combined bar. Each phase has its own monotonic 0-100% bar so the progress never snaps back to zero when historic-sql starts after scan completes. Completed phases keep their final bar, summary, and elapsed time visible as an inline audit trail; queued and skipped phases are shown explicitly. Also rename user-facing "work units" / "Failed work units" to "tasks" / "Failed tasks" in ingest output and parseIngestSummary. The parser still accepts the legacy "Work units:" wording in captured output for backward compat. Internal memory-flow event names and type fields are left alone. * Fix test harness failures * Fix CI smoke checks --------- Co-authored-by: Andrey Avtomonov <7889985+andreybavt@users.noreply.github.com>
This commit is contained in:
Andrey Avtomonov
GitHub
parent
1a472cf3ed
commit
b00c1a11a9
118 changed files with 16890 additions and 2992 deletions
|
|
@ -7,7 +7,7 @@ import { writeKtxSetupState } from '@ktx/context/project';
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { localFakeBundleReport, persistLocalBundleReport } from './ingest.test-utils.js';
|
||||
import { contextBuildCommands, writeKtxSetupContextState } from './setup-context.js';
|
||||
import { contextBuildCommands, readKtxSetupContextState, writeKtxSetupContextState } from './setup-context.js';
|
||||
import { runDemoTour } from './setup-demo-tour.js';
|
||||
import { formatKtxSetupStatus, readKtxSetupStatus, runKtxSetup } from './setup.js';
|
||||
|
||||
|
|
@ -297,10 +297,10 @@ describe('setup status', () => {
|
|||
await expect(readKtxSetupStatus(tempDir)).resolves.toMatchObject({
|
||||
context: {
|
||||
ready: false,
|
||||
status: 'running',
|
||||
status: 'stale',
|
||||
runId: 'setup-context-local-abc123',
|
||||
watchCommand: `ktx setup --project-dir ${tempDir}`,
|
||||
statusCommand: `ktx status --project-dir ${tempDir}`,
|
||||
detail: 'Previous foreground context build did not finish. Rerun setup or ktx ingest.',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
|
@ -377,6 +377,8 @@ describe('setup status', () => {
|
|||
expect(rendered).toContain(`KTX project: ${tempDir}`);
|
||||
expect(rendered).toContain('Project ready: yes');
|
||||
expect(rendered).toContain('LLM ready: no');
|
||||
expect(rendered).toContain('Databases configured: no');
|
||||
expect(rendered).not.toContain(['Primary sources', 'configured'].join(' '));
|
||||
expect(rendered).toContain('KTX context built: no');
|
||||
expect(rendered).not.toContain('No KTX project found.');
|
||||
});
|
||||
|
|
@ -1141,11 +1143,11 @@ describe('setup status', () => {
|
|||
|
||||
expect(databasePrompts.select).not.toHaveBeenCalled();
|
||||
expect(testIo.stdout()).toContain(
|
||||
'KTX cannot work without at least one primary source. Select a source or press Escape to go back.',
|
||||
'KTX cannot work without at least one database. Select a database or press Escape to go back.',
|
||||
);
|
||||
expect(embeddings).toHaveBeenCalledTimes(2);
|
||||
expect(embeddings).toHaveBeenNthCalledWith(2, expect.objectContaining({ forcePrompt: true }), testIo.io);
|
||||
expect(testIo.stderr()).not.toContain('No primary sources selected.');
|
||||
expect(testIo.stderr()).not.toContain('No databases selected.');
|
||||
});
|
||||
|
||||
it('lets Back from the first setup step return to the entry menu instead of exiting', async () => {
|
||||
|
|
@ -1221,6 +1223,11 @@ describe('setup status', () => {
|
|||
databaseConnectionId: 'warehouse',
|
||||
databaseUrl: 'env:DATABASE_URL',
|
||||
databaseSchemas: ['public'],
|
||||
enableQueryHistory: true,
|
||||
queryHistoryWindowDays: 30,
|
||||
queryHistoryMinExecutions: 12,
|
||||
queryHistoryServiceAccountPatterns: ['^svc_'],
|
||||
queryHistoryRedactionPatterns: ['(?i)secret'],
|
||||
skipDatabases: false,
|
||||
skipSources: true,
|
||||
},
|
||||
|
|
@ -1237,6 +1244,11 @@ describe('setup status', () => {
|
|||
databaseConnectionId: 'warehouse',
|
||||
databaseUrl: 'env:DATABASE_URL',
|
||||
databaseSchemas: ['public'],
|
||||
enableQueryHistory: true,
|
||||
queryHistoryWindowDays: 30,
|
||||
queryHistoryMinExecutions: 12,
|
||||
queryHistoryServiceAccountPatterns: ['^svc_'],
|
||||
queryHistoryRedactionPatterns: ['(?i)secret'],
|
||||
skipDatabases: false,
|
||||
}),
|
||||
testIo.io,
|
||||
|
|
@ -1621,51 +1633,7 @@ describe('setup status', () => {
|
|||
expect(io.stderr()).toContain('KTX context is not ready for agents.');
|
||||
});
|
||||
|
||||
it('does not install agents when full setup context build is detached', async () => {
|
||||
const calls: string[] = [];
|
||||
const io = makeIo();
|
||||
await writeFile(join(tempDir, 'ktx.yaml'), ['project: revenue', 'connections: {}', ''].join('\n'), 'utf-8');
|
||||
|
||||
await expect(
|
||||
runKtxSetup(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: tempDir,
|
||||
mode: 'existing',
|
||||
agents: false,
|
||||
inputMode: 'disabled',
|
||||
yes: true,
|
||||
cliVersion: '0.2.0',
|
||||
skipLlm: true,
|
||||
skipEmbeddings: true,
|
||||
skipDatabases: true,
|
||||
skipSources: true,
|
||||
skipAgents: false,
|
||||
databaseSchemas: [],
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
context: async () => {
|
||||
calls.push('context');
|
||||
return { status: 'detached', projectDir: tempDir, runId: 'setup-context-local-test' };
|
||||
},
|
||||
agents: async () => {
|
||||
calls.push('agents');
|
||||
return {
|
||||
status: 'ready',
|
||||
projectDir: tempDir,
|
||||
installs: [{ target: 'codex', scope: 'project', mode: 'cli' }],
|
||||
};
|
||||
},
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(calls).toEqual(['context']);
|
||||
});
|
||||
|
||||
it('resumes an active context build before prompting for earlier setup steps', async () => {
|
||||
const io = makeIo();
|
||||
it('does not offer background watch choices from setup status', async () => {
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
[
|
||||
|
|
@ -1682,122 +1650,22 @@ describe('setup status', () => {
|
|||
'utf-8',
|
||||
);
|
||||
await writeKtxSetupContextState(tempDir, {
|
||||
runId: 'setup-context-local-active',
|
||||
runId: 'setup-context-local-stale',
|
||||
status: 'running',
|
||||
startedAt: '2026-05-09T10:00:00.000Z',
|
||||
updatedAt: '2026-05-09T10:00:00.000Z',
|
||||
startedAt: '2026-05-09T09:00:00.000Z',
|
||||
updatedAt: '2026-05-09T09:00:00.000Z',
|
||||
primarySourceConnectionIds: ['warehouse'],
|
||||
contextSourceConnectionIds: [],
|
||||
reportIds: [],
|
||||
artifactPaths: [],
|
||||
retryableFailedTargets: [],
|
||||
commands: contextBuildCommands(tempDir, 'setup-context-local-active'),
|
||||
});
|
||||
const context = vi.fn(async () => ({
|
||||
status: 'detached' as const,
|
||||
projectDir: tempDir,
|
||||
runId: 'setup-context-local-active',
|
||||
}));
|
||||
const databases = vi.fn(async () => {
|
||||
throw new Error('database setup should not run while context build is active');
|
||||
commands: contextBuildCommands(tempDir, 'setup-context-local-stale'),
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxSetup(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: tempDir,
|
||||
mode: 'existing',
|
||||
agents: false,
|
||||
inputMode: 'auto',
|
||||
yes: false,
|
||||
cliVersion: '0.2.0',
|
||||
skipLlm: false,
|
||||
skipEmbeddings: false,
|
||||
skipDatabases: false,
|
||||
skipSources: false,
|
||||
skipAgents: false,
|
||||
databaseSchemas: [],
|
||||
},
|
||||
io.io,
|
||||
{ context, databases },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(context).toHaveBeenCalledWith(
|
||||
{ projectDir: tempDir, inputMode: 'auto', allowEmpty: true },
|
||||
io.io,
|
||||
);
|
||||
expect(databases).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('skips entry menu and auto-watches when context build is active and showEntryMenu is true', async () => {
|
||||
const io = makeIo();
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
[
|
||||
'project: revenue',
|
||||
'setup:',
|
||||
' database_connection_ids:',
|
||||
' - warehouse',
|
||||
'connections:',
|
||||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:DATABASE_URL',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
);
|
||||
await writeKtxSetupContextState(tempDir, {
|
||||
runId: 'setup-context-local-active',
|
||||
status: 'detached',
|
||||
startedAt: '2026-05-09T10:00:00.000Z',
|
||||
updatedAt: '2026-05-09T10:00:00.000Z',
|
||||
primarySourceConnectionIds: ['warehouse'],
|
||||
contextSourceConnectionIds: [],
|
||||
reportIds: [],
|
||||
artifactPaths: [],
|
||||
retryableFailedTargets: [],
|
||||
commands: contextBuildCommands(tempDir, 'setup-context-local-active'),
|
||||
});
|
||||
const context = vi.fn(async () => ({
|
||||
status: 'detached' as const,
|
||||
projectDir: tempDir,
|
||||
runId: 'setup-context-local-active',
|
||||
}));
|
||||
const entryMenuSelect = vi.fn(async () => 'exit');
|
||||
|
||||
await expect(
|
||||
runKtxSetup(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: tempDir,
|
||||
mode: 'existing',
|
||||
agents: false,
|
||||
inputMode: 'auto',
|
||||
yes: false,
|
||||
cliVersion: '0.2.0',
|
||||
skipLlm: false,
|
||||
skipEmbeddings: false,
|
||||
skipDatabases: false,
|
||||
skipSources: false,
|
||||
skipAgents: false,
|
||||
databaseSchemas: [],
|
||||
showEntryMenu: true,
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
context,
|
||||
entryMenuDeps: { prompts: { select: entryMenuSelect, cancel: vi.fn() } },
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(entryMenuSelect).not.toHaveBeenCalled();
|
||||
expect(context).toHaveBeenCalledWith(
|
||||
{ projectDir: tempDir, inputMode: 'auto', allowEmpty: true, autoWatch: true },
|
||||
io.io,
|
||||
);
|
||||
const status = await readKtxSetupStatus(tempDir);
|
||||
expect(status.context.status).toBe('stale');
|
||||
const state = await readKtxSetupContextState(tempDir);
|
||||
expect(state.status).toBe('stale');
|
||||
});
|
||||
|
||||
it('routes a ready project menu selection to agent setup', async () => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue