apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-10 08:05:14 +02:00
Code Issues Projects Releases Packages Wiki Activity

docs: document public ingest command

Browse source
This commit is contained in:
Andrey Avtomonov 2026-05-13 18:09:57 +02:00
parent 9afc5c87c3
commit 220fb5f8ea
7 changed files with 75 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

10
scripts/examples-docs.test.mjs
View file

@ -223,18 +223,18 @@ describe('standalone example docs', () => {
assert.doesNotMatch(readme, /python -m ktx_daemon semantic-validate/);
});
it('documents scan workflows in the docs site', async () => {
it('documents public context build workflows in the docs site', async () => {
const rootReadme = await readText('README.md');
const buildingContext = await readText('docs-site/content/docs/guides/building-context.mdx');
const scanReference = await readText('docs-site/content/docs/cli-reference/ktx-scan.mdx');
assert.match(buildingContext, /ktx scan <connection-id>/);
assert.match(buildingContext, /ktx ingest <connection-id>/);
assert.match(buildingContext, /ktx ingest --all/);
assert.match(buildingContext, /ktx status/);
assert.doesNotMatch(buildingContext, /ktx scan status <run-id>/);
assert.doesNotMatch(buildingContext, /ktx scan report <run-id>/);
assert.match(scanReference, /ktx scan <connectionId> \[options\]/);
assert.match(rootReadme, /raw-sources\//);
assert.match(rootReadme, /live-database\//);
assert.doesNotMatch(rootReadme, /live-database\//);
assert.doesNotMatch(rootReadme, /ktx scan/);
assert.doesNotMatch(rootReadme, /Run a local ingest smoke test/);
assert.doesNotMatch(rootReadme, /ktx ingest run --project-dir/);
assert.doesNotMatch(rootReadme, /ktx ingest status --project-dir/);

24
scripts/installed-live-database-smoke.mjs
View file

@ -96,27 +96,20 @@ export function buildKtxYaml(postgresUrl) {
'storage:',
' state: sqlite',
' search: sqlite-fts5',
'ingest:',
' adapters:',
' - live-database',
'',
].join('\n');
}
export function buildLiveDatabaseIngestArgs(projectDir, databaseIntrospectionUrl) {
export function buildLiveDatabaseIngestArgs(projectDir, _databaseIntrospectionUrl, connectionId = 'warehouse') {
return [
'exec',
'ktx',
'ingest',
'run',
connectionId,
'--project-dir',
projectDir,
'--connection-id',
'warehouse',
'--adapter',
'live-database',
'--database-introspection-url',
databaseIntrospectionUrl,
'--fast',
'--no-input',
];
}
@ -324,12 +317,9 @@ async function main() {
env: managedRuntimeEnv(cleanInstallDir),
timeout: 120_000,
});
requireSuccess('ktx ingest run live-database', ingestRun);
requireOutput('ktx ingest run live-database', ingestRun, /Status: done/);
requireOutput('ktx ingest run live-database', ingestRun, /Adapter: live-database/);
requireOutput('ktx ingest run live-database', ingestRun, /Diff: \+4\/~0\/-0\/=0/);
requireOutput('ktx ingest run live-database', ingestRun, /Raw files: 4/);
requireOutput('ktx ingest run live-database', ingestRun, /Work units: 2/);
requireSuccess('ktx ingest warehouse --fast', ingestRun);
requireOutput('ktx ingest warehouse --fast', ingestRun, /Ingest finished/);
requireOutput('ktx ingest warehouse --fast', ingestRun, /Database schema/);
const runId = getRunId(ingestRun.stdout);
const ingestStatus = await run('pnpm', buildLiveDatabaseStatusArgs(projectDir, runId), {

17
scripts/installed-live-database-smoke.test.mjs
View file

@ -50,7 +50,7 @@ describe('installed live-database artifact smoke helpers', () => {
);
});
it('writes a live-database-only KTX project config with SQLite local state', () => {
it('writes a public database ingest KTX project config with SQLite local state', () => {
assert.equal(
buildKtxYaml('postgresql://ktx:postgres@127.0.0.1:15432/warehouse'), // pragma: allowlist secret
[
@ -63,9 +63,6 @@ describe('installed live-database artifact smoke helpers', () => {
'storage:',
' state: sqlite',
' search: sqlite-fts5',
'ingest:',
' adapters:',
' - live-database',
'',
].join('\n'),
);
@ -98,20 +95,16 @@ describe('installed live-database artifact smoke helpers', () => {
]);
});
it('builds installed CLI live-database ingest and status commands', () => {
it('builds installed CLI public database ingest and status commands', () => {
assert.deepEqual(buildLiveDatabaseIngestArgs('/tmp/project', 'http://127.0.0.1:8765'), [
'exec',
'ktx',
'ingest',
'run',
'warehouse',
'--project-dir',
'/tmp/project',
'--connection-id',
'warehouse',
'--adapter',
'live-database',
'--database-introspection-url',
'http://127.0.0.1:8765',
'--fast',
'--no-input',
]);
assert.deepEqual(buildLiveDatabaseStatusArgs('/tmp/project', 'local-run-1'), [

34
scripts/package-artifacts.mjs
View file

@ -653,10 +653,6 @@ try {
'scan:',
' enrichment:',
' mode: deterministic',
'ingest:',
' adapters:',
' - fake',
' - live-database',
'',
].join('\\n'),
'utf-8',
@ -819,30 +815,32 @@ try {
requireOutput('ktx dev runtime stop', runtimeStop, /Stopped KTX Python daemon/);
process.stdout.write('ktx dev runtime daemon lifecycle verified\\n');
const structuralScan = await run('pnpm', ['exec', 'ktx', 'scan', 'warehouse',
const structuralScan = await run('pnpm', ['exec', 'ktx', 'ingest', 'warehouse',
'--project-dir',
projectDir,
'--fast',
'--no-input',
]);
requireProjectStderr('ktx scan structural', structuralScan, projectDir);
requireOutput('ktx scan structural', structuralScan, /Status: done/);
requireOutput('ktx scan structural', structuralScan, /Mode: structural/);
requireOutput('ktx scan structural', structuralScan, /Needs attention\\s+None/);
requireProjectStderr('ktx ingest fast', structuralScan, projectDir);
requireOutput('ktx ingest fast', structuralScan, /Ingest finished/);
requireOutput('ktx ingest fast', structuralScan, /Database schema/);
requireOutput('ktx ingest fast', structuralScan, /warehouse\\s+done/);
const structuralScanRunId = getRunId(structuralScan.stdout);
await access(join(projectDir, 'semantic-layer', 'warehouse', '_schema', 'public.yaml'));
process.stdout.write('ktx scan structural verified: ' + structuralScanRunId + '\\n');
process.stdout.write('ktx ingest fast verified: ' + structuralScanRunId + '\\n');
const enrichedScan = await run('pnpm', ['exec', 'ktx', 'scan', 'warehouse',
const enrichedScan = await run('pnpm', ['exec', 'ktx', 'ingest', 'warehouse',
'--project-dir',
projectDir,
'--mode',
'enriched',
'--deep',
'--no-input',
]);
requireProjectStderr('ktx scan enriched', enrichedScan, projectDir);
requireOutput('ktx scan enriched', enrichedScan, /Status: done/);
requireOutput('ktx scan enriched', enrichedScan, /Mode: enriched/);
requireOutput('ktx scan enriched', enrichedScan, /Enrichment artifacts:/);
requireProjectStderr('ktx ingest deep', enrichedScan, projectDir);
requireOutput('ktx ingest deep', enrichedScan, /Ingest finished/);
requireOutput('ktx ingest deep', enrichedScan, /Database schema/);
requireOutput('ktx ingest deep', enrichedScan, /warehouse\\s+done/);
const enrichedScanRunId = getRunId(enrichedScan.stdout);
process.stdout.write('ktx scan enriched verified: ' + enrichedScanRunId + '\\n');
process.stdout.write('ktx ingest deep verified: ' + enrichedScanRunId + '\\n');
await mkdir(join(sourceDir, 'orders'), { recursive: true });
await writeFile(join(sourceDir, 'orders', 'orders.json'), '{"name":"orders"}\\n', 'utf-8');

10
scripts/package-artifacts.test.mjs
View file

@ -464,7 +464,7 @@ describe('verification snippets', () => {
assert.match(source, /node:sqlite/);
assert.match(source, /driver: sqlite/);
assert.match(source, /path: warehouse\.db/);
assert.match(source, /live-database/);
assert.doesNotMatch(source, /live-database/);
assert.match(source, /'--execute'/);
assert.match(source, /"mode": "compile_only"/);
assert.match(source, /"mode": "executed"/);
@ -488,11 +488,11 @@ describe('verification snippets', () => {
assert.match(source, /ktx dev runtime stop/);
assert.doesNotMatch(source, /ktx dev runtime prune/);
assert.doesNotMatch(source, /staleRuntimeDir/);
assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'scan',\s*'warehouse'/);
assert.match(source, /'--mode',\s*'enriched'/);
assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'ingest',\s*'warehouse'/);
assert.match(source, /'--deep'/);
assert.doesNotMatch(source, /'--enrich'/);
assert.match(source, /ktx scan structural verified/);
assert.match(source, /ktx scan enriched verified/);
assert.match(source, /ktx ingest fast verified/);
assert.match(source, /ktx ingest deep verified/);
assert.match(source, /enrichment:/);
assert.match(source, /mode: deterministic/);
assert.match(source, /run\('pnpm', \['exec', 'ktx', 'ingest', 'run'/);