apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-25 08:48:08 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(connectors): generalize readiness and constraint handling (#212)

Browse source 
* feat(connectors): add postgres maxConnections

* feat(connectors): add mysql maxConnections

* feat(connectors): add sqlserver maxConnections

* feat(connectors): rename snowflake pool config

* docs: document connector maxConnections

* feat(scan): add constraint discovery warning helper

* feat(scan): carry structural warnings through reports

* feat(postgres): soft-fail denied constraint discovery

* feat(mysql): soft-fail denied constraint discovery

* feat(sqlserver): soft-fail denied constraint discovery

* feat(bigquery): soft-fail denied primary key discovery

* feat(snowflake): report denied primary key discovery

* test(scan): verify constraint discovery warnings

* feat(historic-sql): use shared readiness probes

* docs: document query history readiness probes

* test(historic-sql): verify readiness probe registry

* test(ingest): account for live database warnings artifact

* Add skip option for agent setup
This commit is contained in:
Andrey Avtomonov 2026-05-24 19:30:06 +02:00 • committed by GitHub
parent cfd1749ab9
commit 78b8a0c025
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
42 changed files with 2763 additions and 554 deletions
Download patch file Download diff file Expand all files Collapse all files

83
packages/cli/src/context/scan/local-structural-artifacts.test.ts
View file

@ -165,6 +165,61 @@ describe('readLocalScanStructuralSnapshot', () => {
});
});
it('rebuilds scan warnings from persisted live-database warning files', async () => {
const rawRoot = 'raw-sources/warehouse/live-database/sync-warnings';
await project.fileStore.writeFile(
`${rawRoot}/connection.json`,
'{"connectionId":"warehouse","metadata":{}}\n',
'ktx',
'ktx@example.com',
'Seed connection artifact',
);
await project.fileStore.writeFile(
`${rawRoot}/warnings.json`,
`${JSON.stringify(
{
warnings: [
{
code: 'constraint_discovery_unauthorized',
message: 'Skipped foreign-key discovery in public (insufficient grants on system catalogs)',
recoverable: true,
metadata: { schema: 'public', kind: 'foreign_key' },
},
],
},
null,
2,
)}\n`,
'ktx',
'ktx@example.com',
'Seed warning artifact',
);
await project.fileStore.writeFile(
`${rawRoot}/tables/orders.json`,
'{"name":"orders","catalog":null,"db":"public","kind":"table","comment":null,"estimatedRows":null,"columns":[{"name":"id","nativeType":"integer","normalizedType":"integer","dimensionType":"number","nullable":false,"primaryKey":false,"comment":null}],"foreignKeys":[]}\n',
'ktx',
'ktx@example.com',
'Seed orders artifact',
);
const snapshot = await readLocalScanStructuralSnapshot({
project,
connectionId: 'warehouse',
driver: 'postgres',
rawSourcesDir: rawRoot,
extractedAtFallback: '2026-04-29T13:00:00.000Z',
});
expect(snapshot.warnings).toEqual([
{
code: 'constraint_discovery_unauthorized',
message: 'Skipped foreign-key discovery in public (insufficient grants on system catalogs)',
recoverable: true,
metadata: { schema: 'public', kind: 'foreign_key' },
},
]);
});
it('uses the scan report timestamp when connection.json omits extractedAt', async () => {
const rawRoot = 'raw-sources/warehouse/live-database/sync-2';
await project.fileStore.writeFile(
@ -192,4 +247,32 @@ describe('readLocalScanStructuralSnapshot', () => {
expect(snapshot.extractedAt).toBe('2026-04-29T13:00:00.000Z');
});
it('tolerates older live-database staged directories without warnings.json', async () => {
const rawRoot = 'raw-sources/warehouse/live-database/sync-no-warnings';
await project.fileStore.writeFile(
`${rawRoot}/connection.json`,
'{"connectionId":"warehouse","metadata":{}}\n',
'ktx',
'ktx@example.com',
'Seed connection artifact',
);
await project.fileStore.writeFile(
`${rawRoot}/tables/orders.json`,
'{"name":"orders","catalog":null,"db":null,"kind":"table","comment":null,"estimatedRows":null,"columns":[{"name":"id","nativeType":"integer","normalizedType":"integer","dimensionType":"number","nullable":false,"primaryKey":true,"comment":null}],"foreignKeys":[]}\n',
'ktx',
'ktx@example.com',
'Seed orders artifact',
);
const snapshot = await readLocalScanStructuralSnapshot({
project,
connectionId: 'warehouse',
driver: 'postgres',
rawSourcesDir: rawRoot,
extractedAtFallback: '2026-04-29T13:00:00.000Z',
});
expect(snapshot.warnings).toEqual([]);
});
});