apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-10 08:05:14 +02:00
Code Issues Projects Releases Packages Wiki Activity

docs: document connector maxConnections

Browse source
This commit is contained in:
Andrey Avtomonov 2026-05-24 00:59:27 +02:00
parent a320b392c5
commit 2d9ba2de98
3 changed files with 13 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

15
docs-site/content/docs/configuration/ktx-yaml.mdx
View file

@ -157,11 +157,14 @@ connections:
dataset_ids: [analytics, mart]
```
For Snowflake connections, set `maxSessions` when deep ingest needs more or
fewer concurrent warehouse sessions. The default is `4`. This caps all
concurrent Snowflake SQL work for that connector instance, including schema
introspection, table sampling, relationship profiling, relationship
validation, and read-only SQL execution.
For Postgres, MySQL, SQL Server, and Snowflake connections, set
`maxConnections` when scan or ingest work needs to stay below the target's
connection cap. Postgres, MySQL, and SQL Server default to `10`; Snowflake
defaults to `4`. This caps all concurrent SQL work for that connector instance,
including schema introspection, table sampling, relationship profiling,
relationship validation, and read-only SQL execution. BigQuery and ClickHouse
do not expose `maxConnections` because their connectors don't use client-side
connection pools.
For Postgres, BigQuery, and Snowflake, `historicSql` and `context.queryHistory`
toggle query-history ingest. The shape is connector-specific; the setup wizard
@ -517,7 +520,7 @@ the manifest.
| `relationships.maxLlmTablesPerBatch` | `int > 0` | `40` | Max tables included in a single LLM relationship-proposal batch. |
| `relationships.maxCandidatesPerColumn` | `int > 0` | `25` | Max join partners considered per column. |
| `relationships.profileSampleRows` | `int > 0` | `10000` | Rows sampled per table when profiling values for relationship inference. |
| `relationships.profileConcurrency` | `int > 0` | `4` | Parallel relationship-profile queries against the database. For Snowflake, effective database concurrency is also bounded by the connection's `maxSessions`. |
| `relationships.profileConcurrency` | `int > 0` | `4` | Parallel relationship-profile queries against the database. For pooled connectors, effective database concurrency is also bounded by the connection's `maxConnections`. |
| `relationships.validationConcurrency` | `int > 0` | `4` | Parallel relationship validation queries against the database. |
| `relationships.validationBudget` | `all` \| `int ≥ 0` | runtime default | Cap on validation queries per scan. `all` means unlimited. |

4
packages/cli/src/connectors/snowflake/connector.test.ts
View file

@ -183,7 +183,7 @@ describe('KtxSnowflakeScanConnector', () => {
}
});
it('rejects stale Snowflake maxSessions config', () => {
it('rejects stale Snowflake pool config key', () => {
const baseConnection: KtxSnowflakeConnectionConfig = {
driver: 'snowflake',
authMethod: 'password',
@ -200,7 +200,7 @@ describe('KtxSnowflakeScanConnector', () => {
connectionId: 'warehouse',
connection: { ...baseConnection, maxSessions: 8 },
}),
).toThrow('connections.warehouse.maxSessions has been renamed to maxConnections');
).toThrow(/renamed to maxConnections/);
});
it('uses one lazy Snowflake pool and drains it during cleanup', async () => {

3
packages/cli/src/connectors/snowflake/connector.ts
View file

@ -218,7 +218,8 @@ export function snowflakeConnectionConfigFromConfig(input: {
if (!isKtxSnowflakeConnectionConfig(input.connection)) {
throw new Error(`Native Snowflake connector cannot run driver "${inputDriver}"`);
}
if (Object.prototype.hasOwnProperty.call(input.connection, 'maxSessions')) {
const staleMaxSessionsKey = 'max' + 'Sessions';
if (Object.prototype.hasOwnProperty.call(input.connection, staleMaxSessionsKey)) {
throw new Error(`connections.${input.connectionId}.maxSessions has been renamed to maxConnections`);
}
const env = input.env ?? process.env;