feat(connectors): generalize readiness and constraint handling (#212)

* feat(connectors): add postgres maxConnections

* feat(connectors): add mysql maxConnections

* feat(connectors): add sqlserver maxConnections

* feat(connectors): rename snowflake pool config

* docs: document connector maxConnections

* feat(scan): add constraint discovery warning helper

* feat(scan): carry structural warnings through reports

* feat(postgres): soft-fail denied constraint discovery

* feat(mysql): soft-fail denied constraint discovery

* feat(sqlserver): soft-fail denied constraint discovery

* feat(bigquery): soft-fail denied primary key discovery

* feat(snowflake): report denied primary key discovery

* test(scan): verify constraint discovery warnings

* feat(historic-sql): use shared readiness probes

* docs: document query history readiness probes

* test(historic-sql): verify readiness probe registry

* test(ingest): account for live database warnings artifact

* Add skip option for agent setup

packages/cli/src/context/scan/constraint-discovery.test.ts

@@ -0,0 +1,70 @@
+import { describe, expect, it } from 'vitest';
+import { constraintDiscoveryWarning, tryConstraintQuery } from './constraint-discovery.js';
+
+describe('tryConstraintQuery', () => {
+  it('returns the query value when the query succeeds', async () => {
+    await expect(
+      tryConstraintQuery(
+        {
+          schema: 'public',
+          kind: 'primary_key',
+          isDeniedError: () => false,
+        },
+        async () => ['id'],
+      ),
+    ).resolves.toEqual({ ok: true, value: ['id'] });
+  });
+
+  it('returns a recoverable warning when the classifier recognizes denial', async () => {
+    const error = Object.assign(new Error('permission denied'), { code: '42501' });
+
+    await expect(
+      tryConstraintQuery(
+        {
+          schema: 'analytics',
+          kind: 'foreign_key',
+          isDeniedError: (candidate) => candidate === error,
+        },
+        async () => {
+          throw error;
+        },
+      ),
+    ).resolves.toEqual({
+      ok: false,
+      warning: {
+        code: 'constraint_discovery_unauthorized',
+        message: 'Skipped foreign-key discovery in analytics (insufficient grants on system catalogs)',
+        recoverable: true,
+        metadata: { schema: 'analytics', kind: 'foreign_key' },
+      },
+    });
+  });
+
+  it('rethrows non-denial errors unchanged', async () => {
+    const error = Object.assign(new Error('connection reset'), { code: 'ECONNRESET' });
+
+    await expect(
+      tryConstraintQuery(
+        {
+          schema: 'public',
+          kind: 'primary_key',
+          isDeniedError: () => false,
+        },
+        async () => {
+          throw error;
+        },
+      ),
+    ).rejects.toBe(error);
+  });
+});
+
+describe('constraintDiscoveryWarning', () => {
+  it('formats stable primary-key warning text and metadata', () => {
+    expect(constraintDiscoveryWarning({ schema: 'public', kind: 'primary_key' })).toEqual({
+      code: 'constraint_discovery_unauthorized',
+      message: 'Skipped primary-key discovery in public (insufficient grants on system catalogs)',
+      recoverable: true,
+      metadata: { schema: 'public', kind: 'primary_key' },
+    });
+  });
+});

packages/cli/src/context/scan/constraint-discovery.ts

@@ -0,0 +1,42 @@
+import type { KtxScanWarning } from './types.js';
+
+export type ConstraintDiscoveryKind = 'primary_key' | 'foreign_key';
+
+export interface ConstraintQueryContext {
+  schema: string;
+  kind: ConstraintDiscoveryKind;
+  isDeniedError: (error: unknown) => boolean;
+}
+
+export type ConstraintQueryOutcome<T> = { ok: true; value: T } | { ok: false; warning: KtxScanWarning };
+
+export function constraintDiscoveryWarning(input: {
+  schema: string;
+  kind: ConstraintDiscoveryKind;
+}): KtxScanWarning {
+  return {
+    code: 'constraint_discovery_unauthorized',
+    message:
+      `Skipped ${input.kind === 'primary_key' ? 'primary-key' : 'foreign-key'} ` +
+      `discovery in ${input.schema} (insufficient grants on system catalogs)`,
+    recoverable: true,
+    metadata: { schema: input.schema, kind: input.kind },
+  };
+}
+
+export async function tryConstraintQuery<T>(
+  ctx: ConstraintQueryContext,
+  fn: () => Promise<T>,
+): Promise<ConstraintQueryOutcome<T>> {
+  try {
+    return { ok: true, value: await fn() };
+  } catch (error) {
+    if (!ctx.isDeniedError(error)) {
+      throw error;
+    }
+    return {
+      ok: false,
+      warning: constraintDiscoveryWarning({ schema: ctx.schema, kind: ctx.kind }),
+    };
+  }
+}

packages/cli/src/context/scan/local-scan.test.ts

@@ -180,6 +180,13 @@ function fetchOnlyAdapter(options: { extractedAt?: () => string; snapshot?: KtxS
         'utf-8',
       );
       await writeFile(join(stagedDir, 'foreign-keys.json'), '{"foreignKeys":[]}\n', 'utf-8');
+      if (scanSnapshot.warnings?.length) {
+        await writeFile(
+          join(stagedDir, 'warnings.json'),
+          `${JSON.stringify({ warnings: scanSnapshot.warnings })}\n`,
+          'utf-8',
+        );
+      }
       for (const table of scanSnapshot.tables) {
         await writeFile(join(stagedDir, 'tables', `${table.name}.json`), `${JSON.stringify(table)}\n`, 'utf-8');
       }
@@ -336,6 +343,48 @@ describe('local scan', () => {
     });
   });
 
+  it('threads structural snapshot warnings into the final scan report', async () => {
+    const result = await runLocalScan({
+      project,
+      adapters: [
+        fetchOnlyAdapter({
+          snapshot: {
+            ...defaultFetchSnapshot(),
+            warnings: [
+              {
+                code: 'constraint_discovery_unauthorized',
+                message: 'Skipped primary-key discovery in public (insufficient grants on system catalogs)',
+                recoverable: true,
+                metadata: { schema: 'public', kind: 'primary_key' },
+              },
+            ],
+          },
+        }),
+      ],
+      connectionId: 'warehouse',
+      jobId: 'scan-run-structural-warnings',
+      now: () => new Date('2026-04-29T09:01:00.000Z'),
+    });
+
+    expect(result.report.warnings).toEqual([
+      {
+        code: 'constraint_discovery_unauthorized',
+        message: 'Skipped primary-key discovery in public (insufficient grants on system catalogs)',
+        recoverable: true,
+        metadata: { schema: 'public', kind: 'primary_key' },
+      },
+    ]);
+    await expect(
+      readFile(
+        join(
+          project.projectDir,
+          'raw-sources/warehouse/live-database/2026-04-29-090100-scan-run-structural-warnings/scan-report.json',
+        ),
+        'utf-8',
+      ),
+    ).resolves.toContain('"constraint_discovery_unauthorized"');
+  });
+
   it('passes enabled_tables as fetch context tableScope and does not post-filter staged snapshots', async () => {
     project.config.connections.warehouse = {
       ...project.config.connections.warehouse,

packages/cli/src/context/scan/local-scan.ts

@@ -467,6 +467,9 @@ export async function runLocalScan(options: RunLocalScanOptions): Promise<LocalS
       extractedAtFallback: report.createdAt,
     });
     enrichmentSnapshot = rawSnapshot;
+    if (rawSnapshot.warnings?.length) {
+      report.warnings.push(...rawSnapshot.warnings);
+    }
     const manifestArtifacts = await writeLocalScanManifestShards({
       project: options.project,
       connectionId: options.connectionId,

packages/cli/src/context/scan/local-structural-artifacts.test.ts

@@ -165,6 +165,61 @@ describe('readLocalScanStructuralSnapshot', () => {
     });
   });
 
+  it('rebuilds scan warnings from persisted live-database warning files', async () => {
+    const rawRoot = 'raw-sources/warehouse/live-database/sync-warnings';
+    await project.fileStore.writeFile(
+      `${rawRoot}/connection.json`,
+      '{"connectionId":"warehouse","metadata":{}}\n',
+      'ktx',
+      'ktx@example.com',
+      'Seed connection artifact',
+    );
+    await project.fileStore.writeFile(
+      `${rawRoot}/warnings.json`,
+      `${JSON.stringify(
+        {
+          warnings: [
+            {
+              code: 'constraint_discovery_unauthorized',
+              message: 'Skipped foreign-key discovery in public (insufficient grants on system catalogs)',
+              recoverable: true,
+              metadata: { schema: 'public', kind: 'foreign_key' },
+            },
+          ],
+        },
+        null,
+        2,
+      )}\n`,
+      'ktx',
+      'ktx@example.com',
+      'Seed warning artifact',
+    );
+    await project.fileStore.writeFile(
+      `${rawRoot}/tables/orders.json`,
+      '{"name":"orders","catalog":null,"db":"public","kind":"table","comment":null,"estimatedRows":null,"columns":[{"name":"id","nativeType":"integer","normalizedType":"integer","dimensionType":"number","nullable":false,"primaryKey":false,"comment":null}],"foreignKeys":[]}\n',
+      'ktx',
+      'ktx@example.com',
+      'Seed orders artifact',
+    );
+
+    const snapshot = await readLocalScanStructuralSnapshot({
+      project,
+      connectionId: 'warehouse',
+      driver: 'postgres',
+      rawSourcesDir: rawRoot,
+      extractedAtFallback: '2026-04-29T13:00:00.000Z',
+    });
+
+    expect(snapshot.warnings).toEqual([
+      {
+        code: 'constraint_discovery_unauthorized',
+        message: 'Skipped foreign-key discovery in public (insufficient grants on system catalogs)',
+        recoverable: true,
+        metadata: { schema: 'public', kind: 'foreign_key' },
+      },
+    ]);
+  });
+
   it('uses the scan report timestamp when connection.json omits extractedAt', async () => {
     const rawRoot = 'raw-sources/warehouse/live-database/sync-2';
     await project.fileStore.writeFile(
@@ -192,4 +247,32 @@ describe('readLocalScanStructuralSnapshot', () => {
 
     expect(snapshot.extractedAt).toBe('2026-04-29T13:00:00.000Z');
   });
+
+  it('tolerates older live-database staged directories without warnings.json', async () => {
+    const rawRoot = 'raw-sources/warehouse/live-database/sync-no-warnings';
+    await project.fileStore.writeFile(
+      `${rawRoot}/connection.json`,
+      '{"connectionId":"warehouse","metadata":{}}\n',
+      'ktx',
+      'ktx@example.com',
+      'Seed connection artifact',
+    );
+    await project.fileStore.writeFile(
+      `${rawRoot}/tables/orders.json`,
+      '{"name":"orders","catalog":null,"db":null,"kind":"table","comment":null,"estimatedRows":null,"columns":[{"name":"id","nativeType":"integer","normalizedType":"integer","dimensionType":"number","nullable":false,"primaryKey":true,"comment":null}],"foreignKeys":[]}\n',
+      'ktx',
+      'ktx@example.com',
+      'Seed orders artifact',
+    );
+
+    const snapshot = await readLocalScanStructuralSnapshot({
+      project,
+      connectionId: 'warehouse',
+      driver: 'postgres',
+      rawSourcesDir: rawRoot,
+      extractedAtFallback: '2026-04-29T13:00:00.000Z',
+    });
+
+    expect(snapshot.warnings).toEqual([]);
+  });
 });

packages/cli/src/context/scan/local-structural-artifacts.ts

@@ -1,6 +1,7 @@
 import type { KtxLocalProject } from '../../context/project/project.js';
 import type {
   KtxConnectionDriver,
+  KtxScanWarning,
   KtxSchemaColumn,
   KtxSchemaForeignKey,
   KtxSchemaSnapshot,
@@ -30,6 +31,59 @@ function metadataRecord(value: unknown): Record<string, unknown> {
   return isRecord(value) ? value : {};
 }
 
+const scanWarningCodes = new Set<KtxScanWarning['code']>([
+  'connector_capability_missing',
+  'sampling_failed',
+  'statistics_failed',
+  'llm_unavailable',
+  'embedding_unavailable',
+  'scan_enrichment_backend_not_configured',
+  'relationship_validation_failed',
+  'relationship_llm_invalid_reference',
+  'relationship_llm_proposal_failed',
+  'credential_redacted',
+  'enrichment_failed',
+  'description_fallback_used',
+  'constraint_discovery_unauthorized',
+]);
+
+function parseWarning(rawWarning: unknown, path: string): KtxScanWarning {
+  if (
+    !isRecord(rawWarning) ||
+    typeof rawWarning.code !== 'string' ||
+    !scanWarningCodes.has(rawWarning.code as KtxScanWarning['code']) ||
+    typeof rawWarning.message !== 'string' ||
+    typeof rawWarning.recoverable !== 'boolean'
+  ) {
+    throw new Error(`Invalid KTX schema warning artifact: ${path}`);
+  }
+  return {
+    code: rawWarning.code as KtxScanWarning['code'],
+    message: rawWarning.message,
+    recoverable: rawWarning.recoverable,
+    ...(typeof rawWarning.table === 'string' ? { table: rawWarning.table } : {}),
+    ...(typeof rawWarning.column === 'string' ? { column: rawWarning.column } : {}),
+    ...(isRecord(rawWarning.metadata) ? { metadata: rawWarning.metadata } : {}),
+  };
+}
+
+async function readWarnings(input: ReadLocalScanStructuralSnapshotInput): Promise<KtxScanWarning[]> {
+  const path = `${input.rawSourcesDir}/warnings.json`;
+  try {
+    const warningRaw = await input.project.fileStore.readFile(path);
+    const parsed = JSON.parse(warningRaw.content) as unknown;
+    if (!isRecord(parsed) || !Array.isArray(parsed.warnings)) {
+      throw new Error(`Invalid KTX schema warnings artifact: ${path}`);
+    }
+    return parsed.warnings.map((warning) => parseWarning(warning, path));
+  } catch (error) {
+    if (error instanceof Error && /not found|ENOENT|no such file/i.test(error.message)) {
+      return [];
+    }
+    throw error;
+  }
+}
+
 function optionalStringOrNull(value: unknown): string | null | undefined {
   if (value === undefined) {
     return undefined;
@@ -113,6 +167,7 @@ export async function readLocalScanStructuralSnapshot(
     const tableRaw = await input.project.fileStore.readFile(path);
     tables.push(parseTable(tableRaw.content, path));
   }
+  const warnings = await readWarnings(input);
 
   return {
     connectionId: typeof connection.connectionId === 'string' ? connection.connectionId : input.connectionId,
@@ -121,5 +176,6 @@ export async function readLocalScanStructuralSnapshot(
     scope: isRecord(connection.scope) ? connection.scope : {},
     metadata: metadataRecord(connection.metadata),
     tables,
+    warnings,
   };
 }

packages/cli/src/context/scan/types.ts

@@ -90,6 +90,7 @@ export interface KtxSchemaSnapshot {
   scope: KtxSchemaScope;
   tables: KtxSchemaTable[];
   metadata: Record<string, unknown>;
+  warnings?: KtxScanWarning[];
 }
 
 interface KtxCredentialEnvReference {
@@ -364,7 +365,8 @@ type KtxScanWarningCode =
   | 'relationship_llm_proposal_failed'
   | 'credential_redacted'
   | 'enrichment_failed'
-  | 'description_fallback_used';
+  | 'description_fallback_used'
+  | 'constraint_discovery_unauthorized';
 
 export interface KtxScanWarning {
   code: KtxScanWarningCode;