diff --git a/packages/context/src/ingest/adapters/live-database/manifest.test.ts b/packages/context/src/ingest/adapters/live-database/manifest.test.ts index 75a41067..a97140a9 100644 --- a/packages/context/src/ingest/adapters/live-database/manifest.test.ts +++ b/packages/context/src/ingest/adapters/live-database/manifest.test.ts @@ -186,6 +186,62 @@ describe('buildLiveDatabaseManifestShards', () => { }); }); + it('preserves external usage keys while replacing historic SQL managed keys', () => { + const existingUsage = new Map([ + [ + 'orders', + { + narrative: 'Old generated usage narrative.', + frequencyTier: 'low' as const, + commonFilters: ['old_status'], + commonJoins: [], + ownerNote: 'Pinned analyst note', + }, + ], + ]); + + const result = buildLiveDatabaseManifestShards({ + connectionType: 'POSTGRESQL', + mapColumnType: (nativeType) => nativeType.toLowerCase(), + existingUsage, + tables: [ + { + name: 'orders', + catalog: null, + db: 'public', + usage: { + narrative: 'Fresh generated usage narrative.', + frequencyTier: 'high', + commonFilters: ['status'], + commonGroupBys: ['created_at'], + commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], + }, + columns: [{ name: 'id', type: 'INTEGER' }], + }, + ], + joins: [], + }); + + expect(shardObject(result.shards)).toEqual({ + public: { + tables: { + orders: { + table: 'public.orders', + usage: { + ownerNote: 'Pinned analyst note', + narrative: 'Fresh generated usage narrative.', + frequencyTier: 'high', + commonFilters: ['status'], + commonGroupBys: ['created_at'], + commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], + }, + columns: [{ name: 'id', type: 'integer' }], + }, + }, + }, + }); + }); + it('renders ordered multi-column joins in both directions', () => { const result = buildLiveDatabaseManifestShards({ connectionType: 'POSTGRESQL', diff --git a/packages/context/src/ingest/adapters/live-database/manifest.ts b/packages/context/src/ingest/adapters/live-database/manifest.ts index d7315f9e..c6a6e2d5 100644 --- a/packages/context/src/ingest/adapters/live-database/manifest.ts +++ b/packages/context/src/ingest/adapters/live-database/manifest.ts @@ -1,3 +1,5 @@ +import type { TableUsageOutput } from '../historic-sql/skill-schemas.js'; + const RELATIONSHIP_MAP: Record = { MANY_TO_ONE: 'many_to_one', ONE_TO_MANY: 'one_to_many', @@ -11,6 +13,14 @@ const RELATIONSHIP_INVERSE: Record = { }; const SCAN_MANAGED_DESCRIPTION_KEYS = new Set(['db', 'ai']); +const HISTORIC_SQL_MANAGED_USAGE_KEYS = new Set([ + 'narrative', + 'frequencyTier', + 'commonFilters', + 'commonGroupBys', + 'commonJoins', + 'staleSince', +]); export interface LiveDatabaseManifestColumn { name: string; @@ -30,6 +40,7 @@ export interface LiveDatabaseManifestJoinEntry { export interface LiveDatabaseManifestTableEntry { table: string; descriptions?: Record; + usage?: TableUsageOutput; columns: LiveDatabaseManifestColumn[]; joins?: LiveDatabaseManifestJoinEntry[]; } @@ -43,6 +54,7 @@ export interface LiveDatabaseManifestTableData { catalog: string | null; db: string | null; descriptions?: Record; + usage?: TableUsageOutput; columns: Array<{ name: string; type: string; @@ -73,6 +85,7 @@ export interface BuildLiveDatabaseManifestShardsInput { mapColumnType: (nativeType: string) => string; existingPreservedJoins?: Map; existingDescriptions?: Map; + existingUsage?: Map; } export interface BuildLiveDatabaseManifestShardsResult { @@ -101,6 +114,28 @@ function mergeDescriptionsPreservingExternal( return Object.keys(result).length > 0 ? result : undefined; } +export function mergeUsagePreservingExternal( + existing: TableUsageOutput | undefined, + incoming: TableUsageOutput | undefined, +): TableUsageOutput | undefined { + if (!existing && !incoming) { + return undefined; + } + if (!incoming) { + return existing ? { ...existing } : undefined; + } + const result: Record = {}; + if (existing) { + for (const [key, value] of Object.entries(existing)) { + if (!HISTORIC_SQL_MANAGED_USAGE_KEYS.has(key)) { + result[key] = value; + } + } + } + Object.assign(result, incoming); + return Object.keys(result).length > 0 ? (result as TableUsageOutput) : undefined; +} + function getShardKey(connectionType: string, catalog: string | null, db: string | null): string { const normalized = connectionType.toUpperCase(); @@ -254,6 +289,11 @@ export function buildLiveDatabaseManifestShards( entry.descriptions = tableDescriptions; } + const usage = mergeUsagePreservingExternal(input.existingUsage?.get(table.name), table.usage); + if (usage) { + entry.usage = usage; + } + const tableJoins = joinsByTable.get(table.name); if (tableJoins && tableJoins.length > 0) { entry.joins = tableJoins; diff --git a/packages/context/src/scan/local-enrichment-artifacts.test.ts b/packages/context/src/scan/local-enrichment-artifacts.test.ts index d34da036..0123f086 100644 --- a/packages/context/src/scan/local-enrichment-artifacts.test.ts +++ b/packages/context/src/scan/local-enrichment-artifacts.test.ts @@ -742,6 +742,13 @@ describe('writeLocalScanEnrichmentArtifacts', () => { orders: { table: 'public.orders', descriptions: { user: 'Pinned structural description', ai: 'Old generated text' }, + usage: { + narrative: 'Orders are commonly filtered by lifecycle status.', + frequencyTier: 'high', + commonFilters: ['status'], + commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], + ownerNote: 'Preserve analyst note', + }, columns: [ { name: 'id', @@ -797,6 +804,7 @@ describe('writeLocalScanEnrichmentArtifacts', () => { tables: { orders: { descriptions: Record; + usage?: Record; columns: Array<{ name: string; descriptions?: Record }>; joins: Array<{ to: string; on: string; source: string }>; }; @@ -807,6 +815,13 @@ describe('writeLocalScanEnrichmentArtifacts', () => { user: 'Pinned structural description', db: 'DB orders table', }); + expect(manifest.tables.orders.usage).toEqual({ + narrative: 'Orders are commonly filtered by lifecycle status.', + frequencyTier: 'high', + commonFilters: ['status'], + commonJoins: [{ table: 'public.customers', on: ['customer_id'] }], + ownerNote: 'Preserve analyst note', + }); expect(manifest.tables.orders.columns.find((column) => column.name === 'id')?.descriptions).toEqual({ user: 'Pinned structural id', db: 'DB order id', diff --git a/packages/context/src/scan/local-enrichment-artifacts.ts b/packages/context/src/scan/local-enrichment-artifacts.ts index b8186b0e..101d062e 100644 --- a/packages/context/src/scan/local-enrichment-artifacts.ts +++ b/packages/context/src/scan/local-enrichment-artifacts.ts @@ -6,6 +6,7 @@ import { type LiveDatabaseManifestJoinEntry, type LiveDatabaseManifestShard, type LiveDatabaseManifestTableData, + type TableUsageOutput, } from '../ingest/index.js'; import type { KtxScanRelationshipConfig } from '../project/config.js'; import type { KtxLocalProject } from '../project/index.js'; @@ -56,6 +57,7 @@ export interface WriteLocalScanEnrichmentArtifactsResult extends WriteLocalScanM interface ExistingManifestState { descriptions: Map; preservedJoins: Map; + usage: Map; } type LocalDescriptionUpdates = KtxLocalScanEnrichmentResult['descriptionUpdates']; @@ -196,6 +198,7 @@ async function loadExistingManifestState( ): Promise { const descriptions = new Map(); const preservedJoins = new Map(); + const usage = new Map(); const validTableNames = new Set(snapshot.tables.map((table) => table.name)); const columnsByTable = validColumns(snapshot); @@ -203,7 +206,7 @@ async function loadExistingManifestState( try { files = (await project.fileStore.listFiles(schemaDir(connectionId))).files.filter((file) => file.endsWith('.yaml')); } catch { - return { descriptions, preservedJoins }; + return { descriptions, preservedJoins, usage }; } for (const file of files) { @@ -225,6 +228,9 @@ async function loadExistingManifestState( ), ), }); + if (entry.usage) { + usage.set(tableName, { ...entry.usage }); + } const joins = (entry.joins ?? []).filter((join) => { return ( (join.source === 'manual' || join.source === 'inferred') && @@ -241,7 +247,7 @@ async function loadExistingManifestState( } } - return { descriptions, preservedJoins }; + return { descriptions, preservedJoins, usage }; } async function writeJsonArtifact( @@ -276,6 +282,7 @@ export async function writeLocalScanManifestShards( joins: relationshipJoins(input.snapshot, input.relationshipUpdate), existingDescriptions: existing.descriptions, existingPreservedJoins: existing.preservedJoins, + existingUsage: existing.usage, mapColumnType: (dimensionType) => dimensionType, });