diff --git a/packages/cli/src/telemetry/events.snapshot.test.ts b/packages/cli/src/telemetry/events.snapshot.test.ts index ca0fd483..1df95aa0 100644 --- a/packages/cli/src/telemetry/events.snapshot.test.ts +++ b/packages/cli/src/telemetry/events.snapshot.test.ts @@ -47,6 +47,89 @@ describe('telemetry privacy snapshot', () => { hasProject: false, projectGroupAttached: false, }), + buildTelemetryEvent('setup_step', envelope, { + step: 'databases', + outcome: 'completed', + durationMs: 42, + }), + buildTelemetryEvent('connection_added', envelope, { + driver: 'postgres', + isDemoConnection: false, + }), + buildTelemetryEvent('connection_test', envelope, { + driver: 'postgres', + isDemoConnection: false, + outcome: 'error', + errorClass: 'KtxConnectionTestAbortError', + durationMs: 34, + serverVersion: '16', + }), + buildTelemetryEvent('project_stack_snapshot', envelope, { + connectors: [ + { driver: 'sqlite', isDemo: true }, + { driver: 'postgres', isDemo: false }, + ], + connectionCount: 2, + hasSl: true, + hasWiki: true, + hasMcp: true, + hasManagedRuntime: true, + }), + buildTelemetryEvent('ingest_completed', envelope, { + driver: 'postgres', + isDemoConnection: false, + schemaCount: 2, + tableCount: 4, + columnCount: 20, + rowsBucket: '<100k', + durationMs: 100, + outcome: 'ok', + }), + buildTelemetryEvent('scan_completed', envelope, { + driver: 'postgres', + tableCount: 4, + columnCount: 20, + inferredFkCount: 2, + declaredFkCount: 1, + durationMs: 70, + outcome: 'ok', + }), + buildTelemetryEvent('sl_validate_completed', envelope, { + sourceCount: 1, + modelCount: 3, + validationErrorCount: 0, + outcome: 'ok', + durationMs: 15, + }), + buildTelemetryEvent('sl_query_completed', envelope, { + mode: 'compile', + referencedSourceCount: 1, + referencedDimensionCount: 2, + referencedMeasureCount: 1, + durationMs: 18, + outcome: 'ok', + }), + buildTelemetryEvent('sql_completed', envelope, { + driver: 'postgres', + isDemoConnection: false, + queryVerb: 'select', + referencedTableCount: 3, + durationMs: 20, + outcome: 'ok', + }), + buildTelemetryEvent('wiki_query_completed', envelope, { + queryLength: 'select private_table from /Users/alice'.length, + resultCount: 2, + durationMs: 8, + outcome: 'ok', + }), + buildTelemetryEvent('mcp_request_completed', envelope, { + toolName: 'sl_query', + outcome: 'error', + errorClass: 'KtxProjectMissingAbortError', + durationMs: 12, + sampleRate: 0.1, + }), ]; const payload = JSON.stringify(events); diff --git a/packages/cli/src/telemetry/events.test.ts b/packages/cli/src/telemetry/events.test.ts index 8735390b..2b61d084 100644 --- a/packages/cli/src/telemetry/events.test.ts +++ b/packages/cli/src/telemetry/events.test.ts @@ -18,8 +18,22 @@ const envelope: TelemetryCommonEnvelope = { }; describe('telemetry event schemas', () => { - it('catalogs only phase 1 events', () => { - expect(telemetryEventCatalog.map((event) => event.name)).toEqual(['install_first_run', 'command']); + it('catalogs all Node v1 telemetry events', () => { + expect(telemetryEventCatalog.map((event) => event.name)).toEqual([ + 'install_first_run', + 'command', + 'setup_step', + 'connection_added', + 'connection_test', + 'project_stack_snapshot', + 'ingest_completed', + 'scan_completed', + 'sl_validate_completed', + 'sl_query_completed', + 'sql_completed', + 'wiki_query_completed', + 'mcp_request_completed', + ]); }); it('builds a strict install_first_run event', () => { @@ -68,9 +82,43 @@ describe('telemetry event schemas', () => { ).toThrow(); }); - it('rejects raw string fields that are not in the phase 1 schema', () => { + it('builds strict Phase 2 events without private names or text', () => { + expect( + buildTelemetryEvent('connection_test', envelope, { + driver: 'postgres', + isDemoConnection: false, + outcome: 'ok', + durationMs: 34, + serverVersion: '16', + }), + ).toMatchObject({ + name: 'connection_test', + properties: { + driver: 'postgres', + isDemoConnection: false, + outcome: 'ok', + durationMs: 34, + serverVersion: '16', + }, + }); + + expect(() => + telemetryEventSchemas.sql_completed.parse({ + ...envelope, + driver: 'postgres', + isDemoConnection: false, + queryVerb: 'select', + referencedTableCount: 1, + durationMs: 10, + outcome: 'ok', + sql: 'select * from private_table', + }), + ).toThrow(); + }); + + it('rejects raw private field names that are not in the telemetry schemas', () => { expect(JSON.stringify(telemetryEventSchemas)).not.toContain('tableName'); - expect(JSON.stringify(telemetryEventSchemas)).not.toContain('sql'); + expect(Object.keys(telemetryEventSchemas.sql_completed.shape)).not.toContain('sql'); expect(JSON.stringify(telemetryEventSchemas)).not.toContain('path'); }); }); diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts index b72fdc26..534a07a6 100644 --- a/packages/cli/src/telemetry/events.ts +++ b/packages/cli/src/telemetry/events.ts @@ -27,10 +27,155 @@ const commandSchema = telemetryCommonEnvelopeSchema }) .strict(); +const outcomeSchema = z.enum(['ok', 'error']); + +const setupStepSchema = telemetryCommonEnvelopeSchema + .extend({ + step: z.enum([ + 'project', + 'runtime', + 'models', + 'embeddings', + 'secrets', + 'databases', + 'database-context-depth', + 'sources', + 'context', + 'agents', + 'demo-tour', + ]), + outcome: z.enum(['completed', 'skipped', 'abandoned']), + durationMs: z.number().nonnegative(), + }) + .strict(); + +const connectionAddedSchema = telemetryCommonEnvelopeSchema + .extend({ + driver: z.string(), + isDemoConnection: z.boolean(), + }) + .strict(); + +const connectionTestSchema = telemetryCommonEnvelopeSchema + .extend({ + driver: z.string(), + isDemoConnection: z.boolean(), + outcome: outcomeSchema, + errorClass: z.string().optional(), + durationMs: z.number().nonnegative(), + serverVersion: z.string().optional(), + }) + .strict(); + +const projectStackSnapshotSchema = telemetryCommonEnvelopeSchema + .extend({ + connectors: z.array(z.object({ driver: z.string(), isDemo: z.boolean() }).strict()), + connectionCount: z.number().int().nonnegative(), + hasSl: z.boolean(), + hasWiki: z.boolean(), + hasMcp: z.boolean(), + hasManagedRuntime: z.boolean(), + }) + .strict(); + +const rowsBucketSchema = z.enum(['<10k', '<100k', '<1M', '<10M', '>=10M']); + +const ingestCompletedSchema = telemetryCommonEnvelopeSchema + .extend({ + driver: z.string(), + isDemoConnection: z.boolean(), + schemaCount: z.number().int().nonnegative(), + tableCount: z.number().int().nonnegative(), + columnCount: z.number().int().nonnegative(), + rowsBucket: rowsBucketSchema, + durationMs: z.number().nonnegative(), + outcome: outcomeSchema, + errorClass: z.string().optional(), + }) + .strict(); + +const scanCompletedSchema = telemetryCommonEnvelopeSchema + .extend({ + driver: z.string(), + tableCount: z.number().int().nonnegative(), + columnCount: z.number().int().nonnegative(), + inferredFkCount: z.number().int().nonnegative(), + declaredFkCount: z.number().int().nonnegative(), + durationMs: z.number().nonnegative(), + outcome: outcomeSchema, + errorClass: z.string().optional(), + }) + .strict(); + +const slValidateCompletedSchema = telemetryCommonEnvelopeSchema + .extend({ + sourceCount: z.number().int().nonnegative(), + modelCount: z.number().int().nonnegative(), + validationErrorCount: z.number().int().nonnegative(), + outcome: outcomeSchema, + errorClass: z.string().optional(), + durationMs: z.number().nonnegative(), + }) + .strict(); + +const slQueryCompletedSchema = telemetryCommonEnvelopeSchema + .extend({ + mode: z.enum(['compile', 'execute']), + referencedSourceCount: z.number().int().nonnegative(), + referencedDimensionCount: z.number().int().nonnegative(), + referencedMeasureCount: z.number().int().nonnegative(), + durationMs: z.number().nonnegative(), + outcome: outcomeSchema, + errorClass: z.string().optional(), + }) + .strict(); + +const sqlCompletedSchema = telemetryCommonEnvelopeSchema + .extend({ + driver: z.string(), + isDemoConnection: z.boolean(), + queryVerb: z.enum(['select', 'explain', 'show', 'with', 'other']), + referencedTableCount: z.number().int().nonnegative(), + durationMs: z.number().nonnegative(), + outcome: outcomeSchema, + errorClass: z.string().optional(), + }) + .strict(); + +const wikiQueryCompletedSchema = telemetryCommonEnvelopeSchema + .extend({ + queryLength: z.number().int().nonnegative(), + resultCount: z.number().int().nonnegative(), + durationMs: z.number().nonnegative(), + outcome: outcomeSchema, + }) + .strict(); + +const mcpRequestCompletedSchema = telemetryCommonEnvelopeSchema + .extend({ + toolName: z.string(), + outcome: outcomeSchema, + durationMs: z.number().nonnegative(), + errorClass: z.string().optional(), + sampleRate: z.literal(0.1), + }) + .strict(); + /** @internal */ export const telemetryEventSchemas = { install_first_run: installFirstRunSchema, command: commandSchema, + setup_step: setupStepSchema, + connection_added: connectionAddedSchema, + connection_test: connectionTestSchema, + project_stack_snapshot: projectStackSnapshotSchema, + ingest_completed: ingestCompletedSchema, + scan_completed: scanCompletedSchema, + sl_validate_completed: slValidateCompletedSchema, + sl_query_completed: slQueryCompletedSchema, + sql_completed: sqlCompletedSchema, + wiki_query_completed: wikiQueryCompletedSchema, + mcp_request_completed: mcpRequestCompletedSchema, } as const; /** @internal */ @@ -53,6 +198,96 @@ export const telemetryEventCatalog = [ 'projectGroupAttached', ], }, + { + name: 'setup_step', + description: 'Emitted after an interactive setup step completes, skips, or aborts.', + fields: ['step', 'outcome', 'durationMs'], + }, + { + name: 'connection_added', + description: 'Emitted when setup writes a database, source, or demo connection.', + fields: ['driver', 'isDemoConnection'], + }, + { + name: 'connection_test', + description: 'Emitted after ktx connection test completes.', + fields: ['driver', 'isDemoConnection', 'outcome', 'errorClass', 'durationMs', 'serverVersion'], + }, + { + name: 'project_stack_snapshot', + description: 'Emitted after commands that can summarize the local project stack.', + fields: ['connectors', 'connectionCount', 'hasSl', 'hasWiki', 'hasMcp', 'hasManagedRuntime'], + }, + { + name: 'ingest_completed', + description: 'Emitted after a public ingest target completes.', + fields: [ + 'driver', + 'isDemoConnection', + 'schemaCount', + 'tableCount', + 'columnCount', + 'rowsBucket', + 'durationMs', + 'outcome', + 'errorClass', + ], + }, + { + name: 'scan_completed', + description: 'Emitted after schema scan or relationship inference completes.', + fields: [ + 'driver', + 'tableCount', + 'columnCount', + 'inferredFkCount', + 'declaredFkCount', + 'durationMs', + 'outcome', + 'errorClass', + ], + }, + { + name: 'sl_validate_completed', + description: 'Emitted after ktx sl validate completes.', + fields: ['sourceCount', 'modelCount', 'validationErrorCount', 'outcome', 'errorClass', 'durationMs'], + }, + { + name: 'sl_query_completed', + description: 'Emitted after ktx sl query compiles or executes.', + fields: [ + 'mode', + 'referencedSourceCount', + 'referencedDimensionCount', + 'referencedMeasureCount', + 'durationMs', + 'outcome', + 'errorClass', + ], + }, + { + name: 'sql_completed', + description: 'Emitted after ktx sql completes validation and execution.', + fields: [ + 'driver', + 'isDemoConnection', + 'queryVerb', + 'referencedTableCount', + 'durationMs', + 'outcome', + 'errorClass', + ], + }, + { + name: 'wiki_query_completed', + description: 'Emitted after a wiki query completes.', + fields: ['queryLength', 'resultCount', 'durationMs', 'outcome'], + }, + { + name: 'mcp_request_completed', + description: 'Emitted for sampled MCP tool requests.', + fields: ['toolName', 'outcome', 'durationMs', 'errorClass', 'sampleRate'], + }, ] as const; export type TelemetryEventName = keyof typeof telemetryEventSchemas;