diff --git a/packages/cli/src/demo-assets.test.ts b/packages/cli/src/demo-assets.test.ts index 92aad645..052eda83 100644 --- a/packages/cli/src/demo-assets.test.ts +++ b/packages/cli/src/demo-assets.test.ts @@ -2,7 +2,7 @@ import { access, readFile, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { DEMO_ADAPTER, DEMO_CONNECTION_ID, @@ -22,10 +22,27 @@ async function readPackagedJson(relativePath: string): Promise { return JSON.parse(await readFile(packagedDemoAssetPath(relativePath), 'utf-8')) as T; } +function makeIo() { + let stderr = ''; + return { + stdout: { + isTTY: true, + write() {}, + }, + stderr: { + write(chunk: string) { + stderr += chunk; + }, + }, + stderrText: () => stderr, + }; +} + describe('demo assets', () => { const projectDir = join(tmpdir(), `ktx-demo-assets-${process.pid}`); afterEach(async () => { + vi.unstubAllEnvs(); await rm(projectDir, { recursive: true, force: true }); }); @@ -125,6 +142,19 @@ describe('demo assets', () => { await expect(ensureDemoProject({ projectDir, force: true })).resolves.toMatchObject({ projectDir }); }); + it('emits debug telemetry when the demo connection is created', async () => { + vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); + vi.stubEnv('CI', ''); + const io = makeIo(); + + await ensureDemoProject({ projectDir, force: false, io, cliVersion: '0.2.0' }); + + expect(io.stderrText()).toContain('"event":"connection_added"'); + expect(io.stderrText()).toContain('"driver":"sqlite"'); + expect(io.stderrText()).toContain('"isDemoConnection":true'); + expect(io.stderrText()).not.toContain(projectDir); + }); + it('copies the seeded project assets used by the setup wizard tour', async () => { await ensureSeededDemoProject({ projectDir, force: false }); diff --git a/packages/cli/src/demo-assets.ts b/packages/cli/src/demo-assets.ts index dcc7ac1f..f9e7fb32 100644 --- a/packages/cli/src/demo-assets.ts +++ b/packages/cli/src/demo-assets.ts @@ -4,6 +4,7 @@ import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { randomBytes } from 'node:crypto'; +import type { KtxCliIo } from './cli-runtime.js'; interface DemoProjectResult { projectDir: string; @@ -15,6 +16,8 @@ interface DemoProjectResult { interface EnsureDemoProjectOptions { projectDir: string; force: boolean; + io?: KtxCliIo; + cliVersion?: string; } /** @internal */ @@ -143,6 +146,19 @@ export async function ensureDemoProject(options: EnsureDemoProjectOptions): Prom await copyFile(join(assetDir(), 'manifest.json'), join(projectDir, 'manifest.json')); const replayPath = await copyPackagedReplay(projectDir); await writeFile(configPath, demoConfig(databasePath), 'utf-8'); + if (options.io) { + const { emitTelemetryEvent } = await import('./telemetry/index.js'); + await emitTelemetryEvent({ + name: 'connection_added', + projectDir, + io: options.io, + packageInfo: { name: '@kaelio/ktx', version: options.cliVersion ?? '0.0.0' }, + fields: { + driver: 'sqlite', + isDemoConnection: true, + }, + }); + } return { projectDir, configPath, databasePath, replayPath }; } diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/src/setup-databases.test.ts index 178ed076..f048987f 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/src/setup-databases.test.ts @@ -125,6 +125,7 @@ describe('setup databases step', () => { }); afterEach(async () => { + vi.unstubAllEnvs(); await rm(tempDir, { recursive: true, force: true }); }); @@ -316,6 +317,34 @@ describe('setup databases step', () => { }); }); + it('emits debug telemetry when setup writes a database connection', async () => { + vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); + vi.stubEnv('CI', ''); + const io = makeIo(); + const prompts = makePromptAdapter({ + selectValues: ['url'], + textValues: ['', 'env:DATABASE_URL'], + }); + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'auto', + databaseDrivers: ['postgres'], + databaseSchemas: [], + skipDatabases: false, + }, + io.io, + { prompts, testConnection: vi.fn(async () => 0), scanConnection: vi.fn(async () => 0) }, + ); + + expect(result.status).toBe('ready'); + expect(io.stderr()).toContain('"event":"connection_added"'); + expect(io.stderr()).toContain('"driver":"postgres"'); + expect(io.stderr()).toContain('"isDemoConnection":false'); + expect(io.stderr()).not.toContain(tempDir); + }); + it('tells users Escape goes back in free-text connection prompts', async () => { const prompts = makePromptAdapter({ selectValues: ['url'], diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index c8e735c5..befd82bc 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -19,6 +19,8 @@ import { withMultiselectNavigation, withTextInputNavigation } from './prompt-nav import { runKtxScan } from './scan.js'; import { applySetupDatabaseContextDepth } from './setup-database-context-depth.js'; import { writeProjectLocalSecretReference } from './setup-secrets.js'; +import { isDemoConnection } from './telemetry/demo-detect.js'; +import { emitTelemetryEvent } from './telemetry/index.js'; import { createKtxSetupPromptAdapter, type KtxSetupPromptOption, @@ -1198,6 +1200,7 @@ async function writeConnectionConfig(input: { projectDir: string; connectionId: string; connection: KtxProjectConnectionConfig; + io?: KtxCliIo; }): Promise { const project = await loadKtxProject({ projectDir: input.projectDir }); const migratedConnections = Object.fromEntries( @@ -1215,6 +1218,17 @@ async function writeConnectionConfig(input: { }, }; await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); + if (input.io) { + await emitTelemetryEvent({ + name: 'connection_added', + projectDir: input.projectDir, + io: input.io, + fields: { + driver: String(nextConnection.driver ?? 'unknown').toLowerCase(), + isDemoConnection: isDemoConnection(input.connectionId, nextConnection), + }, + }); + } const queryHistory = queryHistoryConfigRecord(nextConnection); if (queryHistory?.enabled === true) { @@ -1479,6 +1493,7 @@ async function maybeConfigureDatabaseScope(input: { projectDir: input.projectDir, connectionId: input.connectionId, connection: { ...currentConnection, enabled_tables: enabledTables }, + io: input.io, }); if (spec && activeSchemas.length > 0) { @@ -1911,6 +1926,7 @@ async function runPrimarySourceFullEdit(input: { }, driver, }), + io: input.io, }); const validated = await validateAndScanConnection({ @@ -2146,6 +2162,7 @@ export async function runKtxSetupDatabasesStep( projectDir: args.projectDir, connectionId: connectionChoice.connectionId, connection: withContextDepth, + io, }); } else { const existing = project.config.connections[connectionChoice.connectionId]; @@ -2171,6 +2188,7 @@ export async function runKtxSetupDatabasesStep( projectDir: args.projectDir, connectionId: connectionChoice.connectionId, connection: withContextDepth, + io, }); } @@ -2254,6 +2272,7 @@ export async function runKtxSetupDatabasesStep( projectDir: args.projectDir, connectionId: connectionChoice.connectionId, connection: withContextDepth, + io, }); setupStatus = await validateAndScanConnection({ projectDir: args.projectDir, diff --git a/packages/cli/src/setup-demo-tour.ts b/packages/cli/src/setup-demo-tour.ts index 79f71fbe..da80b988 100644 --- a/packages/cli/src/setup-demo-tour.ts +++ b/packages/cli/src/setup-demo-tour.ts @@ -339,7 +339,7 @@ export interface DemoTourDeps { } export async function runDemoTour( - args: { inputMode: 'auto' | 'disabled' }, + args: { inputMode: 'auto' | 'disabled'; cliVersion?: string }, io: KtxCliIo, deps: DemoTourDeps = {}, ): Promise { @@ -347,7 +347,7 @@ export async function runDemoTour( const ensureProject = deps.ensureProject ?? ensureSeededDemoProject; const projectDir = defaultDemoProjectDir(); - await ensureProject({ projectDir, force: false }); + await ensureProject({ projectDir, force: false, io, cliVersion: args.cliVersion }); io.stdout.write(renderDemoBanner(projectDir) + '\n'); io.stdout.write(`\n│ ${dim('Press Enter to continue, Escape to go back')}\n└\n`); diff --git a/packages/cli/src/setup-sources.test.ts b/packages/cli/src/setup-sources.test.ts index a9de4436..be8cc0d3 100644 --- a/packages/cli/src/setup-sources.test.ts +++ b/packages/cli/src/setup-sources.test.ts @@ -79,6 +79,7 @@ describe('setup sources step', () => { }); afterEach(async () => { + vi.unstubAllEnvs(); await rm(tempDir, { recursive: true, force: true }); }); @@ -169,6 +170,34 @@ describe('setup sources step', () => { expect(runInitialIngest).toHaveBeenCalledWith(projectDir, 'analytics_dbt', io.io, { inputMode: 'disabled' }); }); + it('emits debug telemetry when setup writes a source connection', async () => { + vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); + vi.stubEnv('CI', ''); + await addPrimarySource(); + const io = makeIo(); + + const result = await runKtxSetupSourcesStep( + { + projectDir, + inputMode: 'disabled', + source: 'dbt', + sourceConnectionId: 'analytics_dbt', + sourcePath: '/repo/dbt', + sourceProjectName: 'analytics', + runInitialSourceIngest: false, + skipSources: false, + }, + io.io, + { validateDbt: vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })) }, + ); + + expect(result.status).toBe('ready'); + expect(io.stderr()).toContain('"event":"connection_added"'); + expect(io.stderr()).toContain('"driver":"dbt"'); + expect(io.stderr()).toContain('"isDemoConnection":false'); + expect(io.stderr()).not.toContain(projectDir); + }); + it('writes Metabase config and validates mapping through existing mapping path', async () => { await addPrimarySource(); const validateMetabase = vi.fn(async () => ({ ok: true as const, detail: 'user=admin@example.com' })); diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index ff8eb420..e6da74ae 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -22,6 +22,8 @@ import { runKtxSourceMapping } from './source-mapping.js'; import { withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js'; import { runKtxPublicIngest } from './public-ingest.js'; import { writeProjectLocalSecretReference } from './setup-secrets.js'; +import { isDemoConnection } from './telemetry/demo-detect.js'; +import { emitTelemetryEvent } from './telemetry/index.js'; import { createKtxSetupPromptAdapter, type KtxSetupPromptOption, @@ -320,6 +322,7 @@ async function writeSourceConnection( connectionId: string, connection: KtxProjectConnectionConfig, adapter: string, + io?: KtxCliIo, ): Promise<() => Promise> { assertSafeConnectionId(connectionId); const project = await loadKtxProject({ projectDir }); @@ -340,6 +343,17 @@ async function writeSourceConnection( }, }; await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); + if (io) { + await emitTelemetryEvent({ + name: 'connection_added', + projectDir, + io, + fields: { + driver: String(connection.driver ?? adapter).toLowerCase(), + isDemoConnection: isDemoConnection(connectionId, connection), + }, + }); + } return async () => { const latest = await loadKtxProject({ projectDir }); const connections = { ...latest.config.connections }; @@ -1730,6 +1744,7 @@ async function saveValidateAndMaybeBuildSource(input: { connectionId, connection, sourceAdapter(input.source), + input.io, ); if (input.sourceChoice.kind === 'existing') { diff --git a/packages/cli/src/setup.test.ts b/packages/cli/src/setup.test.ts index 9c4a3b58..a4f2b724 100644 --- a/packages/cli/src/setup.test.ts +++ b/packages/cli/src/setup.test.ts @@ -23,6 +23,7 @@ function makeIo() { return { io: { stdout: { + isTTY: false, write: (chunk: string) => { stdout += chunk; }, @@ -91,6 +92,7 @@ describe('setup status', () => { }); afterEach(async () => { + vi.unstubAllEnvs(); await rm(tempDir, { recursive: true, force: true }); }); @@ -528,6 +530,43 @@ describe('setup status', () => { expect(output).not.toContain('Finish agent setup'); }); + it('emits debug telemetry for setup steps without project paths', async () => { + vi.stubEnv('KTX_TELEMETRY_DEBUG', '1'); + vi.stubEnv('CI', ''); + const testIo = makeIo(); + testIo.io.stdout.isTTY = true; + + await expect( + runKtxSetup( + { + command: 'run', + projectDir: tempDir, + mode: 'auto', + agents: false, + skipAgents: true, + inputMode: 'disabled', + yes: true, + cliVersion: '0.2.0', + skipLlm: true, + skipEmbeddings: true, + skipDatabases: true, + skipSources: true, + databaseSchemas: [], + }, + testIo.io, + { + runtime: async () => runtimeReady(tempDir), + context: async () => ({ status: 'ready', projectDir: tempDir, runId: 'setup-context-local-test' }), + }, + ), + ).resolves.toBe(0); + + expect(testIo.stderr()).toContain('"event":"setup_step"'); + expect(testIo.stderr()).toContain('"step":"project"'); + expect(testIo.stderr()).toContain('"step":"models"'); + expect(testIo.stderr()).not.toContain(tempDir); + }); + it('prints the setup shell intro for auto-created run mode', async () => { const testIo = makeIo(); @@ -1047,7 +1086,7 @@ describe('setup status', () => { ).resolves.toBe(0); expect(runDemoTour).toHaveBeenCalledWith( - { inputMode: 'auto' }, + { inputMode: 'auto', cliVersion: '0.2.0' }, testIo.io, expect.objectContaining({}), ); diff --git a/packages/cli/src/setup.ts b/packages/cli/src/setup.ts index 9b4c3a24..2977a9f4 100644 --- a/packages/cli/src/setup.ts +++ b/packages/cli/src/setup.ts @@ -6,7 +6,7 @@ import { savedMemoryCountsForReport } from './context/ingest/reports.js'; import { ktxLocalStateDbPath } from './context/project/local-state-db.js'; import { loadKtxProject, type KtxLocalProject } from './context/project/project.js'; import { readKtxSetupState } from './context/project/setup-config.js'; -import type { KtxCliIo } from './cli-runtime.js'; +import { getKtxCliPackageInfo, type KtxCliIo } from './cli-runtime.js'; import { formatSetupNextStepLines } from './next-steps.js'; import { runtimeInstallPolicyFromFlags } from './managed-python-command.js'; import { readManagedPythonRuntimeStatus } from './managed-python-runtime.js'; @@ -179,6 +179,16 @@ type KtxSetupFlowStatus = | 'back' | 'missing-input' | 'failed'; +type TelemetrySetupStep = + | 'project' + | 'runtime' + | 'models' + | 'embeddings' + | 'databases' + | 'sources' + | 'context' + | 'agents' + | 'demo-tour'; export interface KtxSetupEntryMenuPromptAdapter { select(options: { message: string; options: KtxSetupPromptOption[] }): Promise; @@ -196,6 +206,36 @@ function createEntryMenuPromptAdapter(): KtxSetupEntryMenuPromptAdapter { }); } +function setupTelemetryOutcome( + status: KtxSetupFlowStatus | Extract>, { status: string }>['status'], +): 'completed' | 'skipped' | 'abandoned' { + if (status === 'ready') return 'completed'; + if (status === 'skipped') return 'skipped'; + return 'abandoned'; +} + +async function recordSetupStep(input: { + projectDir: string; + step: TelemetrySetupStep; + status: KtxSetupFlowStatus | Extract>, { status: string }>['status']; + startedAt: number; + io: KtxCliIo; + cliVersion?: string; +}): Promise { + const { emitTelemetryEvent } = await import('./telemetry/index.js'); + await emitTelemetryEvent({ + name: 'setup_step', + projectDir: input.projectDir, + io: input.io, + packageInfo: { name: '@kaelio/ktx', version: input.cliVersion ?? getKtxCliPackageInfo().version }, + fields: { + step: input.step, + outcome: setupTelemetryOutcome(input.status), + durationMs: Math.max(0, performance.now() - input.startedAt), + }, + }); +} + async function runKtxSetupEntryMenu( status: KtxSetupStatus, deps: KtxSetupEntryMenuDeps = {}, @@ -229,11 +269,21 @@ async function runKtxSetupDemoFromEntryMenu( deps: KtxSetupDeps, ): Promise { const { runDemoTour } = await import('./setup-demo-tour.js'); - return await runDemoTour( - { inputMode: args.inputMode }, + const startedAt = performance.now(); + const result = await runDemoTour( + { inputMode: args.inputMode, cliVersion: args.cliVersion }, io, { agents: deps.agents }, ); + await recordSetupStep({ + projectDir: args.projectDir, + step: 'demo-tour', + status: result === 0 ? 'ready' : 'failed', + startedAt, + io, + cliVersion: args.cliVersion, + }); + return result; } function embeddingsReady(status: KtxSetupStatus['embeddings']): boolean { @@ -564,6 +614,7 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup } const projectMode = entryAction === 'new-project' ? 'prompt-new' : args.mode; + const projectStepStartedAt = performance.now(); projectResult = await runKtxSetupProjectStep( { projectDir: args.projectDir, @@ -575,6 +626,14 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup io, deps.project, ); + await recordSetupStep({ + projectDir: projectResult.projectDir, + step: 'project', + status: projectResult.status, + startedAt: projectStepStartedAt, + io, + cliVersion: args.cliVersion, + }); if (projectResult.status === 'back') { continue; @@ -640,6 +699,7 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup const step = setupSteps[stepIndex]; if (!step) break; + const stepStartedAt = performance.now(); let stepResult: { status: KtxSetupFlowStatus }; if (step === 'models') { const modelRunner = @@ -792,6 +852,15 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup } } + await recordSetupStep({ + projectDir: projectResult.projectDir, + step, + status: stepResult.status, + startedAt: stepStartedAt, + io, + cliVersion: args.cliVersion, + }); + if (stepResult.status === 'failed') { await cleanupCreatedProjectScaffold(projectResult.createdProjectCleanup); return 1;