ktx/packages/cli/test/cli-program-telemetry.test.ts

134 lines
4.5 KiB
TypeScript
Raw Permalink Normal View History

feat(telemetry): anonymous posthog usage telemetry across node cli and python daemon (#205) * feat: add telemetry phase 1 * feat: add node telemetry event catalog * feat: add telemetry event helpers * feat: emit setup and connection telemetry * feat: emit connection and stack telemetry * feat: emit ingest and scan telemetry * feat: emit query telemetry * feat: emit sampled mcp telemetry * docs: expand telemetry event catalog * feat: add telemetry schema sync artifact * feat: pass telemetry project id to semantic daemon * feat: add daemon telemetry foundation * feat: emit semantic daemon telemetry * feat: emit daemon lifecycle telemetry * docs: document full telemetry event catalog * feat(telemetry): dim first-run notice * feat(telemetry): show first-run notice before command output * feat(telemetry): wire ktx PostHog project for live ingestion * docs(telemetry): drop posthog project name and host from storage section * docs(telemetry): trim to general overview and disclaimer * docs(agents): add short telemetry guidelines * feat(telemetry): enable posthog geoip enrichment * docs(telemetry): drop ip-geoip note from public overview * refactor(telemetry): drop no-op groupIdentify, rely on capture groups field * fix(telemetry): respect CI kill switch in python daemon identity * fix(sql): route table-count analysis to existing analyze-batch endpoint * fix(telemetry): emit install_first_run from notice path and derive flagsPresent from commander * fix(telemetry): read package info via getKtxCliPackageInfo to satisfy boundary check * fix(telemetry): make python identity env={} bypass os.environ and unset CI in tests * fix(telemetry): unset CI kill switch in cli-program-telemetry tests
2026-05-22 18:18:47 +02:00
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
2026-05-25 13:17:46 +02:00
import { runCommanderKtxCli } from '../src/cli-program.js';
import type { KtxCliDeps, KtxCliIo, KtxCliPackageInfo } from '../src/cli-runtime.js';
feat(telemetry): anonymous posthog usage telemetry across node cli and python daemon (#205) * feat: add telemetry phase 1 * feat: add node telemetry event catalog * feat: add telemetry event helpers * feat: emit setup and connection telemetry * feat: emit connection and stack telemetry * feat: emit ingest and scan telemetry * feat: emit query telemetry * feat: emit sampled mcp telemetry * docs: expand telemetry event catalog * feat: add telemetry schema sync artifact * feat: pass telemetry project id to semantic daemon * feat: add daemon telemetry foundation * feat: emit semantic daemon telemetry * feat: emit daemon lifecycle telemetry * docs: document full telemetry event catalog * feat(telemetry): dim first-run notice * feat(telemetry): show first-run notice before command output * feat(telemetry): wire ktx PostHog project for live ingestion * docs(telemetry): drop posthog project name and host from storage section * docs(telemetry): trim to general overview and disclaimer * docs(agents): add short telemetry guidelines * feat(telemetry): enable posthog geoip enrichment * docs(telemetry): drop ip-geoip note from public overview * refactor(telemetry): drop no-op groupIdentify, rely on capture groups field * fix(telemetry): respect CI kill switch in python daemon identity * fix(sql): route table-count analysis to existing analyze-batch endpoint * fix(telemetry): emit install_first_run from notice path and derive flagsPresent from commander * fix(telemetry): read package info via getKtxCliPackageInfo to satisfy boundary check * fix(telemetry): make python identity env={} bypass os.environ and unset CI in tests * fix(telemetry): unset CI kill switch in cli-program-telemetry tests
2026-05-22 18:18:47 +02:00
function makeIo(stdoutIsTTY = true): { io: KtxCliIo; stdout: () => string; stderr: () => string } {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
isTTY: stdoutIsTTY,
write: (chunk) => {
stdout += chunk;
},
},
stderr: {
write: (chunk) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
const info: KtxCliPackageInfo = { name: '@kaelio/ktx', version: '0.4.1' };
describe('runCommanderKtxCli telemetry', () => {
let tempDir: string;
const originalEnv = process.env;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-telemetry-'));
await writeFile(join(tempDir, 'ktx.yaml'), '{}\n', 'utf-8');
vi.stubEnv('KTX_TELEMETRY_DEBUG', '1');
vi.stubEnv('HOME', tempDir);
vi.stubEnv('CI', '');
vi.stubEnv('KTX_TELEMETRY_DISABLED', '');
vi.stubEnv('DO_NOT_TRACK', '');
});
afterEach(async () => {
vi.unstubAllEnvs();
process.env = originalEnv;
await rm(tempDir, { recursive: true, force: true });
});
it('emits debug command telemetry for registered actions', async () => {
const io = makeIo(true);
await expect(
runCommanderKtxCli(
['--project-dir', tempDir, 'status', '--help'],
io.io,
{},
info,
{ runInit: async () => 0 },
),
).resolves.toBe(0);
expect(io.stderr()).not.toContain('[telemetry]');
const statusIo = makeIo(true);
const deps: KtxCliDeps = { doctor: async () => 0 };
await expect(
runCommanderKtxCli(
['--project-dir', tempDir, 'status', '--json'],
statusIo.io,
deps,
info,
{ runInit: async () => 0 },
),
).resolves.toBe(0);
expect(statusIo.stderr()).toContain('[telemetry]');
expect(statusIo.stderr()).toContain('"event":"install_first_run"');
expect(statusIo.stderr()).toContain('"event":"command"');
expect(statusIo.stderr()).toContain('"commandPath":["ktx","status"]');
expect(statusIo.stderr()).toContain('"event":"project_stack_snapshot"');
expect(statusIo.stderr()).toContain('"connectionCount"');
expect(statusIo.stderr()).not.toContain(tempDir);
const noticeIndex = statusIo.stderr().indexOf('ktx collects anonymous usage data');
const firstTelemetryIndex = statusIo.stderr().indexOf('[telemetry]');
expect(noticeIndex).toBeGreaterThanOrEqual(0);
expect(firstTelemetryIndex).toBeGreaterThan(noticeIndex);
});
it('emits aborted telemetry when project validation aborts after preAction starts', async () => {
const missingProjectDir = join(tempDir, 'missing');
await mkdir(missingProjectDir, { recursive: true });
const io = makeIo(true);
await expect(
runCommanderKtxCli(
['--project-dir', missingProjectDir, 'connection'],
io.io,
{},
info,
{ runInit: async () => 0 },
),
).resolves.toBe(1);
expect(io.stderr()).toContain('[telemetry]');
expect(io.stderr()).toContain('"outcome":"aborted"');
expect(io.stderr()).toContain('"hasProject":false');
expect(io.stderr()).toContain('"projectGroupAttached":false');
expect(io.stderr()).not.toContain(missingProjectDir);
});
it('does not import or emit telemetry for help, version, bare non-TTY, or unknown top-level command', async () => {
const helpIo = makeIo(true);
await expect(runCommanderKtxCli(['--help'], helpIo.io, {}, info, { runInit: async () => 0 })).resolves.toBe(0);
expect(helpIo.stderr()).not.toContain('[telemetry]');
const versionIo = makeIo(true);
await expect(runCommanderKtxCli(['--version'], versionIo.io, {}, info, { runInit: async () => 0 })).resolves.toBe(0);
expect(versionIo.stderr()).not.toContain('[telemetry]');
const bareIo = makeIo(false);
await expect(runCommanderKtxCli([], bareIo.io, {}, info, { runInit: async () => 0 })).resolves.toBe(0);
expect(bareIo.stderr()).not.toContain('[telemetry]');
const unknownIo = makeIo(true);
await expect(runCommanderKtxCli(['unknown'], unknownIo.io, {}, info, { runInit: async () => 0 })).resolves.toBe(1);
expect(unknownIo.stderr()).not.toContain('[telemetry]');
});
});