diff --git a/apps/x/ANALYTICS.md b/apps/x/ANALYTICS.md index 59183479..cf3a0790 100644 --- a/apps/x/ANALYTICS.md +++ b/apps/x/ANALYTICS.md @@ -28,7 +28,7 @@ Emitted whenever ai-sdk returns token usage (one event per LLM call, not per run | `sub_use_case` | string? | Refines `use_case` — see taxonomy table below | | `agent_name` | string? | Present when the call goes through an agent run (`createRun`); omitted for direct `generateText`/`generateObject` | | `model` | string | e.g. `claude-sonnet-4-6` | -| `provider` | string | `rowboat` = cloud LLM gateway; otherwise the BYOK provider (`openai`, `anthropic`, `ollama`, etc.) | +| `provider` | string | The provider FLAVOR: `rowboat` = cloud LLM gateway, `codex` = ChatGPT subscription, else the BYOK flavor (`openai`, `anthropic`, `ollama`, …). Call sites pass instance ids; `captureLlmUsage` maps id → flavor so charts never fracture if user-named provider instances ship (ids never leave the app) | | `input_tokens` | number | | | `output_tokens` | number | | | `total_tokens` | number | | @@ -84,6 +84,14 @@ Emitted on rowboat disconnect. No properties. Followed immediately by `posthog.r Emit points: `apps/main/src/oauth-handler.ts:369` and `apps/renderer/src/hooks/useAnalyticsIdentity.ts:82`. +### Model-provider lifecycle + +Privacy rules (enforced in `packages/core/src/analytics/model-providers.ts`): only provider **flavors** are captured — never instance ids (future-proofing for user-named instances), never `apiKey`/`headers`, and never `baseURL` (local endpoints can carry internal hostnames). Model ids are allowed. + +- `llm_provider_connected` / `llm_provider_disconnected` — `{ flavor }` — one event family across every surface. BYOK fires from `FSModelConfigRepo.setProvider` (new entries only — key rotation is not a connect) / `removeProvider`; `rowboat` from sign-in/out (`apps/main/src/oauth-handler.ts`); `codex` from ChatGPT sign-in/out (`apps/main/src/ipc.ts`). +- `llm_initial_model_selected` — `{ flavor, model, recommended, source: 'connect' | 'onboarding' | 'sign_in' }` — a connect seeded the assistant model (only when none was configured). `recommended: false` = first-listed fallback; the hit rate measures backend recommendation quality. Emit points: `apps/renderer/src/components/settings/providers-section.tsx` (connect/onboarding) and `packages/core/src/models/rowboat-selection.ts` (sign-in). +- `models_config_migrated` — `{ had_assistant, materialized_overrides, provider_count }` — one-shot per install at the models.json v1 → v2 boot migration (`FSModelConfigRepo.ensureConfig`); rollout health for the schema change. + ### Other events (pre-existing, not added by the LLM-usage work) All in `apps/renderer/src/lib/analytics.ts`: @@ -210,6 +218,9 @@ Persistent across sessions for the same user. Set via `posthog.people.set` or as | `has_used_search`, `has_used_voice` | renderer | One-shot first-use flags | | `has_used_email`, `has_used_meetings`, `has_used_live_notes`, `has_used_bg_agents`, `has_used_apps`, `has_used_code` | renderer (`view_opened`) | One-shot first-use flags per feature view | | `has_created_bg_agent` | renderer | One-shot: user set up a background agent | +| `llm_provider_flavors` | main | Sorted array of connected provider flavors incl. `rowboat`/`codex` from auth state (e.g. `["openai","openrouter","rowboat"]`). Synced on every launch and after any provider/assistant change (`packages/core/src/analytics/model-providers.ts`) | +| `llm_provider_count` | main | Size of `llm_provider_flavors` | +| `assistant_model`, `assistant_model_flavor` | main | The configured primary model (complements `llm_usage`, which reports actual usage). Absent until an assistant is configured | ## How to add a new event diff --git a/apps/x/apps/main/src/ipc.ts b/apps/x/apps/main/src/ipc.ts index cd8e586f..cc9d177c 100644 --- a/apps/x/apps/main/src/ipc.ts +++ b/apps/x/apps/main/src/ipc.ts @@ -35,6 +35,7 @@ import type { ITurnEventBus } from '@x/core/dist/runtime/turns/event-hub.js'; import container from '@x/core/dist/di/container.js'; import { testModelConnection, listModelsForProvider, generateOneShot } from '@x/core/dist/models/models.js'; import { getModelCatalog } from '@x/core/dist/models/catalog.js'; +import { captureProviderConnected, captureProviderDisconnected } from '@x/core/dist/analytics/model-providers.js'; import { getDefaultModelAndProvider } from '@x/core/dist/models/defaults.js'; import { isSignedIn } from '@x/core/dist/account/account.js'; import type { IModelConfigRepo } from '@x/core/dist/models/repo.js'; @@ -1336,6 +1337,7 @@ export function setupIpcHandlers() { // Model lists gate on sign-in state (composer picker, models:list) — // push the change so they refresh without polling. broadcastToWindows('chatgpt:statusChanged', { signedIn: true }); + captureProviderConnected('codex'); } return result; }, @@ -1347,6 +1349,7 @@ export function setupIpcHandlers() { try { await signOutChatGPT(); broadcastToWindows('chatgpt:statusChanged', { signedIn: false }); + captureProviderDisconnected('codex'); return { success: true }; } catch (error) { console.error('[ChatGPTAuth] Sign-out failed:', error); diff --git a/apps/x/apps/main/src/main.ts b/apps/x/apps/main/src/main.ts index e442cba3..8b627ca2 100644 --- a/apps/x/apps/main/src/main.ts +++ b/apps/x/apps/main/src/main.ts @@ -43,6 +43,7 @@ import { setTokenCipher as setGithubTokenCipher } from "@x/core/dist/apps/github import { setTokenCipher as setChatGPTTokenCipher } from "@x/core/dist/auth/chatgpt-auth.js"; import { shutdown as shutdownAnalytics } from "@x/core/dist/analytics/posthog.js"; import { identifyIfSignedIn } from "@x/core/dist/analytics/identify.js"; +import { syncModelProviderPersonProperties } from "@x/core/dist/analytics/model-providers.js"; import { migrateRuns } from "@x/core/dist/migrations/runs/migrate.js"; import { initConfigs } from "@x/core/dist/config/initConfigs.js"; @@ -511,6 +512,11 @@ app.whenReady().then(async () => { identifyIfSignedIn().catch((error) => { console.error('[Analytics] Failed to identify on startup:', error); }); + // Baseline the provider person properties (llm_provider_flavors et al) on + // every launch — existing installs get them without any provider action. + syncModelProviderPersonProperties().catch((error) => { + console.error('[Analytics] Failed to sync provider properties:', error); + }); registerBrowserControlService(new ElectronBrowserControlService()); registerNotificationService(new ElectronNotificationService(APP_LAUNCHED_AT)); diff --git a/apps/x/apps/main/src/oauth-handler.ts b/apps/x/apps/main/src/oauth-handler.ts index 78d6c4d1..a5c458ba 100644 --- a/apps/x/apps/main/src/oauth-handler.ts +++ b/apps/x/apps/main/src/oauth-handler.ts @@ -18,6 +18,7 @@ import { isSignedIn } from '@x/core/dist/account/account.js'; import { getWebappUrl } from '@x/core/dist/config/remote-config.js'; import { claimTokensViaBackend } from '@x/core/dist/auth/google-backend-oauth.js'; import { applyRowboatInitialSelection, clearRowboatSelections } from '@x/core/dist/models/rowboat-selection.js'; +import { captureProviderConnected, captureProviderDisconnected } from '@x/core/dist/analytics/model-providers.js'; function buildRedirectUri(port: number): string { return `http://localhost:${port}/oauth/callback`; @@ -345,6 +346,7 @@ export async function connectProvider(provider: string, credentials?: { clientId // the gateway lists it, else first listed). Never replaces a // saved choice; best-effort by design. await applyRowboatInitialSelection(); + captureProviderConnected('rowboat'); try { const billing = await getBillingInfo(); if (billing.userId) { @@ -540,6 +542,7 @@ export async function disconnectProvider(provider: string): Promise<{ success: b // selections that reference it (same dangling-ref cleanup as removing // any provider). The composer prompts for a new pick. await clearRowboatSelections(); + captureProviderDisconnected('rowboat'); } // Notify renderer so sidebar, voice, and billing re-check state emitOAuthEvent({ provider, success: false }); diff --git a/apps/x/apps/renderer/src/components/settings/providers-section.tsx b/apps/x/apps/renderer/src/components/settings/providers-section.tsx index 508df12f..0eb15424 100644 --- a/apps/x/apps/renderer/src/components/settings/providers-section.tsx +++ b/apps/x/apps/renderer/src/components/settings/providers-section.tsx @@ -1,5 +1,6 @@ import { useCallback, useEffect, useMemo, useState } from "react" import { toast } from "sonner" +import * as analytics from "@/lib/analytics" import { ArrowLeft, CheckCircle2, Loader2, Plus, RefreshCw } from "lucide-react" import { Button } from "@/components/ui/button" import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog" @@ -225,6 +226,7 @@ export function ProvidersSection({ dialogOpen, variant = "settings" }: { onChatGPTSignIn={chatgpt.signIn} hadAssistant={selections.assistantModel !== null} modelRecommendations={modelRecommendations} + analyticsSource={variant === "onboarding" ? "onboarding" : "connect"} /> {manageCard && ( @@ -249,7 +251,7 @@ type AddStep = | { kind: "result"; name: string; first: boolean; pickedModel: string | null; modelCount: number | null } | { kind: "error"; flavor: ByokFlavor; message: string } -function AddProviderDialog({ open, onOpenChange, connectedIds, isRowboatConnected, chatgptSignedIn, onChatGPTSignIn, hadAssistant, modelRecommendations }: { +function AddProviderDialog({ open, onOpenChange, connectedIds, isRowboatConnected, chatgptSignedIn, onChatGPTSignIn, hadAssistant, modelRecommendations, analyticsSource }: { open: boolean onOpenChange: (open: boolean) => void connectedIds: string[] @@ -258,6 +260,7 @@ function AddProviderDialog({ open, onOpenChange, connectedIds, isRowboatConnecte onChatGPTSignIn: () => Promise | void hadAssistant: boolean modelRecommendations: Record | undefined + analyticsSource: 'connect' | 'onboarding' }) { const [step, setStep] = useState({ kind: "choose" }) const [apiKey, setApiKey] = useState("") @@ -392,6 +395,12 @@ function AddProviderDialog({ open, onOpenChange, connectedIds, isRowboatConnecte const hasAssistantNow = cfgNow ? cfgNow.assistantModel !== null : hadAssistant if (!hasAssistantNow) { await window.ipc.invoke("models:updateConfig", { assistantModel: { provider: flavor, model } }) + analytics.llmInitialModelSelected({ + flavor, + model, + recommended: model === modelRecommendations?.[flavor], + source: analyticsSource, + }) } for (const warning of testRes.warnings ?? []) { toast.warning(warning, { duration: 12000 }) diff --git a/apps/x/apps/renderer/src/lib/analytics.ts b/apps/x/apps/renderer/src/lib/analytics.ts index 0774e574..621c637b 100644 --- a/apps/x/apps/renderer/src/lib/analytics.ts +++ b/apps/x/apps/renderer/src/lib/analytics.ts @@ -336,3 +336,16 @@ export function settingsTabChanged(tab: string) { export function onboardingCompleted() { posthog.capture('onboarding_completed') } + +// A provider connect seeded the assistant model (only happens when none was +// configured). `recommended` = the backend's flavor recommendation was in +// the provider's live list; false = first-listed fallback. Flavor only — +// never provider instance ids, keys, or endpoints. +export function llmInitialModelSelected(props: { + flavor: string + model: string + recommended: boolean + source: 'connect' | 'onboarding' +}) { + posthog.capture('llm_initial_model_selected', { ...props }) +} diff --git a/apps/x/packages/core/src/analytics/model-providers.ts b/apps/x/packages/core/src/analytics/model-providers.ts new file mode 100644 index 00000000..3dc86d2f --- /dev/null +++ b/apps/x/packages/core/src/analytics/model-providers.ts @@ -0,0 +1,105 @@ +import { capture, setPersonProperties } from "./posthog.js"; +import type { IModelConfigRepo } from "../models/repo.js"; + +/** + * Provider-level analytics for model selection. + * + * Privacy rules, encoded here so call sites can't get them wrong: + * - Only provider FLAVORS ever leave the app. Instance ids equal flavor keys + * today, but a future multi-key setup makes ids user-named — so every + * surface maps id → flavor before capturing. + * - Never credentials: no apiKey, no headers, and no baseURL (local + * endpoints can carry internal hostnames). + * - Model ids are fine (they already ride on llm_usage). + * + * All I/O is lazy (dynamic imports, container resolution at call time) so + * this module stays import-cycle-free — models/repo.ts imports it. + */ + +const FLAVOR_CACHE_TTL_MS = 10_000; +let flavorCache: { at: number; byId: Map } | null = null; + +async function resolveRepo(): Promise { + const { default: container } = await import("../di/container.js"); + return container.resolve("modelConfigRepo"); +} + +async function providerFlavorsById(): Promise> { + if (flavorCache && Date.now() - flavorCache.at < FLAVOR_CACHE_TTL_MS) { + return flavorCache.byId; + } + const byId = new Map(); + try { + const cfg = await (await resolveRepo()).getConfig(); + for (const [id, entry] of Object.entries(cfg.providers)) { + byId.set(id, entry.flavor); + } + } catch { + // No config yet — empty map; ids fall through unchanged. + } + flavorCache = { at: Date.now(), byId }; + return byId; +} + +/** + * Map a provider instance id to its flavor for analytics. Unknown ids fall + * back to the raw value — which today always equals the flavor key. + */ +export async function flavorForProviderId(id: string): Promise { + if (id === "rowboat" || id === "codex") return id; + return (await providerFlavorsById()).get(id) ?? id; +} + +export function invalidateFlavorCache(): void { + flavorCache = null; +} + +/** + * Refresh the person properties describing the user's provider setup: + * `llm_provider_flavors` (sorted, includes rowboat/codex from auth state), + * `llm_provider_count`, and the configured assistant model. Call after any + * provider or assistant change; also called on every app launch so existing + * installs get baselined without waiting for an action. + */ +export async function syncModelProviderPersonProperties(): Promise { + try { + const cfg = await (await resolveRepo()).getConfig().catch(() => null); + const { isSignedIn } = await import("../account/account.js"); + const { getChatGPTStatus } = await import("../auth/chatgpt-auth.js"); + const flavors = new Set(); + for (const entry of Object.values(cfg?.providers ?? {})) { + flavors.add(entry.flavor); + } + if (await isSignedIn().catch(() => false)) flavors.add("rowboat"); + const chatgpt = await getChatGPTStatus().catch(() => ({ signedIn: false })); + if (chatgpt.signedIn) flavors.add("codex"); + + const assistant = cfg?.assistantModel ?? null; + setPersonProperties({ + llm_provider_flavors: [...flavors].sort(), + llm_provider_count: flavors.size, + ...(assistant + ? { + assistant_model: assistant.model, + assistant_model_flavor: await flavorForProviderId(assistant.provider), + } + : {}), + }); + } catch (err) { + console.error("[Analytics] provider person-props sync failed:", err); + } +} + +/** One provider became connected (any surface: settings, onboarding, sign-in). */ +export function captureProviderConnected(flavor: string): void { + capture("llm_provider_connected", { flavor }); + invalidateFlavorCache(); + void syncModelProviderPersonProperties(); +} + +/** One provider was disconnected / signed out. */ +export function captureProviderDisconnected(flavor: string): void { + capture("llm_provider_disconnected", { flavor }); + invalidateFlavorCache(); + void syncModelProviderPersonProperties(); +} diff --git a/apps/x/packages/core/src/analytics/posthog.ts b/apps/x/packages/core/src/analytics/posthog.ts index 81e24eb8..3b93091c 100644 --- a/apps/x/packages/core/src/analytics/posthog.ts +++ b/apps/x/packages/core/src/analytics/posthog.ts @@ -91,6 +91,21 @@ export function reset(): void { identifiedUserId = null; } +/** + * Merge person properties onto the CURRENT identity — the rowboat user once + * identified, else the anonymous installation id (identify on the same + * distinctId merges properties without changing identity). + */ +export function setPersonProperties(properties: Record): void { + const ph = getClient(); + if (!ph) return; + try { + ph.identify({ distinctId: activeDistinctId(), properties }); + } catch (err) { + console.error('[Analytics] setPersonProperties failed:', err); + } +} + /** * Evaluate a PostHog feature flag for the current identity (rowboat user id * once identified, installation id before that). `defaultValue` is returned diff --git a/apps/x/packages/core/src/analytics/usage.ts b/apps/x/packages/core/src/analytics/usage.ts index 31b703dc..a5421f33 100644 --- a/apps/x/packages/core/src/analytics/usage.ts +++ b/apps/x/packages/core/src/analytics/usage.ts @@ -21,18 +21,29 @@ export interface CaptureLlmUsageArgs { } export function captureLlmUsage(args: CaptureLlmUsageArgs): void { - const usage = args.usage ?? {}; - const properties: Record = { - use_case: args.useCase, - model: args.model, - provider: args.provider, - input_tokens: usage.inputTokens ?? 0, - output_tokens: usage.outputTokens ?? 0, - total_tokens: usage.totalTokens ?? (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0), - }; - if (args.subUseCase) properties.sub_use_case = args.subUseCase; - if (args.agentName) properties.agent_name = args.agentName; - if (usage.cachedInputTokens != null) properties.cached_input_tokens = usage.cachedInputTokens; - if (usage.reasoningTokens != null) properties.reasoning_tokens = usage.reasoningTokens; - capture('llm_usage', properties); + // Fire-and-forget: callers pass the provider INSTANCE id (ModelRef + // .provider); analytics reports the FLAVOR — ids may one day be + // user-named, and charts must not fracture when "openai-work" appears. + // Today ids equal flavor keys, so the fallback is lossless. + void (async () => { + let provider = args.provider; + try { + const { flavorForProviderId } = await import('./model-providers.js'); + provider = await flavorForProviderId(args.provider); + } catch { /* keep the raw value */ } + const usage = args.usage ?? {}; + const properties: Record = { + use_case: args.useCase, + model: args.model, + provider, + input_tokens: usage.inputTokens ?? 0, + output_tokens: usage.outputTokens ?? 0, + total_tokens: usage.totalTokens ?? (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0), + }; + if (args.subUseCase) properties.sub_use_case = args.subUseCase; + if (args.agentName) properties.agent_name = args.agentName; + if (usage.cachedInputTokens != null) properties.cached_input_tokens = usage.cachedInputTokens; + if (usage.reasoningTokens != null) properties.reasoning_tokens = usage.reasoningTokens; + capture('llm_usage', properties); + })(); } diff --git a/apps/x/packages/core/src/models/chatgpt-selection.ts b/apps/x/packages/core/src/models/chatgpt-selection.ts index 36d04005..1f1a57c4 100644 --- a/apps/x/packages/core/src/models/chatgpt-selection.ts +++ b/apps/x/packages/core/src/models/chatgpt-selection.ts @@ -3,6 +3,7 @@ import { IModelConfigRepo } from "./repo.js"; import { listCodexModels } from "./codex.js"; import { getRowboatConfig } from "../config/rowboat.js"; import { selectInitialModel } from "./initial-selection.js"; +import { capture } from "../analytics/posthog.js"; /** * Model-selection hooks for the ChatGPT-subscription (codex) sign-in @@ -27,6 +28,12 @@ export async function applyCodexInitialSelection(): Promise { const model = selectInitialModel("codex", ids, recommendations); if (model) { await repo.updateConfig({ assistantModel: { provider: "codex", model } }); + capture("llm_initial_model_selected", { + flavor: "codex", + model, + recommended: model === recommendations?.["codex"], + source: "sign_in", + }); } } catch (error) { // Best-effort: a failed initial selection must never break sign-in. diff --git a/apps/x/packages/core/src/models/repo.ts b/apps/x/packages/core/src/models/repo.ts index 6936c447..0311c5eb 100644 --- a/apps/x/packages/core/src/models/repo.ts +++ b/apps/x/packages/core/src/models/repo.ts @@ -1,6 +1,12 @@ import { LlmModelConfig, LlmProvider, ModelRef, TaskModels } from "@x/shared/dist/models.js"; import { WorkDir } from "../config/config.js"; import { isSignedIn } from "../account/account.js"; +import { capture } from "../analytics/posthog.js"; +import { + captureProviderConnected, + captureProviderDisconnected, + syncModelProviderPersonProperties, +} from "../analytics/model-providers.js"; import { migrateModelsConfig } from "./migrate.js"; import fs from "fs/promises"; import path from "path"; @@ -81,6 +87,12 @@ export class FSModelConfigRepo implements IModelConfigRepo { // record of the old selections once it runs. await fs.writeFile(`${this.configPath}.v1.bak`, rawText).catch(() => {}); await this.write(migrated); + // One-shot rollout signal for the v1 → v2 schema migration. + capture("models_config_migrated", { + had_assistant: Boolean(migrated.assistantModel), + materialized_overrides: Object.keys(migrated.taskModels ?? {}).length, + provider_count: Object.keys(migrated.providers).length, + }); } } @@ -98,6 +110,7 @@ export class FSModelConfigRepo implements IModelConfigRepo { throw new Error(`Provider flavor '${provider.flavor}' is auth-derived and cannot be stored in models.json`); } const config = await this.read(); + const isNew = !config.providers[id]; // Merge over an existing entry: replacing a key must not wipe // hand-tuned connection prefs (contextLength, reasoningEffort). config.providers[id] = LlmProvider.parse({ @@ -105,10 +118,13 @@ export class FSModelConfigRepo implements IModelConfigRepo { ...provider, }); await this.write(config); + // A brand-new entry is a connect; a key rotation is not. + if (isNew) captureProviderConnected(provider.flavor); } async removeProvider(id: string): Promise { const config = await this.read(); + const removed = config.providers[id]; delete config.providers[id]; if (config.assistantModel?.provider === id) { delete config.assistantModel; @@ -122,6 +138,7 @@ export class FSModelConfigRepo implements IModelConfigRepo { if (Object.keys(config.taskModels).length === 0) delete config.taskModels; } await this.write(config); + if (removed) captureProviderDisconnected(removed.flavor); } async updateConfig(patch: ModelConfigPatch): Promise { @@ -145,6 +162,10 @@ export class FSModelConfigRepo implements IModelConfigRepo { else config.deferBackgroundTasks = patch.deferBackgroundTasks; } await this.write(config); + // The assistant person properties track the config. + if (patch.assistantModel !== undefined) { + void syncModelProviderPersonProperties(); + } } private async read(): Promise { diff --git a/apps/x/packages/core/src/models/rowboat-selection.ts b/apps/x/packages/core/src/models/rowboat-selection.ts index 7e3c94e0..7a6bc1dc 100644 --- a/apps/x/packages/core/src/models/rowboat-selection.ts +++ b/apps/x/packages/core/src/models/rowboat-selection.ts @@ -3,6 +3,7 @@ import { IModelConfigRepo } from "./repo.js"; import { listGatewayModels } from "./gateway.js"; import { getRowboatConfig } from "../config/rowboat.js"; import { selectInitialModel } from "./initial-selection.js"; +import { capture } from "../analytics/posthog.js"; /** * Model-selection hooks for the Rowboat sign-in lifecycle. Signing in is @@ -28,6 +29,14 @@ export async function applyRowboatInitialSelection(): Promise { const model = selectInitialModel("rowboat", ids, recommendations); if (model) { await repo.updateConfig({ assistantModel: { provider: "rowboat", model } }); + // Measures recommendation quality: hit = the backend's pick was + // in the gateway list; miss = first-listed fallback. + capture("llm_initial_model_selected", { + flavor: "rowboat", + model, + recommended: model === recommendations?.["rowboat"], + source: "sign_in", + }); } } catch (error) { // Best-effort: a failed initial selection must never break sign-in.