feat(x): llm_usage analytics for turn model calls + voice-output parity

Analytics: the new runtime emitted usage into durable events but never
reported it to PostHog (only the permission classifier self-reported).
New IUsageReporter seam on the turn loop — invoked once per completed
model call, after the durable append, failure-isolated. The real bridge
emits the same llm_usage event as the old loop; useCase/subUseCase come
from the AsyncLocalStorage context the stage-6 headless runners already
establish via withUseCase, defaulting to copilot_chat for UI-driven
session turns (matching the old createRun default). Capturing at the
loop covers ALL turns — the session bus never sees headless ones.

Voice: the new renderer path rendered <voice> tags literally and never
fired TTS. The live overlay now extracts completed <voice> blocks
(robust to tags split across deltas) into chatState.voiceSegments;
App speaks new segments via the existing useVoiceTTS when voice output
is on, skipping segments streamed before a session became active. Tags
are stripped from both the streaming message and persisted assistant
messages, mirroring the legacy display-time strip.

Tests: usage report assertion in the runtime suite; four voice tests
(split-delta extraction, per-call scan reset, state stripping+segments,
persisted-message stripping).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Ramnique Singh 2026-07-02 14:07:49 +05:30
parent 49e1e99d16
commit fa6943e905
9 changed files with 212 additions and 9 deletions

View file

@ -980,6 +980,28 @@ function App() {
const ttsRef = useRef(tts)
ttsRef.current = tts
// Speak newly completed <voice> blocks from the new runtime's live stream
// (parity with the legacy text-delta voice extraction below). The store
// accumulates completed blocks in chatState.voiceSegments; we speak only
// segments that appeared after the current session became active.
const spokenVoiceRef = useRef<{ key: string | null; count: number }>({ key: null, count: 0 })
const voiceSegments = sessionChat.chatState?.voiceSegments
useEffect(() => {
if (!voiceSegments) return
if (spokenVoiceRef.current.key !== runId) {
// Session switch: skip anything already streamed before we arrived.
spokenVoiceRef.current = { key: runId, count: voiceSegments.length }
return
}
while (spokenVoiceRef.current.count < voiceSegments.length) {
const segment = voiceSegments[spokenVoiceRef.current.count]
spokenVoiceRef.current.count += 1
if (ttsEnabledRef.current) {
ttsRef.current.speak(segment)
}
}
}, [voiceSegments, runId])
const voice = useVoiceMode()
const voiceRef = useRef(voice)
voiceRef.current = voice

View file

@ -82,6 +82,49 @@ describe('applyOverlay', () => {
})
})
describe('voice output', () => {
const delta = (d: string): Parameters<typeof applyOverlay>[1] =>
({ type: 'text_delta', turnId: T1, modelCallIndex: 0, delta: d })
it('extracts completed <voice> blocks across split deltas', () => {
let overlay = emptyOverlay()
overlay = applyOverlay(overlay, delta('Sure! <voi'))
expect(overlay.voiceSegments).toEqual([])
overlay = applyOverlay(overlay, delta('ce>hello there</voice> and <voice>bye'))
expect(overlay.voiceSegments).toEqual(['hello there'])
overlay = applyOverlay(overlay, delta('</voice> done'))
expect(overlay.voiceSegments).toEqual(['hello there', 'bye'])
})
it('keeps segments but resets the scan on model_call_completed', () => {
let overlay = emptyOverlay()
overlay = applyOverlay(overlay, delta('<voice>one</voice>'))
overlay = applyOverlay(overlay, completed(T1, 0, assistantText('<voice>one</voice>')))
expect(overlay.voiceSegments).toEqual(['one'])
expect(overlay.text).toBe('')
overlay = applyOverlay(overlay, delta('<voice>two</voice>'))
expect(overlay.voiceSegments).toEqual(['one', 'two'])
})
it('strips voice tags from the streaming message and exposes segments on state', () => {
const turn = reduceTurn([created(T1, S1), requested(T1, 0)])
let overlay = emptyOverlay()
overlay = applyOverlay(overlay, delta('Plan: <voice>speak this</voice> rest'))
const state = buildSessionChatState([turn], overlay)
expect(state.currentAssistantMessage).toBe('Plan: speak this rest')
expect(state.voiceSegments).toEqual(['speak this'])
})
it('strips voice tags from persisted assistant messages', () => {
const state = reduceTurn(
completedTurnLog(T1, S1, 'q', 'Sure. <voice>Here you go.</voice> Done.'),
)
const items = buildTurnConversation(state)
const assistant = items.find((i) => isChatMessage(i) && i.role === 'assistant')
expect(isChatMessage(assistant!) && assistant.content).toBe('Sure. Here you go. Done.')
})
})
describe('buildTurnConversation', () => {
it('maps user input, assistant text, and settled tool calls', () => {
const call = assistantCalls(toolCallPart('tc1', 'echo', { x: 1 }))

View file

@ -34,20 +34,62 @@ export type LiveOverlay = {
text: string
reasoning: string
toolOutput: Record<string, string>
// Contents of completed <voice>…</voice> blocks seen while streaming, in
// order, monotonically growing for the lifetime of the overlay (i.e. one
// active turn). Consumers speak segments beyond what they've already
// spoken; the overlay reset on turn switch starts a fresh list.
voiceSegments: string[]
// Scan cursor into `text` — everything before it has been checked for
// complete voice blocks.
voiceScanIndex: number
}
export const emptyOverlay = (): LiveOverlay => ({ text: '', reasoning: '', toolOutput: {} })
export const emptyOverlay = (): LiveOverlay => ({
text: '',
reasoning: '',
toolOutput: {},
voiceSegments: [],
voiceScanIndex: 0,
})
// The model emits <voice>…</voice> around speakable text when voice output
// is enabled; tags are never shown to the user.
export function stripVoiceTags(text: string): string {
return text.replace(/<\/?voice>/g, '')
}
const VOICE_BLOCK = /<voice>([\s\S]*?)<\/voice>/g
// Accumulates deltas; canonical durable events supersede the buffers (the
// committed transcript now contains what was streaming).
export function applyOverlay(overlay: LiveOverlay, event: TurnStreamEvent): LiveOverlay {
switch (event.type) {
case 'text_delta':
return { ...overlay, text: overlay.text + event.delta }
case 'text_delta': {
const text = overlay.text + event.delta
// Extract complete voice blocks past the scan cursor. Incomplete
// blocks (opening tag seen, closing not yet) stay unconsumed until a
// later delta completes them.
const segments: string[] = []
let scanIndex = overlay.voiceScanIndex
VOICE_BLOCK.lastIndex = scanIndex
for (let m = VOICE_BLOCK.exec(text); m; m = VOICE_BLOCK.exec(text)) {
const content = m[1].trim()
if (content) segments.push(content)
scanIndex = m.index + m[0].length
}
return {
...overlay,
text,
...(segments.length > 0
? { voiceSegments: [...overlay.voiceSegments, ...segments] }
: {}),
voiceScanIndex: scanIndex,
}
}
case 'reasoning_delta':
return { ...overlay, reasoning: overlay.reasoning + event.delta }
case 'model_call_completed':
return { ...overlay, text: '', reasoning: '' }
return { ...overlay, text: '', reasoning: '', voiceScanIndex: 0 }
case 'tool_progress': {
const progress = event.progress
if (
@ -140,13 +182,16 @@ export function buildTurnConversation(state: TurnState): ConversationItem[] {
for (const call of state.modelCalls) {
if (call.response === undefined) continue
const content = call.response.content
const text =
// Voice tags are model-facing markup, never shown (parity with the
// legacy path's display-time strip).
const text = stripVoiceTags(
typeof content === 'string'
? content
: content
.map((part) => (part.type === 'text' ? part.text : ''))
.filter(Boolean)
.join('\n')
.join('\n'),
)
if (text) {
items.push({
id: `${turnId}:a${call.index}`,
@ -192,6 +237,8 @@ type PermMeta = z.infer<typeof ToolPermissionMetadata>
export type SessionChatState = {
conversation: ConversationItem[]
currentAssistantMessage: string
// See LiveOverlay.voiceSegments.
voiceSegments: string[]
pendingAskHumanRequests: Map<string, z.infer<typeof AskHumanRequestEvent>>
allPermissionRequests: Map<string, z.infer<typeof ToolPermissionRequestEvent>>
permissionResponses: Map<string, PermissionResponse>
@ -289,7 +336,8 @@ export function buildSessionChatState(
const settled = status === 'completed' || status === 'failed' || status === 'cancelled'
return {
conversation,
currentAssistantMessage: overlay.text,
currentAssistantMessage: stripVoiceTags(overlay.text),
voiceSegments: overlay.voiceSegments,
pendingAskHumanRequests,
allPermissionRequests,
permissionResponses,