mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-07-12 21:02:17 +02:00
feat(x): llm_usage analytics for turn model calls + voice-output parity
Analytics: the new runtime emitted usage into durable events but never reported it to PostHog (only the permission classifier self-reported). New IUsageReporter seam on the turn loop — invoked once per completed model call, after the durable append, failure-isolated. The real bridge emits the same llm_usage event as the old loop; useCase/subUseCase come from the AsyncLocalStorage context the stage-6 headless runners already establish via withUseCase, defaulting to copilot_chat for UI-driven session turns (matching the old createRun default). Capturing at the loop covers ALL turns — the session bus never sees headless ones. Voice: the new renderer path rendered <voice> tags literally and never fired TTS. The live overlay now extracts completed <voice> blocks (robust to tags split across deltas) into chatState.voiceSegments; App speaks new segments via the existing useVoiceTTS when voice output is on, skipping segments streamed before a session became active. Tags are stripped from both the streaming message and persisted assistant messages, mirroring the legacy display-time strip. Tests: usage report assertion in the runtime suite; four voice tests (split-delta extraction, per-call scan reset, state stripping+segments, persisted-message stripping). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
49e1e99d16
commit
fa6943e905
9 changed files with 212 additions and 9 deletions
|
|
@ -980,6 +980,28 @@ function App() {
|
|||
const ttsRef = useRef(tts)
|
||||
ttsRef.current = tts
|
||||
|
||||
// Speak newly completed <voice> blocks from the new runtime's live stream
|
||||
// (parity with the legacy text-delta voice extraction below). The store
|
||||
// accumulates completed blocks in chatState.voiceSegments; we speak only
|
||||
// segments that appeared after the current session became active.
|
||||
const spokenVoiceRef = useRef<{ key: string | null; count: number }>({ key: null, count: 0 })
|
||||
const voiceSegments = sessionChat.chatState?.voiceSegments
|
||||
useEffect(() => {
|
||||
if (!voiceSegments) return
|
||||
if (spokenVoiceRef.current.key !== runId) {
|
||||
// Session switch: skip anything already streamed before we arrived.
|
||||
spokenVoiceRef.current = { key: runId, count: voiceSegments.length }
|
||||
return
|
||||
}
|
||||
while (spokenVoiceRef.current.count < voiceSegments.length) {
|
||||
const segment = voiceSegments[spokenVoiceRef.current.count]
|
||||
spokenVoiceRef.current.count += 1
|
||||
if (ttsEnabledRef.current) {
|
||||
ttsRef.current.speak(segment)
|
||||
}
|
||||
}
|
||||
}, [voiceSegments, runId])
|
||||
|
||||
const voice = useVoiceMode()
|
||||
const voiceRef = useRef(voice)
|
||||
voiceRef.current = voice
|
||||
|
|
|
|||
|
|
@ -82,6 +82,49 @@ describe('applyOverlay', () => {
|
|||
})
|
||||
})
|
||||
|
||||
describe('voice output', () => {
|
||||
const delta = (d: string): Parameters<typeof applyOverlay>[1] =>
|
||||
({ type: 'text_delta', turnId: T1, modelCallIndex: 0, delta: d })
|
||||
|
||||
it('extracts completed <voice> blocks across split deltas', () => {
|
||||
let overlay = emptyOverlay()
|
||||
overlay = applyOverlay(overlay, delta('Sure! <voi'))
|
||||
expect(overlay.voiceSegments).toEqual([])
|
||||
overlay = applyOverlay(overlay, delta('ce>hello there</voice> and <voice>bye'))
|
||||
expect(overlay.voiceSegments).toEqual(['hello there'])
|
||||
overlay = applyOverlay(overlay, delta('</voice> done'))
|
||||
expect(overlay.voiceSegments).toEqual(['hello there', 'bye'])
|
||||
})
|
||||
|
||||
it('keeps segments but resets the scan on model_call_completed', () => {
|
||||
let overlay = emptyOverlay()
|
||||
overlay = applyOverlay(overlay, delta('<voice>one</voice>'))
|
||||
overlay = applyOverlay(overlay, completed(T1, 0, assistantText('<voice>one</voice>')))
|
||||
expect(overlay.voiceSegments).toEqual(['one'])
|
||||
expect(overlay.text).toBe('')
|
||||
overlay = applyOverlay(overlay, delta('<voice>two</voice>'))
|
||||
expect(overlay.voiceSegments).toEqual(['one', 'two'])
|
||||
})
|
||||
|
||||
it('strips voice tags from the streaming message and exposes segments on state', () => {
|
||||
const turn = reduceTurn([created(T1, S1), requested(T1, 0)])
|
||||
let overlay = emptyOverlay()
|
||||
overlay = applyOverlay(overlay, delta('Plan: <voice>speak this</voice> rest'))
|
||||
const state = buildSessionChatState([turn], overlay)
|
||||
expect(state.currentAssistantMessage).toBe('Plan: speak this rest')
|
||||
expect(state.voiceSegments).toEqual(['speak this'])
|
||||
})
|
||||
|
||||
it('strips voice tags from persisted assistant messages', () => {
|
||||
const state = reduceTurn(
|
||||
completedTurnLog(T1, S1, 'q', 'Sure. <voice>Here you go.</voice> Done.'),
|
||||
)
|
||||
const items = buildTurnConversation(state)
|
||||
const assistant = items.find((i) => isChatMessage(i) && i.role === 'assistant')
|
||||
expect(isChatMessage(assistant!) && assistant.content).toBe('Sure. Here you go. Done.')
|
||||
})
|
||||
})
|
||||
|
||||
describe('buildTurnConversation', () => {
|
||||
it('maps user input, assistant text, and settled tool calls', () => {
|
||||
const call = assistantCalls(toolCallPart('tc1', 'echo', { x: 1 }))
|
||||
|
|
|
|||
|
|
@ -34,20 +34,62 @@ export type LiveOverlay = {
|
|||
text: string
|
||||
reasoning: string
|
||||
toolOutput: Record<string, string>
|
||||
// Contents of completed <voice>…</voice> blocks seen while streaming, in
|
||||
// order, monotonically growing for the lifetime of the overlay (i.e. one
|
||||
// active turn). Consumers speak segments beyond what they've already
|
||||
// spoken; the overlay reset on turn switch starts a fresh list.
|
||||
voiceSegments: string[]
|
||||
// Scan cursor into `text` — everything before it has been checked for
|
||||
// complete voice blocks.
|
||||
voiceScanIndex: number
|
||||
}
|
||||
|
||||
export const emptyOverlay = (): LiveOverlay => ({ text: '', reasoning: '', toolOutput: {} })
|
||||
export const emptyOverlay = (): LiveOverlay => ({
|
||||
text: '',
|
||||
reasoning: '',
|
||||
toolOutput: {},
|
||||
voiceSegments: [],
|
||||
voiceScanIndex: 0,
|
||||
})
|
||||
|
||||
// The model emits <voice>…</voice> around speakable text when voice output
|
||||
// is enabled; tags are never shown to the user.
|
||||
export function stripVoiceTags(text: string): string {
|
||||
return text.replace(/<\/?voice>/g, '')
|
||||
}
|
||||
|
||||
const VOICE_BLOCK = /<voice>([\s\S]*?)<\/voice>/g
|
||||
|
||||
// Accumulates deltas; canonical durable events supersede the buffers (the
|
||||
// committed transcript now contains what was streaming).
|
||||
export function applyOverlay(overlay: LiveOverlay, event: TurnStreamEvent): LiveOverlay {
|
||||
switch (event.type) {
|
||||
case 'text_delta':
|
||||
return { ...overlay, text: overlay.text + event.delta }
|
||||
case 'text_delta': {
|
||||
const text = overlay.text + event.delta
|
||||
// Extract complete voice blocks past the scan cursor. Incomplete
|
||||
// blocks (opening tag seen, closing not yet) stay unconsumed until a
|
||||
// later delta completes them.
|
||||
const segments: string[] = []
|
||||
let scanIndex = overlay.voiceScanIndex
|
||||
VOICE_BLOCK.lastIndex = scanIndex
|
||||
for (let m = VOICE_BLOCK.exec(text); m; m = VOICE_BLOCK.exec(text)) {
|
||||
const content = m[1].trim()
|
||||
if (content) segments.push(content)
|
||||
scanIndex = m.index + m[0].length
|
||||
}
|
||||
return {
|
||||
...overlay,
|
||||
text,
|
||||
...(segments.length > 0
|
||||
? { voiceSegments: [...overlay.voiceSegments, ...segments] }
|
||||
: {}),
|
||||
voiceScanIndex: scanIndex,
|
||||
}
|
||||
}
|
||||
case 'reasoning_delta':
|
||||
return { ...overlay, reasoning: overlay.reasoning + event.delta }
|
||||
case 'model_call_completed':
|
||||
return { ...overlay, text: '', reasoning: '' }
|
||||
return { ...overlay, text: '', reasoning: '', voiceScanIndex: 0 }
|
||||
case 'tool_progress': {
|
||||
const progress = event.progress
|
||||
if (
|
||||
|
|
@ -140,13 +182,16 @@ export function buildTurnConversation(state: TurnState): ConversationItem[] {
|
|||
for (const call of state.modelCalls) {
|
||||
if (call.response === undefined) continue
|
||||
const content = call.response.content
|
||||
const text =
|
||||
// Voice tags are model-facing markup, never shown (parity with the
|
||||
// legacy path's display-time strip).
|
||||
const text = stripVoiceTags(
|
||||
typeof content === 'string'
|
||||
? content
|
||||
: content
|
||||
.map((part) => (part.type === 'text' ? part.text : ''))
|
||||
.filter(Boolean)
|
||||
.join('\n')
|
||||
.join('\n'),
|
||||
)
|
||||
if (text) {
|
||||
items.push({
|
||||
id: `${turnId}:a${call.index}`,
|
||||
|
|
@ -192,6 +237,8 @@ type PermMeta = z.infer<typeof ToolPermissionMetadata>
|
|||
export type SessionChatState = {
|
||||
conversation: ConversationItem[]
|
||||
currentAssistantMessage: string
|
||||
// See LiveOverlay.voiceSegments.
|
||||
voiceSegments: string[]
|
||||
pendingAskHumanRequests: Map<string, z.infer<typeof AskHumanRequestEvent>>
|
||||
allPermissionRequests: Map<string, z.infer<typeof ToolPermissionRequestEvent>>
|
||||
permissionResponses: Map<string, PermissionResponse>
|
||||
|
|
@ -289,7 +336,8 @@ export function buildSessionChatState(
|
|||
const settled = status === 'completed' || status === 'failed' || status === 'cancelled'
|
||||
return {
|
||||
conversation,
|
||||
currentAssistantMessage: overlay.text,
|
||||
currentAssistantMessage: stripVoiceTags(overlay.text),
|
||||
voiceSegments: overlay.voiceSegments,
|
||||
pendingAskHumanRequests,
|
||||
allPermissionRequests,
|
||||
permissionResponses,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue