diff --git a/apps/x/ANALYTICS.md b/apps/x/ANALYTICS.md index becf99e2..b58d5f25 100644 --- a/apps/x/ANALYTICS.md +++ b/apps/x/ANALYTICS.md @@ -93,6 +93,7 @@ All in `apps/renderer/src/lib/analytics.ts`: - `oauth_connected` / `oauth_disconnected` — `{ provider }` - `voice_input_started` — no properties - `call_started` — `{ preset: 'voice' | 'video' | 'share' | 'practice' }` — a hands-free call began (see `apps/x/VIDEO_MODE.md`) +- `call_turn_latency` — `{ endpoint_to_submit_ms, submit_to_speak_ms, speak_to_audio_ms, total_ms }` — voice-to-voice latency breakdown for one call turn (utterance accepted → submitted → first TTS speak → audio playing) - `search_executed` — `{ types: string[] }` - `note_exported` — `{ format }` diff --git a/apps/x/VIDEO_MODE.md b/apps/x/VIDEO_MODE.md index b2dc058d..b9cfc673 100644 --- a/apps/x/VIDEO_MODE.md +++ b/apps/x/VIDEO_MODE.md @@ -167,6 +167,28 @@ Voice input/output prompt sections (`# Voice Input`, `# Voice Output`) are reused untouched — calls set `voiceInput` per utterance and force `voiceOutput: 'full'`. +## Latency + +Voice-to-voice latency (user stops talking → assistant audio) is engineered +at four points; the `call_turn_latency` PostHog event measures the real +distribution (utterance → submit → first speak → audio playing): + +- **Smart endpointing** (`useVoiceMode.ts`): Deepgram endpoints at 600ms and + the client decides — a transcript ending in terminal punctuation fires + immediately (~600ms after last word); a mid-thought trail holds another + 1.2s (resumed speech cancels the hold). Complete sentences turn around + ~1.2s faster than the old fixed 1800ms endpoint. +- **Streaming TTS** (`voice:synthesizeStreamStart` → `voice:tts-chunk` → + MediaSource playback in `useVoiceTTS.ts`): the first segment of an idle + queue plays from the first MP3 chunk instead of after the full body + (ElevenLabs `/stream`, flash model). Follow-up segments keep the gapless + full-body prefetch path. Falls back to non-streaming on any failure. +- **Early clause speech** (`turn-view.ts` `applyOverlay`): a still-open + `` block ≥60 chars emits its last complete clause immediately, so + speech starts while the rest of the sentence generates. +- **Acknowledgment cue** (`lib/call-sounds.ts`): a soft blip the instant an + utterance is accepted — perceived latency matters as much as measured. + ## Cost notes Webcam frames ≈ 250–350 tokens each (≤12/message ≈ 3–4k); screen frames ≈ diff --git a/apps/x/apps/main/src/ipc.ts b/apps/x/apps/main/src/ipc.ts index 9df109e7..dde72aa1 100644 --- a/apps/x/apps/main/src/ipc.ts +++ b/apps/x/apps/main/src/ipc.ts @@ -382,6 +382,9 @@ type InvokeHandlers = { [K in InvokeChannels]: InvokeHandler; }; +// In-flight streaming TTS requests, keyed by renderer-chosen requestId. +const activeTtsStreams = new Map(); + // Video-mode popout window (shown for the whole duration of a screen share, // floating over every app including Rowboat itself) and the last call state // pushed by the main window — replayed to the popout when it finishes loading. @@ -1720,6 +1723,51 @@ export function setupIpcHandlers() { 'voice:synthesize': async (_event, args) => { return voice.synthesizeSpeech(args.text); }, + 'voice:synthesizeStreamStart': async (event, args) => { + const { requestId, text } = args; + const sender = event.sender; + const controller = new AbortController(); + activeTtsStreams.set(requestId, controller); + // Fire-and-forget: chunks are pushed to the renderer as they arrive so + // playback can begin immediately; the invoke returns once started. + void voice + .synthesizeSpeechStream( + text, + (chunk) => { + if (!sender.isDestroyed()) { + sender.send('voice:tts-chunk', { + requestId, + chunkBase64: chunk.toString('base64'), + done: false, + }); + } + }, + controller.signal, + ) + .then(() => { + if (!sender.isDestroyed()) { + sender.send('voice:tts-chunk', { requestId, done: true }); + } + }) + .catch((err: unknown) => { + if (!sender.isDestroyed() && !controller.signal.aborted) { + sender.send('voice:tts-chunk', { + requestId, + done: true, + error: err instanceof Error ? err.message : String(err), + }); + } + }) + .finally(() => { + activeTtsStreams.delete(requestId); + }); + return { ok: true }; + }, + 'voice:synthesizeStreamCancel': async (_event, args) => { + activeTtsStreams.get(args.requestId)?.abort(); + activeTtsStreams.delete(args.requestId); + return {}; + }, 'voice:ensureMicAccess': async () => { if (process.platform !== 'darwin') return { granted: true }; const status = systemPreferences.getMediaAccessStatus('microphone'); diff --git a/apps/x/apps/renderer/src/App.tsx b/apps/x/apps/renderer/src/App.tsx index 880b9816..4de60e33 100644 --- a/apps/x/apps/renderer/src/App.tsx +++ b/apps/x/apps/renderer/src/App.tsx @@ -125,6 +125,7 @@ import { ProductTour, type TourNavTarget } from '@/components/product-tour' import { useMeetingTranscription, type CalendarEventMeta } from '@/hooks/useMeetingTranscription' import { useAnalyticsIdentity } from '@/hooks/useAnalyticsIdentity' import * as analytics from '@/lib/analytics' +import { playAckCue } from '@/lib/call-sounds' import { useTheme } from '@/contexts/theme-context' type DirEntry = z.infer @@ -973,6 +974,10 @@ function App() { // TTS plays only during calls now (the standing read-aloud toggle was // retired; a per-message "read aloud" action may replace it later). const ttsEnabledRef = useRef(false) + // Voice-to-voice latency marks for the current call turn (performance.now): + // t0 = utterance accepted, submit = message sent, speak = first TTS + // speak(). Emitted as call_turn_latency when audio actually starts. + const callTurnMarksRef = useRef<{ t0: number; submit?: number; speak?: number } | null>(null) // Read-aloud style: 'summary' for typed chat, forced to 'full' during a // call and restored after. Context decides — the user never picks it. const ttsModeRef = useRef<'summary' | 'full'>('summary') @@ -1010,12 +1015,29 @@ function App() { const segment = voiceSegments[spokenVoiceRef.current.count] spokenVoiceRef.current.count += 1 if (ttsEnabledRef.current) { + const marks = callTurnMarksRef.current + if (marks && marks.speak === undefined) marks.speak = performance.now() ttsRef.current.speak(segment) setAssistantCaption(segment) } } }, [voiceSegments, runId]) + // Emit the turn's voice-to-voice latency breakdown once audio is audible. + useEffect(() => { + if (tts.state !== 'speaking') return + const marks = callTurnMarksRef.current + if (!marks || marks.submit === undefined || marks.speak === undefined) return + callTurnMarksRef.current = null + const now = performance.now() + analytics.callTurnLatency({ + endpointToSubmitMs: marks.submit - marks.t0, + submitToSpeakMs: marks.speak - marks.submit, + speakToAudioMs: now - marks.speak, + totalMs: now - marks.t0, + }) + }, [tts.state]) + const voice = useVoiceMode() const voiceRef = useRef(voice) voiceRef.current = voice @@ -1160,6 +1182,9 @@ function App() { ttsEnabledRef.current = true ttsModeRef.current = 'full' void voiceRef.current.startContinuous((text) => { + // Instant "heard you" feedback + start of the latency clock. + playAckCue() + callTurnMarksRef.current = { t0: performance.now() } pendingVoiceInputRef.current = true handlePromptSubmitRef.current?.({ text, files: [] }) }) @@ -1179,6 +1204,7 @@ function App() { ttsEnabledRef.current = false ttsModeRef.current = 'summary' ttsRef.current.cancel() + callTurnMarksRef.current = null video.stop() setPracticeMode(false) practiceModeRef.current = false @@ -2781,6 +2807,11 @@ function App() { // Video chat mode: drain the webcam frames buffered since the last send // so they ride along with this message as inline image parts. + const marks = callTurnMarksRef.current + if (inCallRef.current && marks && marks.submit === undefined) { + marks.submit = performance.now() + } + const videoFrames = inCallRef.current ? video.collectFrames() : [] const userMessageId = `user-${Date.now()}` diff --git a/apps/x/apps/renderer/src/hooks/useVoiceMode.ts b/apps/x/apps/renderer/src/hooks/useVoiceMode.ts index 61f98f74..d8c49098 100644 --- a/apps/x/apps/renderer/src/hooks/useVoiceMode.ts +++ b/apps/x/apps/renderer/src/hooks/useVoiceMode.ts @@ -19,14 +19,24 @@ const DEEPGRAM_PARAMS = new URLSearchParams({ endpointing: '100', no_delay: 'true', }); -// Hands-free (continuous) mode uses a much longer endpoint than push-to-talk: -// speech_final fires only after this much silence, so a thinking pause doesn't -// cut the user off mid-sentence during a call. -const CONTINUOUS_ENDPOINTING_MS = 1800; +// Hands-free (continuous) mode: Deepgram's endpoint fires FAST (600ms of +// silence) and we apply smart hold logic on our side — if the transcript +// already reads as a complete thought (terminal punctuation) the utterance +// fires immediately, otherwise we hold INCOMPLETE_HOLD_MS longer in case the +// user was mid-thought. Net effect: complete sentences turn around ~1.2s +// faster than the old fixed 1800ms endpoint, while thinking pauses still get +// the same total grace (~1.8s). +const CONTINUOUS_ENDPOINTING_MS = 600; +const INCOMPLETE_HOLD_MS = 1200; // While the mic is paused (assistant speaking), keep the idle Deepgram socket // alive — it closes after ~10s without audio otherwise. const KEEPALIVE_INTERVAL_MS = 5000; +// Deepgram punctuates finals (punctuate=true) — a transcript ending in +// terminal punctuation (optionally inside a closing quote/paren) is treated +// as a complete thought. +const COMPLETE_THOUGHT_RE = /[.!?…]["')\]]*\s*$/; + function deepgramParams(continuous: boolean): URLSearchParams { if (!continuous) return DEEPGRAM_PARAMS; const params = new URLSearchParams(DEEPGRAM_PARAMS); @@ -35,7 +45,7 @@ function deepgramParams(continuous: boolean): URLSearchParams { // on a result with an empty transcript, or never fires when background // noise keeps the endpointer engaged). UtteranceEnd is word-timing based // and arrives as its own message type, so we listen for both. - params.set('utterance_end_ms', '2000'); + params.set('utterance_end_ms', '1000'); return params; } @@ -76,6 +86,8 @@ export function useVoiceMode() { // While true (assistant is speaking), mic audio is dropped instead of streamed. const pausedRef = useRef(false); const keepAliveTimerRef = useRef | null>(null); + // Pending mid-thought hold (smart endpointing) — see maybeEndUtterance. + const holdTimerRef = useRef | null>(null); // Refresh cached auth details (called on warmup, not on mic click) const refreshAuth = useCallback(async () => { @@ -95,10 +107,13 @@ export function useVoiceMode() { }, [refreshRowboatAccount]); // Hands-free mode: flush the accumulated utterance to the callback. - // Called on either end-of-speech signal (speech_final or UtteranceEnd); - // both may fire for the same utterance — the second finds an empty - // buffer and is a no-op. + // Both end-of-speech signals may fire for the same utterance — the second + // finds an empty buffer and is a no-op. const fireContinuousUtterance = useCallback(() => { + if (holdTimerRef.current) { + clearTimeout(holdTimerRef.current); + holdTimerRef.current = null; + } if (!continuousCbRef.current || pausedRef.current) return; const utterance = transcriptBufferRef.current.trim(); transcriptBufferRef.current = ''; @@ -107,6 +122,25 @@ export function useVoiceMode() { if (utterance) continuousCbRef.current(utterance); }, []); + // Smart endpoint: Deepgram's endpoint fires fast (600ms). If the + // transcript reads as a complete thought, hand it off immediately; if it + // trails off mid-sentence ("so what I want is…"), hold a little longer — + // resumed speech cancels the hold and the utterance keeps growing. + const maybeEndUtterance = useCallback(() => { + if (!continuousCbRef.current || pausedRef.current) return; + const buffered = transcriptBufferRef.current.trim(); + if (!buffered) return; + if (COMPLETE_THOUGHT_RE.test(buffered)) { + fireContinuousUtterance(); + return; + } + if (holdTimerRef.current) clearTimeout(holdTimerRef.current); + holdTimerRef.current = setTimeout(() => { + holdTimerRef.current = null; + fireContinuousUtterance(); + }, INCOMPLETE_HOLD_MS); + }, [fireContinuousUtterance]); + // Create and connect a Deepgram WebSocket using cached auth. // Starts the connection and returns immediately (does not wait for open). const connectWs = useCallback(async (continuous = false) => { @@ -143,13 +177,20 @@ export function useVoiceMode() { // Hands-free mode: word-timing based end-of-speech marker. if (data.type === 'UtteranceEnd') { - fireContinuousUtterance(); + maybeEndUtterance(); return; } if (!data.channel?.alternatives?.[0]) return; const transcript = data.channel.alternatives[0].transcript; + // The user resumed speaking — cancel any pending mid-thought hold + // so the utterance keeps growing instead of firing under them. + if (transcript && holdTimerRef.current) { + clearTimeout(holdTimerRef.current); + holdTimerRef.current = null; + } + if (data.is_final) { // NOTE: the endpoint marker (speech_final) usually arrives on a // result whose transcript is EMPTY — the silence after the user @@ -159,10 +200,11 @@ export function useVoiceMode() { transcriptBufferRef.current += (transcriptBufferRef.current ? ' ' : '') + transcript; interimRef.current = ''; } - // Hands-free mode: an endpoint completes the utterance — hand - // it off and reset for the next one. + // Hands-free mode: an endpoint may complete the utterance — + // immediately for complete thoughts, after a short hold for + // mid-sentence trails. if (continuousCbRef.current && data.speech_final) { - fireContinuousUtterance(); + maybeEndUtterance(); return; } if (transcript) { @@ -193,7 +235,7 @@ export function useVoiceMode() { }, 1000); } }; - }, [refreshAuth, fireContinuousUtterance]); + }, [refreshAuth, maybeEndUtterance]); const waitForWsOpen = useCallback(async (timeoutMs = 1500): Promise => { const ws = wsRef.current; @@ -258,6 +300,10 @@ export function useVoiceMode() { } continuousCbRef.current = null; pausedRef.current = false; + if (holdTimerRef.current) { + clearTimeout(holdTimerRef.current); + holdTimerRef.current = null; + } if (keepAliveTimerRef.current) { clearInterval(keepAliveTimerRef.current); keepAliveTimerRef.current = null; @@ -415,6 +461,10 @@ export function useVoiceMode() { if (pausedRef.current === paused) return; pausedRef.current = paused; if (paused) { + if (holdTimerRef.current) { + clearTimeout(holdTimerRef.current); + holdTimerRef.current = null; + } transcriptBufferRef.current = ''; interimRef.current = ''; setInterimText(''); diff --git a/apps/x/apps/renderer/src/hooks/useVoiceTTS.ts b/apps/x/apps/renderer/src/hooks/useVoiceTTS.ts index 4e81a2c5..bfdcc4ae 100644 --- a/apps/x/apps/renderer/src/hooks/useVoiceTTS.ts +++ b/apps/x/apps/renderer/src/hooks/useVoiceTTS.ts @@ -47,6 +47,8 @@ function playAudio( /** A queue entry: text to synthesize, or a ready-to-play audio URL (e.g. a bundled clip). */ type QueueItem = { text: string } | { url: string }; +type TtsChunkMsg = { requestId: string; chunkBase64?: string; done: boolean; error?: string }; + export function useVoiceTTS() { const [state, setState] = useState('idle'); const audioRef = useRef(null); @@ -54,6 +56,10 @@ export function useVoiceTTS() { const processingRef = useRef(false); // Pre-fetched audio ready to play immediately const prefetchedRef = useRef | null>(null); + // Streaming synthesis: per-request chunk handlers + the in-flight request + // id (so cancel() can abort the main-process fetch). + const streamHandlersRef = useRef void>>(new Map()); + const activeStreamIdRef = useRef(null); // Bumped by cancel(). A queue loop that awaited across a cancel sees a // stale generation and exits instead of playing audio that was cancelled // while still synthesizing (which would overlap the next utterance). @@ -107,6 +113,127 @@ export function useVoiceTTS() { analyserRef.current = null; }, []); + // Route streaming TTS chunks to whichever request is waiting for them. + useEffect(() => { + return window.ipc.on('voice:tts-chunk', (msg) => { + streamHandlersRef.current.get(msg.requestId)?.(msg); + }); + }, []); + + /** + * Streaming synthesis + playback via MediaSource: audio starts on the + * first chunk instead of after the full body. Rejects (for caller + * fallback to non-streaming synth) if the stream fails before any audio + * arrived; resolves when playback finishes. + */ + const streamSynthesizeAndPlay = useCallback((text: string, onStarted: () => void): Promise => { + return new Promise((resolve, reject) => { + if (typeof MediaSource === 'undefined' || !MediaSource.isTypeSupported('audio/mpeg')) { + reject(new Error('MSE audio/mpeg unsupported')); + return; + } + const requestId = `tts-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; + const mediaSource = new MediaSource(); + const audio = new Audio(); + audio.src = URL.createObjectURL(mediaSource); + audioRef.current = audio; + connectAnalyser(audio); + activeStreamIdRef.current = requestId; + + let sourceBuffer: SourceBuffer | null = null; + const pending: Uint8Array[] = []; + let streamDone = false; + let gotAudio = false; + let settled = false; + + const cleanup = () => { + streamHandlersRef.current.delete(requestId); + if (activeStreamIdRef.current === requestId) activeStreamIdRef.current = null; + URL.revokeObjectURL(audio.src); + }; + const finish = (err?: Error) => { + if (settled) return; + settled = true; + cleanup(); + if (err) reject(err); + else resolve(); + }; + + // Drain pending chunks into the SourceBuffer one at a time + // (appendBuffer is async; only one append may be in flight). + const pump = () => { + if (!sourceBuffer || sourceBuffer.updating || settled) return; + const chunk = pending.shift(); + if (chunk) { + try { + sourceBuffer.appendBuffer(chunk as BufferSource); + } catch (e) { + finish(e as Error); + } + return; + } + if (streamDone && mediaSource.readyState === 'open') { + try { + mediaSource.endOfStream(); + } catch { /* already ended */ } + } + }; + + mediaSource.addEventListener('sourceopen', () => { + try { + sourceBuffer = mediaSource.addSourceBuffer('audio/mpeg'); + } catch (e) { + finish(e as Error); + return; + } + sourceBuffer.addEventListener('updateend', pump); + pump(); + }, { once: true }); + + streamHandlersRef.current.set(requestId, (msg) => { + if (msg.error && !gotAudio) { + streamDone = true; + finish(new Error(msg.error)); + return; + } + if (msg.chunkBase64) { + gotAudio = true; + const bin = atob(msg.chunkBase64); + const bytes = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); + pending.push(bytes); + pump(); + } + if (msg.done) { + streamDone = true; + pump(); + } + }); + + audio.addEventListener('playing', () => onStarted(), { once: true }); + audio.onended = () => finish(); + // pause() (from cancel) must settle this promise too; natural end + // also fires 'pause' just before 'ended'; double-settle is a no-op. + audio.onpause = () => finish(); + audio.onerror = () => finish(new Error('stream playback failed')); + + window.ipc + .invoke('voice:synthesizeStreamStart', { requestId, text }) + .then((res) => { + if (!res.ok) finish(new Error(res.error || 'stream start failed')); + }) + .catch((e) => finish(e as Error)); + + // Starts as soon as the first appended data is decodable. + audio.play().catch(() => { /* surfaced via onerror / chunk error */ }); + + // Nothing arrived at all — bail so the caller can fall back. + setTimeout(() => { + if (!gotAudio && !settled) finish(new Error('stream timeout')); + }, 10_000); + }); + }, [connectAnalyser]); + const getLevel = useCallback((): number => { const analyser = analyserRef.current; if (!analyser) return 0; @@ -130,10 +257,41 @@ export function useVoiceTTS() { processingRef.current = true; const gen = generationRef.current; + // Kick off full-body pre-fetch for the next queued text while the + // current one plays — keeps sentence-to-sentence playback gapless. + const prefetchNext = () => { + const next = queueRef.current[0]; + if (next && 'text' in next && next.text.trim() && !prefetchedRef.current) { + console.log('[tts] pre-fetching next:', next.text.substring(0, 80)); + prefetchedRef.current = synthesize(next.text); + } + }; + while (queueRef.current.length > 0) { const item = queueRef.current.shift()!; if ('text' in item && !item.text.trim()) continue; + // Cold start (nothing playing, nothing pre-fetched): stream the + // synthesis so audio begins on the first chunk instead of after + // the full body — this is where first-response latency lives. + if ('text' in item && !prefetchedRef.current) { + setState('synthesizing'); + console.log('[tts] stream-synthesizing:', item.text.substring(0, 80)); + try { + await streamSynthesizeAndPlay(item.text, () => { + if (generationRef.current !== gen) return; + setState('speaking'); + prefetchNext(); + }); + if (generationRef.current !== gen) return; + continue; + } catch (err) { + if (generationRef.current !== gen) return; + console.error('[tts] stream failed, falling back to full synth:', err); + // fall through to the non-streaming path below + } + } + try { // Pre-recorded URL plays as-is; text uses the pre-fetched // result if available, otherwise synthesizes now. @@ -156,12 +314,7 @@ export function useVoiceTTS() { if (generationRef.current !== gen) return; setState('speaking'); - // Kick off pre-fetch for next chunk while this one plays - const next = queueRef.current[0]; - if (next && 'text' in next && next.text.trim()) { - console.log('[tts] pre-fetching next:', next.text.substring(0, 80)); - prefetchedRef.current = synthesize(next.text); - } + prefetchNext(); await playAudio(audio.dataUrl, audioRef, connectAnalyser); if (generationRef.current !== gen) return; @@ -176,7 +329,7 @@ export function useVoiceTTS() { prefetchedRef.current = null; processingRef.current = false; setState('idle'); - }, [connectAnalyser]); + }, [connectAnalyser, streamSynthesizeAndPlay]); const speak = useCallback((text: string) => { console.log('[tts] speak() called:', text.substring(0, 80)); @@ -196,6 +349,13 @@ export function useVoiceTTS() { generationRef.current++; queueRef.current = []; prefetchedRef.current = null; + // Abort any in-flight streaming synthesis in the main process. + if (activeStreamIdRef.current) { + void window.ipc + .invoke('voice:synthesizeStreamCancel', { requestId: activeStreamIdRef.current }) + .catch(() => {}); + activeStreamIdRef.current = null; + } if (audioRef.current) { audioRef.current.pause(); audioRef.current = null; diff --git a/apps/x/apps/renderer/src/lib/analytics.ts b/apps/x/apps/renderer/src/lib/analytics.ts index 9173ebc3..3234d24b 100644 --- a/apps/x/apps/renderer/src/lib/analytics.ts +++ b/apps/x/apps/renderer/src/lib/analytics.ts @@ -69,6 +69,22 @@ export function callStarted(preset: 'voice' | 'video' | 'share' | 'practice') { posthog.capture('call_started', { preset }) } +// Voice-to-voice latency breakdown for one call turn (all milliseconds): +// utterance accepted → message submitted → first TTS speak() → audio playing. +export function callTurnLatency(props: { + endpointToSubmitMs: number + submitToSpeakMs: number + speakToAudioMs: number + totalMs: number +}) { + posthog.capture('call_turn_latency', { + endpoint_to_submit_ms: Math.round(props.endpointToSubmitMs), + submit_to_speak_ms: Math.round(props.submitToSpeakMs), + speak_to_audio_ms: Math.round(props.speakToAudioMs), + total_ms: Math.round(props.totalMs), + }) +} + export function searchExecuted(types: string[]) { posthog.capture('search_executed', { types }) } diff --git a/apps/x/apps/renderer/src/lib/call-sounds.ts b/apps/x/apps/renderer/src/lib/call-sounds.ts new file mode 100644 index 00000000..f7d20607 --- /dev/null +++ b/apps/x/apps/renderer/src/lib/call-sounds.ts @@ -0,0 +1,30 @@ +// Tiny synthesized UI sounds for calls — no audio assets, one lazy context. + +let ctx: AudioContext | null = null + +/** + * Soft rising blip played the instant an utterance is accepted — sub-second + * acknowledgment makes the (still ongoing) model turn feel responsive + * instead of dead air. + */ +export function playAckCue() { + try { + if (!ctx) ctx = new AudioContext() + if (ctx.state === 'suspended') void ctx.resume() + const t = ctx.currentTime + const osc = ctx.createOscillator() + const gain = ctx.createGain() + osc.type = 'sine' + osc.frequency.setValueAtTime(880, t) + osc.frequency.exponentialRampToValueAtTime(1320, t + 0.08) + gain.gain.setValueAtTime(0.0001, t) + gain.gain.exponentialRampToValueAtTime(0.08, t + 0.015) + gain.gain.exponentialRampToValueAtTime(0.0001, t + 0.12) + osc.connect(gain) + gain.connect(ctx.destination) + osc.start(t) + osc.stop(t + 0.13) + } catch { + // cosmetic — never let a sound failure affect the call + } +} diff --git a/apps/x/apps/renderer/src/lib/session-chat/turn-view.test.ts b/apps/x/apps/renderer/src/lib/session-chat/turn-view.test.ts index 66a781ce..342dc16f 100644 --- a/apps/x/apps/renderer/src/lib/session-chat/turn-view.test.ts +++ b/apps/x/apps/renderer/src/lib/session-chat/turn-view.test.ts @@ -96,6 +96,25 @@ describe('voice output', () => { expect(overlay.voiceSegments).toEqual(['hello there', 'bye']) }) + it('emits an early clause from a long open block, then the remainder on close', () => { + let overlay = emptyOverlay() + const longClause = 'Okay so the first thing I would look at here is the error message,' + overlay = applyOverlay(overlay, delta(`${longClause} because`)) + // Open block crossed the early-speech threshold at a clause boundary. + expect(overlay.voiceSegments).toEqual([longClause]) + overlay = applyOverlay(overlay, delta(' it tells you the root cause.')) + // Remainder only — the early clause is not repeated. + expect(overlay.voiceSegments).toEqual([longClause, 'because it tells you the root cause.']) + }) + + it('does not emit early clauses from short open blocks', () => { + let overlay = emptyOverlay() + overlay = applyOverlay(overlay, delta('Sure, one sec')) + expect(overlay.voiceSegments).toEqual([]) + overlay = applyOverlay(overlay, delta('.')) + expect(overlay.voiceSegments).toEqual(['Sure, one sec.']) + }) + it('keeps segments but resets the scan on model_call_completed', () => { let overlay = emptyOverlay() overlay = applyOverlay(overlay, delta('one')) diff --git a/apps/x/apps/renderer/src/lib/session-chat/turn-view.ts b/apps/x/apps/renderer/src/lib/session-chat/turn-view.ts index e4ca3cdd..0d5413c1 100644 --- a/apps/x/apps/renderer/src/lib/session-chat/turn-view.ts +++ b/apps/x/apps/renderer/src/lib/session-chat/turn-view.ts @@ -34,14 +34,20 @@ export type LiveOverlay = { text: string reasoning: string toolOutput: Record - // Contents of completed … blocks seen while streaming, in - // order, monotonically growing for the lifetime of the overlay (i.e. one - // active turn). Consumers speak segments beyond what they've already - // spoken; the overlay reset on turn switch starts a fresh list. + // Speakable segments seen while streaming, in order, monotonically growing + // for the lifetime of the overlay (i.e. one active turn). Usually the + // contents of completed … blocks, but a long still-open + // block may emit an early clause (see EARLY_SPEECH_MIN_CHARS) so speech + // can start before the sentence finishes generating. Consumers speak + // segments beyond what they've already spoken; the overlay reset on turn + // switch starts a fresh list. voiceSegments: string[] // Scan cursor into `text` — everything before it has been checked for // complete voice blocks. voiceScanIndex: number + // Chars of the currently-open voice block's content already emitted as an + // early clause — the block's remainder (on close) excludes them. + voicePartialConsumed: number } export const emptyOverlay = (): LiveOverlay => ({ @@ -50,6 +56,7 @@ export const emptyOverlay = (): LiveOverlay => ({ toolOutput: {}, voiceSegments: [], voiceScanIndex: 0, + voicePartialConsumed: 0, }) // The model emits … around speakable text when voice output @@ -59,6 +66,17 @@ export function stripVoiceTags(text: string): string { } const VOICE_BLOCK = /([\s\S]*?)<\/voice>/g +const VOICE_OPEN_TAG = '' + +// Early speech: once an open block has this many unconsumed chars, its last +// complete clause is emitted immediately instead of waiting for — +// TTS starts on the first clause while the rest of the sentence generates. +const EARLY_SPEECH_MIN_CHARS = 60 +// ...but never emit a fragment shorter than this (prosody suffers). +const EARLY_SPEECH_MIN_EMIT = 30 +// Clause boundaries (punctuation, optionally inside closing quote/paren, +// followed by whitespace or end-of-buffer). +const CLAUSE_BOUNDARY = /[,;:.!?…—]["')\]]*(?=\s|$)/g // Accumulates deltas; canonical durable events supersede the buffers (the // committed transcript now contains what was streaming). @@ -68,15 +86,43 @@ export function applyOverlay(overlay: LiveOverlay, event: TurnStreamEvent): Live const text = overlay.text + event.delta // Extract complete voice blocks past the scan cursor. Incomplete // blocks (opening tag seen, closing not yet) stay unconsumed until a - // later delta completes them. + // later delta completes them. The first complete block may have had an + // early clause emitted while it was open — skip those chars. const segments: string[] = [] let scanIndex = overlay.voiceScanIndex + let partialConsumed = overlay.voicePartialConsumed VOICE_BLOCK.lastIndex = scanIndex for (let m = VOICE_BLOCK.exec(text); m; m = VOICE_BLOCK.exec(text)) { - const content = m[1].trim() + const content = m[1].slice(partialConsumed).trim() + partialConsumed = 0 if (content) segments.push(content) scanIndex = m.index + m[0].length } + + // Early speech: if a voice block is still open and has accumulated a + // long unconsumed run, emit its last complete clause now — speech can + // start while the rest of the sentence is still generating. + const openIdx = text.indexOf(VOICE_OPEN_TAG, scanIndex) + if (openIdx !== -1) { + const unconsumed = text.slice(openIdx + VOICE_OPEN_TAG.length + partialConsumed) + if (unconsumed.length >= EARLY_SPEECH_MIN_CHARS) { + let lastBoundaryEnd = -1 + CLAUSE_BOUNDARY.lastIndex = 0 + for (let b = CLAUSE_BOUNDARY.exec(unconsumed); b; b = CLAUSE_BOUNDARY.exec(unconsumed)) { + lastBoundaryEnd = b.index + b[0].length + } + if (lastBoundaryEnd >= EARLY_SPEECH_MIN_EMIT) { + const clause = unconsumed.slice(0, lastBoundaryEnd).trim() + if (clause) segments.push(clause) + partialConsumed += lastBoundaryEnd + } + } + } else { + // No open block — any partial bookkeeping belongs to a block that + // has since closed. + partialConsumed = 0 + } + return { ...overlay, text, @@ -84,12 +130,13 @@ export function applyOverlay(overlay: LiveOverlay, event: TurnStreamEvent): Live ? { voiceSegments: [...overlay.voiceSegments, ...segments] } : {}), voiceScanIndex: scanIndex, + voicePartialConsumed: partialConsumed, } } case 'reasoning_delta': return { ...overlay, reasoning: overlay.reasoning + event.delta } case 'model_call_completed': - return { ...overlay, text: '', reasoning: '', voiceScanIndex: 0 } + return { ...overlay, text: '', reasoning: '', voiceScanIndex: 0, voicePartialConsumed: 0 } case 'tool_progress': { const progress = event.progress if ( diff --git a/apps/x/packages/core/src/voice/voice.ts b/apps/x/packages/core/src/voice/voice.ts index fd5f3d2d..775918ed 100644 --- a/apps/x/packages/core/src/voice/voice.ts +++ b/apps/x/packages/core/src/voice/voice.ts @@ -32,46 +32,57 @@ export async function getVoiceConfig(): Promise { }; } -export async function synthesizeSpeech(text: string): Promise<{ audioBase64: string; mimeType: string }> { +async function resolveTtsEndpoint(streaming: boolean): Promise<{ url: string; headers: Record }> { const config = await getVoiceConfig(); const signedIn = await isSignedIn(); - let url: string; - let headers: Record; - if (signedIn) { const voiceId = config.elevenlabs?.voiceId || 's3TPKV1kjDlVtZbl4Ksh'; const accessToken = await getAccessToken(); - url = `${API_URL}/v1/voice/text-to-speech/${voiceId}`; - headers = { - 'Authorization': `Bearer ${accessToken}`, - 'Content-Type': 'application/json', + // The proxy has no dedicated /stream route — the same endpoint is + // used and the body is consumed progressively; if the proxy buffers, + // streaming degrades to today's full-body latency, never worse. + return { + url: `${API_URL}/v1/voice/text-to-speech/${voiceId}`, + headers: { + 'Authorization': `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, }; - console.log('[voice] synthesizing speech via Rowboat proxy, text length:', text.length, 'voiceId:', voiceId); - } else { - if (!config.elevenlabs) { - throw new Error(`ElevenLabs not configured. Create ${path.join(WorkDir, 'config', 'elevenlabs.json')} with { "apiKey": "" }`); - } - const voiceId = config.elevenlabs.voiceId || 's3TPKV1kjDlVtZbl4Ksh'; - url = `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`; - headers = { + } + + if (!config.elevenlabs) { + throw new Error(`ElevenLabs not configured. Create ${path.join(WorkDir, 'config', 'elevenlabs.json')} with { "apiKey": "" }`); + } + const voiceId = config.elevenlabs.voiceId || 's3TPKV1kjDlVtZbl4Ksh'; + return { + url: `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}${streaming ? '/stream' : ''}`, + headers: { 'xi-api-key': config.elevenlabs.apiKey, 'Content-Type': 'application/json', - }; - console.log('[voice] synthesizing speech via ElevenLabs, text length:', text.length, 'voiceId:', voiceId); - } + }, + }; +} + +function ttsRequestBody(text: string): string { + return JSON.stringify({ + text, + model_id: 'eleven_flash_v2_5', + voice_settings: { + stability: 0.5, + similarity_boost: 0.75, + }, + }); +} + +export async function synthesizeSpeech(text: string): Promise<{ audioBase64: string; mimeType: string }> { + const { url, headers } = await resolveTtsEndpoint(false); + console.log('[voice] synthesizing speech, text length:', text.length); const response = await fetch(url, { method: 'POST', headers, - body: JSON.stringify({ - text, - model_id: 'eleven_flash_v2_5', - voice_settings: { - stability: 0.5, - similarity_boost: 0.75, - }, - }), + body: ttsRequestBody(text), }); if (!response.ok) { @@ -85,3 +96,42 @@ export async function synthesizeSpeech(text: string): Promise<{ audioBase64: str console.log('[voice] synthesized audio, base64 length:', audioBase64.length); return { audioBase64, mimeType: 'audio/mpeg' }; } + +/** + * Streaming synthesis: invokes `onChunk` with MP3 bytes as they arrive so + * playback can start on the first chunk. Resolves when the stream ends; + * rejects on HTTP/stream errors. Abort via the provided signal. + */ +export async function synthesizeSpeechStream( + text: string, + onChunk: (chunk: Buffer) => void, + signal?: AbortSignal, +): Promise { + const { url, headers } = await resolveTtsEndpoint(true); + console.log('[voice] streaming speech synthesis, text length:', text.length); + + const response = await fetch(url, { + method: 'POST', + headers, + body: ttsRequestBody(text), + signal: signal ?? null, + }); + + if (!response.ok) { + const errText = await response.text().catch(() => 'Unknown error'); + console.error('[voice] TTS stream API error:', response.status, errText); + throw new Error(`TTS API error ${response.status}: ${errText}`); + } + if (!response.body) { + throw new Error('TTS API returned no body'); + } + + const reader = response.body.getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + if (value && value.byteLength > 0) { + onChunk(Buffer.from(value)); + } + } +} diff --git a/apps/x/packages/shared/src/ipc.ts b/apps/x/packages/shared/src/ipc.ts index 50bb0a62..e54df4e8 100644 --- a/apps/x/packages/shared/src/ipc.ts +++ b/apps/x/packages/shared/src/ipc.ts @@ -1362,6 +1362,34 @@ const ipcSchemas = { mimeType: z.string(), }), }, + // Streaming TTS: main starts the synthesis and pushes audio chunks over + // 'voice:tts-chunk' as they arrive, so playback can begin on the first + // chunk instead of after the full body (~0.5-1s earlier first-audio). + 'voice:synthesizeStreamStart': { + req: z.object({ + requestId: z.string(), + text: z.string(), + }), + res: z.object({ + ok: z.boolean(), + error: z.string().optional(), + }), + }, + 'voice:synthesizeStreamCancel': { + req: z.object({ requestId: z.string() }), + res: z.object({}), + }, + // Push channel: main → renderer with streaming TTS audio. `done: true` + // (possibly with a final chunk) ends the stream; `error` aborts it. + 'voice:tts-chunk': { + req: z.object({ + requestId: z.string(), + chunkBase64: z.string().optional(), + done: z.boolean(), + error: z.string().optional(), + }), + res: z.null(), + }, // Ensures the OS-level microphone permission is settled before capturing. // On first-ever use (macOS) the permission is 'not-determined'; resolving // the native prompt up front prevents the in-flight getUserMedia from