phase 1 improvements to reduce latency: smart endpointing, streaming tts, early clause speech, ack

apps/x/ANALYTICS.md

@@ -93,6 +93,7 @@ All in `apps/renderer/src/lib/analytics.ts`:
 - `oauth_connected` / `oauth_disconnected` — `{ provider }`
 - `voice_input_started` — no properties
 - `call_started` — `{ preset: 'voice' | 'video' | 'share' | 'practice' }` — a hands-free call began (see `apps/x/VIDEO_MODE.md`)
+- `call_turn_latency` — `{ endpoint_to_submit_ms, submit_to_speak_ms, speak_to_audio_ms, total_ms }` — voice-to-voice latency breakdown for one call turn (utterance accepted → submitted → first TTS speak → audio playing)
 - `search_executed` — `{ types: string[] }`
 - `note_exported` — `{ format }`
 

apps/x/VIDEO_MODE.md

@@ -167,6 +167,28 @@ Voice input/output prompt sections (`# Voice Input`, `# Voice Output`) are
 reused untouched — calls set `voiceInput` per utterance and force
 `voiceOutput: 'full'`.
 
+## Latency
+
+Voice-to-voice latency (user stops talking → assistant audio) is engineered
+at four points; the `call_turn_latency` PostHog event measures the real
+distribution (utterance → submit → first speak → audio playing):
+
+- **Smart endpointing** (`useVoiceMode.ts`): Deepgram endpoints at 600ms and
+  the client decides — a transcript ending in terminal punctuation fires
+  immediately (~600ms after last word); a mid-thought trail holds another
+  1.2s (resumed speech cancels the hold). Complete sentences turn around
+  ~1.2s faster than the old fixed 1800ms endpoint.
+- **Streaming TTS** (`voice:synthesizeStreamStart` → `voice:tts-chunk` →
+  MediaSource playback in `useVoiceTTS.ts`): the first segment of an idle
+  queue plays from the first MP3 chunk instead of after the full body
+  (ElevenLabs `/stream`, flash model). Follow-up segments keep the gapless
+  full-body prefetch path. Falls back to non-streaming on any failure.
+- **Early clause speech** (`turn-view.ts` `applyOverlay`): a still-open
+  `<voice>` block ≥60 chars emits its last complete clause immediately, so
+  speech starts while the rest of the sentence generates.
+- **Acknowledgment cue** (`lib/call-sounds.ts`): a soft blip the instant an
+  utterance is accepted — perceived latency matters as much as measured.
+
 ## Cost notes
 
 Webcam frames ≈ 250–350 tokens each (≤12/message ≈ 3–4k); screen frames ≈

apps/x/apps/main/src/ipc.ts

@@ -382,6 +382,9 @@ type InvokeHandlers = {
   [K in InvokeChannels]: InvokeHandler<K>;
 };
 
+// In-flight streaming TTS requests, keyed by renderer-chosen requestId.
+const activeTtsStreams = new Map<string, AbortController>();
+
 // Video-mode popout window (shown for the whole duration of a screen share,
 // floating over every app including Rowboat itself) and the last call state
 // pushed by the main window — replayed to the popout when it finishes loading.
@@ -1720,6 +1723,51 @@ export function setupIpcHandlers() {
     'voice:synthesize': async (_event, args) => {
       return voice.synthesizeSpeech(args.text);
     },
+    'voice:synthesizeStreamStart': async (event, args) => {
+      const { requestId, text } = args;
+      const sender = event.sender;
+      const controller = new AbortController();
+      activeTtsStreams.set(requestId, controller);
+      // Fire-and-forget: chunks are pushed to the renderer as they arrive so
+      // playback can begin immediately; the invoke returns once started.
+      void voice
+        .synthesizeSpeechStream(
+          text,
+          (chunk) => {
+            if (!sender.isDestroyed()) {
+              sender.send('voice:tts-chunk', {
+                requestId,
+                chunkBase64: chunk.toString('base64'),
+                done: false,
+              });
+            }
+          },
+          controller.signal,
+        )
+        .then(() => {
+          if (!sender.isDestroyed()) {
+            sender.send('voice:tts-chunk', { requestId, done: true });
+          }
+        })
+        .catch((err: unknown) => {
+          if (!sender.isDestroyed() && !controller.signal.aborted) {
+            sender.send('voice:tts-chunk', {
+              requestId,
+              done: true,
+              error: err instanceof Error ? err.message : String(err),
+            });
+          }
+        })
+        .finally(() => {
+          activeTtsStreams.delete(requestId);
+        });
+      return { ok: true };
+    },
+    'voice:synthesizeStreamCancel': async (_event, args) => {
+      activeTtsStreams.get(args.requestId)?.abort();
+      activeTtsStreams.delete(args.requestId);
+      return {};
+    },
     'voice:ensureMicAccess': async () => {
       if (process.platform !== 'darwin') return { granted: true };
       const status = systemPreferences.getMediaAccessStatus('microphone');

apps/x/apps/renderer/src/App.tsx

@@ -125,6 +125,7 @@ import { ProductTour, type TourNavTarget } from '@/components/product-tour'
 import { useMeetingTranscription, type CalendarEventMeta } from '@/hooks/useMeetingTranscription'
 import { useAnalyticsIdentity } from '@/hooks/useAnalyticsIdentity'
 import * as analytics from '@/lib/analytics'
+import { playAckCue } from '@/lib/call-sounds'
 import { useTheme } from '@/contexts/theme-context'
 
 type DirEntry = z.infer<typeof workspace.DirEntry>
@@ -973,6 +974,10 @@ function App() {
   // TTS plays only during calls now (the standing read-aloud toggle was
   // retired; a per-message "read aloud" action may replace it later).
   const ttsEnabledRef = useRef(false)
+  // Voice-to-voice latency marks for the current call turn (performance.now):
+  // t0 = utterance accepted, submit = message sent, speak = first TTS
+  // speak(). Emitted as call_turn_latency when audio actually starts.
+  const callTurnMarksRef = useRef<{ t0: number; submit?: number; speak?: number } | null>(null)
   // Read-aloud style: 'summary' for typed chat, forced to 'full' during a
   // call and restored after. Context decides — the user never picks it.
   const ttsModeRef = useRef<'summary' | 'full'>('summary')
@@ -1010,12 +1015,29 @@ function App() {
       const segment = voiceSegments[spokenVoiceRef.current.count]
       spokenVoiceRef.current.count += 1
       if (ttsEnabledRef.current) {
+        const marks = callTurnMarksRef.current
+        if (marks && marks.speak === undefined) marks.speak = performance.now()
         ttsRef.current.speak(segment)
         setAssistantCaption(segment)
       }
     }
   }, [voiceSegments, runId])
 
+  // Emit the turn's voice-to-voice latency breakdown once audio is audible.
+  useEffect(() => {
+    if (tts.state !== 'speaking') return
+    const marks = callTurnMarksRef.current
+    if (!marks || marks.submit === undefined || marks.speak === undefined) return
+    callTurnMarksRef.current = null
+    const now = performance.now()
+    analytics.callTurnLatency({
+      endpointToSubmitMs: marks.submit - marks.t0,
+      submitToSpeakMs: marks.speak - marks.submit,
+      speakToAudioMs: now - marks.speak,
+      totalMs: now - marks.t0,
+    })
+  }, [tts.state])
+
   const voice = useVoiceMode()
   const voiceRef = useRef(voice)
   voiceRef.current = voice
@@ -1160,6 +1182,9 @@ function App() {
     ttsEnabledRef.current = true
     ttsModeRef.current = 'full'
     void voiceRef.current.startContinuous((text) => {
+      // Instant "heard you" feedback + start of the latency clock.
+      playAckCue()
+      callTurnMarksRef.current = { t0: performance.now() }
       pendingVoiceInputRef.current = true
       handlePromptSubmitRef.current?.({ text, files: [] })
     })
@@ -1179,6 +1204,7 @@ function App() {
     ttsEnabledRef.current = false
     ttsModeRef.current = 'summary'
     ttsRef.current.cancel()
+    callTurnMarksRef.current = null
     video.stop()
     setPracticeMode(false)
     practiceModeRef.current = false
@@ -2781,6 +2807,11 @@ function App() {
 
     // Video chat mode: drain the webcam frames buffered since the last send
     // so they ride along with this message as inline image parts.
+    const marks = callTurnMarksRef.current
+    if (inCallRef.current && marks && marks.submit === undefined) {
+      marks.submit = performance.now()
+    }
+
     const videoFrames = inCallRef.current ? video.collectFrames() : []
 
     const userMessageId = `user-${Date.now()}`

apps/x/apps/renderer/src/hooks/useVoiceMode.ts

@@ -19,14 +19,24 @@ const DEEPGRAM_PARAMS = new URLSearchParams({
     endpointing: '100',
     no_delay: 'true',
 });
-// Hands-free (continuous) mode uses a much longer endpoint than push-to-talk:
-// speech_final fires only after this much silence, so a thinking pause doesn't
-// cut the user off mid-sentence during a call.
-const CONTINUOUS_ENDPOINTING_MS = 1800;
+// Hands-free (continuous) mode: Deepgram's endpoint fires FAST (600ms of
+// silence) and we apply smart hold logic on our side — if the transcript
+// already reads as a complete thought (terminal punctuation) the utterance
+// fires immediately, otherwise we hold INCOMPLETE_HOLD_MS longer in case the
+// user was mid-thought. Net effect: complete sentences turn around ~1.2s
+// faster than the old fixed 1800ms endpoint, while thinking pauses still get
+// the same total grace (~1.8s).
+const CONTINUOUS_ENDPOINTING_MS = 600;
+const INCOMPLETE_HOLD_MS = 1200;
 // While the mic is paused (assistant speaking), keep the idle Deepgram socket
 // alive — it closes after ~10s without audio otherwise.
 const KEEPALIVE_INTERVAL_MS = 5000;
 
+// Deepgram punctuates finals (punctuate=true) — a transcript ending in
+// terminal punctuation (optionally inside a closing quote/paren) is treated
+// as a complete thought.
+const COMPLETE_THOUGHT_RE = /[.!?…]["')\]]*\s*$/;
+
 function deepgramParams(continuous: boolean): URLSearchParams {
     if (!continuous) return DEEPGRAM_PARAMS;
     const params = new URLSearchParams(DEEPGRAM_PARAMS);
@@ -35,7 +45,7 @@ function deepgramParams(continuous: boolean): URLSearchParams {
     // on a result with an empty transcript, or never fires when background
     // noise keeps the endpointer engaged). UtteranceEnd is word-timing based
     // and arrives as its own message type, so we listen for both.
-    params.set('utterance_end_ms', '2000');
+    params.set('utterance_end_ms', '1000');
     return params;
 }
 
@@ -76,6 +86,8 @@ export function useVoiceMode() {
     // While true (assistant is speaking), mic audio is dropped instead of streamed.
     const pausedRef = useRef(false);
     const keepAliveTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
+    // Pending mid-thought hold (smart endpointing) — see maybeEndUtterance.
+    const holdTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
 
     // Refresh cached auth details (called on warmup, not on mic click)
     const refreshAuth = useCallback(async () => {
@@ -95,10 +107,13 @@ export function useVoiceMode() {
     }, [refreshRowboatAccount]);
 
     // Hands-free mode: flush the accumulated utterance to the callback.
-    // Called on either end-of-speech signal (speech_final or UtteranceEnd);
-    // both may fire for the same utterance — the second finds an empty
-    // buffer and is a no-op.
+    // Both end-of-speech signals may fire for the same utterance — the second
+    // finds an empty buffer and is a no-op.
     const fireContinuousUtterance = useCallback(() => {
+        if (holdTimerRef.current) {
+            clearTimeout(holdTimerRef.current);
+            holdTimerRef.current = null;
+        }
         if (!continuousCbRef.current || pausedRef.current) return;
         const utterance = transcriptBufferRef.current.trim();
         transcriptBufferRef.current = '';
@@ -107,6 +122,25 @@ export function useVoiceMode() {
         if (utterance) continuousCbRef.current(utterance);
     }, []);
 
+    // Smart endpoint: Deepgram's endpoint fires fast (600ms). If the
+    // transcript reads as a complete thought, hand it off immediately; if it
+    // trails off mid-sentence ("so what I want is…"), hold a little longer —
+    // resumed speech cancels the hold and the utterance keeps growing.
+    const maybeEndUtterance = useCallback(() => {
+        if (!continuousCbRef.current || pausedRef.current) return;
+        const buffered = transcriptBufferRef.current.trim();
+        if (!buffered) return;
+        if (COMPLETE_THOUGHT_RE.test(buffered)) {
+            fireContinuousUtterance();
+            return;
+        }
+        if (holdTimerRef.current) clearTimeout(holdTimerRef.current);
+        holdTimerRef.current = setTimeout(() => {
+            holdTimerRef.current = null;
+            fireContinuousUtterance();
+        }, INCOMPLETE_HOLD_MS);
+    }, [fireContinuousUtterance]);
+
     // Create and connect a Deepgram WebSocket using cached auth.
     // Starts the connection and returns immediately (does not wait for open).
     const connectWs = useCallback(async (continuous = false) => {
@@ -143,13 +177,20 @@ export function useVoiceMode() {
 
             // Hands-free mode: word-timing based end-of-speech marker.
             if (data.type === 'UtteranceEnd') {
-                fireContinuousUtterance();
+                maybeEndUtterance();
                 return;
             }
 
             if (!data.channel?.alternatives?.[0]) return;
             const transcript = data.channel.alternatives[0].transcript;
 
+            // The user resumed speaking — cancel any pending mid-thought hold
+            // so the utterance keeps growing instead of firing under them.
+            if (transcript && holdTimerRef.current) {
+                clearTimeout(holdTimerRef.current);
+                holdTimerRef.current = null;
+            }
+
             if (data.is_final) {
                 // NOTE: the endpoint marker (speech_final) usually arrives on a
                 // result whose transcript is EMPTY — the silence after the user
@@ -159,10 +200,11 @@ export function useVoiceMode() {
                     transcriptBufferRef.current += (transcriptBufferRef.current ? ' ' : '') + transcript;
                     interimRef.current = '';
                 }
-                // Hands-free mode: an endpoint completes the utterance — hand
-                // it off and reset for the next one.
+                // Hands-free mode: an endpoint may complete the utterance —
+                // immediately for complete thoughts, after a short hold for
+                // mid-sentence trails.
                 if (continuousCbRef.current && data.speech_final) {
-                    fireContinuousUtterance();
+                    maybeEndUtterance();
                     return;
                 }
                 if (transcript) {
@@ -193,7 +235,7 @@ export function useVoiceMode() {
                 }, 1000);
             }
         };
-    }, [refreshAuth, fireContinuousUtterance]);
+    }, [refreshAuth, maybeEndUtterance]);
 
     const waitForWsOpen = useCallback(async (timeoutMs = 1500): Promise<boolean> => {
         const ws = wsRef.current;
@@ -258,6 +300,10 @@ export function useVoiceMode() {
         }
         continuousCbRef.current = null;
         pausedRef.current = false;
+        if (holdTimerRef.current) {
+            clearTimeout(holdTimerRef.current);
+            holdTimerRef.current = null;
+        }
         if (keepAliveTimerRef.current) {
             clearInterval(keepAliveTimerRef.current);
             keepAliveTimerRef.current = null;
@@ -415,6 +461,10 @@ export function useVoiceMode() {
         if (pausedRef.current === paused) return;
         pausedRef.current = paused;
         if (paused) {
+            if (holdTimerRef.current) {
+                clearTimeout(holdTimerRef.current);
+                holdTimerRef.current = null;
+            }
             transcriptBufferRef.current = '';
             interimRef.current = '';
             setInterimText('');

apps/x/apps/renderer/src/hooks/useVoiceTTS.ts

@@ -47,6 +47,8 @@ function playAudio(
 /** A queue entry: text to synthesize, or a ready-to-play audio URL (e.g. a bundled clip). */
 type QueueItem = { text: string } | { url: string };
 
+type TtsChunkMsg = { requestId: string; chunkBase64?: string; done: boolean; error?: string };
+
 export function useVoiceTTS() {
     const [state, setState] = useState<TTSState>('idle');
     const audioRef = useRef<HTMLAudioElement | null>(null);
@@ -54,6 +56,10 @@ export function useVoiceTTS() {
     const processingRef = useRef(false);
     // Pre-fetched audio ready to play immediately
     const prefetchedRef = useRef<Promise<SynthesizedAudio> | null>(null);
+    // Streaming synthesis: per-request chunk handlers + the in-flight request
+    // id (so cancel() can abort the main-process fetch).
+    const streamHandlersRef = useRef<Map<string, (msg: TtsChunkMsg) => void>>(new Map());
+    const activeStreamIdRef = useRef<string | null>(null);
     // Bumped by cancel(). A queue loop that awaited across a cancel sees a
     // stale generation and exits instead of playing audio that was cancelled
     // while still synthesizing (which would overlap the next utterance).
@@ -107,6 +113,127 @@ export function useVoiceTTS() {
         analyserRef.current = null;
     }, []);
 
+    // Route streaming TTS chunks to whichever request is waiting for them.
+    useEffect(() => {
+        return window.ipc.on('voice:tts-chunk', (msg) => {
+            streamHandlersRef.current.get(msg.requestId)?.(msg);
+        });
+    }, []);
+
+    /**
+     * Streaming synthesis + playback via MediaSource: audio starts on the
+     * first chunk instead of after the full body. Rejects (for caller
+     * fallback to non-streaming synth) if the stream fails before any audio
+     * arrived; resolves when playback finishes.
+     */
+    const streamSynthesizeAndPlay = useCallback((text: string, onStarted: () => void): Promise<void> => {
+        return new Promise<void>((resolve, reject) => {
+            if (typeof MediaSource === 'undefined' || !MediaSource.isTypeSupported('audio/mpeg')) {
+                reject(new Error('MSE audio/mpeg unsupported'));
+                return;
+            }
+            const requestId = `tts-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
+            const mediaSource = new MediaSource();
+            const audio = new Audio();
+            audio.src = URL.createObjectURL(mediaSource);
+            audioRef.current = audio;
+            connectAnalyser(audio);
+            activeStreamIdRef.current = requestId;
+
+            let sourceBuffer: SourceBuffer | null = null;
+            const pending: Uint8Array[] = [];
+            let streamDone = false;
+            let gotAudio = false;
+            let settled = false;
+
+            const cleanup = () => {
+                streamHandlersRef.current.delete(requestId);
+                if (activeStreamIdRef.current === requestId) activeStreamIdRef.current = null;
+                URL.revokeObjectURL(audio.src);
+            };
+            const finish = (err?: Error) => {
+                if (settled) return;
+                settled = true;
+                cleanup();
+                if (err) reject(err);
+                else resolve();
+            };
+
+            // Drain pending chunks into the SourceBuffer one at a time
+            // (appendBuffer is async; only one append may be in flight).
+            const pump = () => {
+                if (!sourceBuffer || sourceBuffer.updating || settled) return;
+                const chunk = pending.shift();
+                if (chunk) {
+                    try {
+                        sourceBuffer.appendBuffer(chunk as BufferSource);
+                    } catch (e) {
+                        finish(e as Error);
+                    }
+                    return;
+                }
+                if (streamDone && mediaSource.readyState === 'open') {
+                    try {
+                        mediaSource.endOfStream();
+                    } catch { /* already ended */ }
+                }
+            };
+
+            mediaSource.addEventListener('sourceopen', () => {
+                try {
+                    sourceBuffer = mediaSource.addSourceBuffer('audio/mpeg');
+                } catch (e) {
+                    finish(e as Error);
+                    return;
+                }
+                sourceBuffer.addEventListener('updateend', pump);
+                pump();
+            }, { once: true });
+
+            streamHandlersRef.current.set(requestId, (msg) => {
+                if (msg.error && !gotAudio) {
+                    streamDone = true;
+                    finish(new Error(msg.error));
+                    return;
+                }
+                if (msg.chunkBase64) {
+                    gotAudio = true;
+                    const bin = atob(msg.chunkBase64);
+                    const bytes = new Uint8Array(bin.length);
+                    for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
+                    pending.push(bytes);
+                    pump();
+                }
+                if (msg.done) {
+                    streamDone = true;
+                    pump();
+                }
+            });
+
+            audio.addEventListener('playing', () => onStarted(), { once: true });
+            audio.onended = () => finish();
+            // pause() (from cancel) must settle this promise too; natural end
+            // also fires 'pause' just before 'ended'; double-settle is a no-op.
+            audio.onpause = () => finish();
+            audio.onerror = () => finish(new Error('stream playback failed'));
+
+            window.ipc
+                .invoke('voice:synthesizeStreamStart', { requestId, text })
+                .then((res) => {
+                    if (!res.ok) finish(new Error(res.error || 'stream start failed'));
+                })
+                .catch((e) => finish(e as Error));
+
+            // Starts as soon as the first appended data is decodable.
+            audio.play().catch(() => { /* surfaced via onerror / chunk error */ });
+
+            // Nothing arrived at all — bail so the caller can fall back.
+            setTimeout(() => {
+                if (!gotAudio && !settled) finish(new Error('stream timeout'));
+            }, 10_000);
+        });
+    }, [connectAnalyser]);
+
     const getLevel = useCallback((): number => {
         const analyser = analyserRef.current;
         if (!analyser) return 0;
@@ -130,10 +257,41 @@ export function useVoiceTTS() {
         processingRef.current = true;
         const gen = generationRef.current;
 
+        // Kick off full-body pre-fetch for the next queued text while the
+        // current one plays — keeps sentence-to-sentence playback gapless.
+        const prefetchNext = () => {
+            const next = queueRef.current[0];
+            if (next && 'text' in next && next.text.trim() && !prefetchedRef.current) {
+                console.log('[tts] pre-fetching next:', next.text.substring(0, 80));
+                prefetchedRef.current = synthesize(next.text);
+            }
+        };
+
         while (queueRef.current.length > 0) {
             const item = queueRef.current.shift()!;
             if ('text' in item && !item.text.trim()) continue;
 
+            // Cold start (nothing playing, nothing pre-fetched): stream the
+            // synthesis so audio begins on the first chunk instead of after
+            // the full body — this is where first-response latency lives.
+            if ('text' in item && !prefetchedRef.current) {
+                setState('synthesizing');
+                console.log('[tts] stream-synthesizing:', item.text.substring(0, 80));
+                try {
+                    await streamSynthesizeAndPlay(item.text, () => {
+                        if (generationRef.current !== gen) return;
+                        setState('speaking');
+                        prefetchNext();
+                    });
+                    if (generationRef.current !== gen) return;
+                    continue;
+                } catch (err) {
+                    if (generationRef.current !== gen) return;
+                    console.error('[tts] stream failed, falling back to full synth:', err);
+                    // fall through to the non-streaming path below
+                }
+            }
+
             try {
                 // Pre-recorded URL plays as-is; text uses the pre-fetched
                 // result if available, otherwise synthesizes now.
@@ -156,12 +314,7 @@ export function useVoiceTTS() {
                 if (generationRef.current !== gen) return;
                 setState('speaking');
 
-                // Kick off pre-fetch for next chunk while this one plays
-                const next = queueRef.current[0];
-                if (next && 'text' in next && next.text.trim()) {
-                    console.log('[tts] pre-fetching next:', next.text.substring(0, 80));
-                    prefetchedRef.current = synthesize(next.text);
-                }
+                prefetchNext();
 
                 await playAudio(audio.dataUrl, audioRef, connectAnalyser);
                 if (generationRef.current !== gen) return;
@@ -176,7 +329,7 @@ export function useVoiceTTS() {
         prefetchedRef.current = null;
         processingRef.current = false;
         setState('idle');
-    }, [connectAnalyser]);
+    }, [connectAnalyser, streamSynthesizeAndPlay]);
 
     const speak = useCallback((text: string) => {
         console.log('[tts] speak() called:', text.substring(0, 80));
@@ -196,6 +349,13 @@ export function useVoiceTTS() {
         generationRef.current++;
         queueRef.current = [];
         prefetchedRef.current = null;
+        // Abort any in-flight streaming synthesis in the main process.
+        if (activeStreamIdRef.current) {
+            void window.ipc
+                .invoke('voice:synthesizeStreamCancel', { requestId: activeStreamIdRef.current })
+                .catch(() => {});
+            activeStreamIdRef.current = null;
+        }
         if (audioRef.current) {
             audioRef.current.pause();
             audioRef.current = null;

apps/x/apps/renderer/src/lib/analytics.ts

@@ -69,6 +69,22 @@ export function callStarted(preset: 'voice' | 'video' | 'share' | 'practice') {
   posthog.capture('call_started', { preset })
 }
 
+// Voice-to-voice latency breakdown for one call turn (all milliseconds):
+// utterance accepted → message submitted → first TTS speak() → audio playing.
+export function callTurnLatency(props: {
+  endpointToSubmitMs: number
+  submitToSpeakMs: number
+  speakToAudioMs: number
+  totalMs: number
+}) {
+  posthog.capture('call_turn_latency', {
+    endpoint_to_submit_ms: Math.round(props.endpointToSubmitMs),
+    submit_to_speak_ms: Math.round(props.submitToSpeakMs),
+    speak_to_audio_ms: Math.round(props.speakToAudioMs),
+    total_ms: Math.round(props.totalMs),
+  })
+}
+
 export function searchExecuted(types: string[]) {
   posthog.capture('search_executed', { types })
 }

apps/x/apps/renderer/src/lib/call-sounds.ts

@@ -0,0 +1,30 @@
+// Tiny synthesized UI sounds for calls — no audio assets, one lazy context.
+
+let ctx: AudioContext | null = null
+
+/**
+ * Soft rising blip played the instant an utterance is accepted — sub-second
+ * acknowledgment makes the (still ongoing) model turn feel responsive
+ * instead of dead air.
+ */
+export function playAckCue() {
+  try {
+    if (!ctx) ctx = new AudioContext()
+    if (ctx.state === 'suspended') void ctx.resume()
+    const t = ctx.currentTime
+    const osc = ctx.createOscillator()
+    const gain = ctx.createGain()
+    osc.type = 'sine'
+    osc.frequency.setValueAtTime(880, t)
+    osc.frequency.exponentialRampToValueAtTime(1320, t + 0.08)
+    gain.gain.setValueAtTime(0.0001, t)
+    gain.gain.exponentialRampToValueAtTime(0.08, t + 0.015)
+    gain.gain.exponentialRampToValueAtTime(0.0001, t + 0.12)
+    osc.connect(gain)
+    gain.connect(ctx.destination)
+    osc.start(t)
+    osc.stop(t + 0.13)
+  } catch {
+    // cosmetic — never let a sound failure affect the call
+  }
+}

apps/x/apps/renderer/src/lib/session-chat/turn-view.test.ts

@@ -96,6 +96,25 @@ describe('voice output', () => {
     expect(overlay.voiceSegments).toEqual(['hello there', 'bye'])
   })
 
+  it('emits an early clause from a long open block, then the remainder on close', () => {
+    let overlay = emptyOverlay()
+    const longClause = 'Okay so the first thing I would look at here is the error message,'
+    overlay = applyOverlay(overlay, delta(`<voice>${longClause} because`))
+    // Open block crossed the early-speech threshold at a clause boundary.
+    expect(overlay.voiceSegments).toEqual([longClause])
+    overlay = applyOverlay(overlay, delta(' it tells you the root cause.</voice>'))
+    // Remainder only — the early clause is not repeated.
+    expect(overlay.voiceSegments).toEqual([longClause, 'because it tells you the root cause.'])
+  })
+
+  it('does not emit early clauses from short open blocks', () => {
+    let overlay = emptyOverlay()
+    overlay = applyOverlay(overlay, delta('<voice>Sure, one sec'))
+    expect(overlay.voiceSegments).toEqual([])
+    overlay = applyOverlay(overlay, delta('.</voice>'))
+    expect(overlay.voiceSegments).toEqual(['Sure, one sec.'])
+  })
+
   it('keeps segments but resets the scan on model_call_completed', () => {
     let overlay = emptyOverlay()
     overlay = applyOverlay(overlay, delta('<voice>one</voice>'))

apps/x/apps/renderer/src/lib/session-chat/turn-view.ts

@@ -34,14 +34,20 @@ export type LiveOverlay = {
   text: string
   reasoning: string
   toolOutput: Record<string, string>
-  // Contents of completed <voice>…</voice> blocks seen while streaming, in
-  // order, monotonically growing for the lifetime of the overlay (i.e. one
-  // active turn). Consumers speak segments beyond what they've already
-  // spoken; the overlay reset on turn switch starts a fresh list.
+  // Speakable segments seen while streaming, in order, monotonically growing
+  // for the lifetime of the overlay (i.e. one active turn). Usually the
+  // contents of completed <voice>…</voice> blocks, but a long still-open
+  // block may emit an early clause (see EARLY_SPEECH_MIN_CHARS) so speech
+  // can start before the sentence finishes generating. Consumers speak
+  // segments beyond what they've already spoken; the overlay reset on turn
+  // switch starts a fresh list.
   voiceSegments: string[]
   // Scan cursor into `text` — everything before it has been checked for
   // complete voice blocks.
   voiceScanIndex: number
+  // Chars of the currently-open voice block's content already emitted as an
+  // early clause — the block's remainder (on close) excludes them.
+  voicePartialConsumed: number
 }
 
 export const emptyOverlay = (): LiveOverlay => ({
@@ -50,6 +56,7 @@ export const emptyOverlay = (): LiveOverlay => ({
   toolOutput: {},
   voiceSegments: [],
   voiceScanIndex: 0,
+  voicePartialConsumed: 0,
 })
 
 // The model emits <voice>…</voice> around speakable text when voice output
@@ -59,6 +66,17 @@ export function stripVoiceTags(text: string): string {
 }
 
 const VOICE_BLOCK = /<voice>([\s\S]*?)<\/voice>/g
+const VOICE_OPEN_TAG = '<voice>'
+
+// Early speech: once an open block has this many unconsumed chars, its last
+// complete clause is emitted immediately instead of waiting for </voice> —
+// TTS starts on the first clause while the rest of the sentence generates.
+const EARLY_SPEECH_MIN_CHARS = 60
+// ...but never emit a fragment shorter than this (prosody suffers).
+const EARLY_SPEECH_MIN_EMIT = 30
+// Clause boundaries (punctuation, optionally inside closing quote/paren,
+// followed by whitespace or end-of-buffer).
+const CLAUSE_BOUNDARY = /[,;:.!?…—]["')\]]*(?=\s|$)/g
 
 // Accumulates deltas; canonical durable events supersede the buffers (the
 // committed transcript now contains what was streaming).
@@ -68,15 +86,43 @@ export function applyOverlay(overlay: LiveOverlay, event: TurnStreamEvent): Live
       const text = overlay.text + event.delta
       // Extract complete voice blocks past the scan cursor. Incomplete
       // blocks (opening tag seen, closing not yet) stay unconsumed until a
-      // later delta completes them.
+      // later delta completes them. The first complete block may have had an
+      // early clause emitted while it was open — skip those chars.
       const segments: string[] = []
       let scanIndex = overlay.voiceScanIndex
+      let partialConsumed = overlay.voicePartialConsumed
       VOICE_BLOCK.lastIndex = scanIndex
       for (let m = VOICE_BLOCK.exec(text); m; m = VOICE_BLOCK.exec(text)) {
-        const content = m[1].trim()
+        const content = m[1].slice(partialConsumed).trim()
+        partialConsumed = 0
         if (content) segments.push(content)
         scanIndex = m.index + m[0].length
       }
+
+      // Early speech: if a voice block is still open and has accumulated a
+      // long unconsumed run, emit its last complete clause now — speech can
+      // start while the rest of the sentence is still generating.
+      const openIdx = text.indexOf(VOICE_OPEN_TAG, scanIndex)
+      if (openIdx !== -1) {
+        const unconsumed = text.slice(openIdx + VOICE_OPEN_TAG.length + partialConsumed)
+        if (unconsumed.length >= EARLY_SPEECH_MIN_CHARS) {
+          let lastBoundaryEnd = -1
+          CLAUSE_BOUNDARY.lastIndex = 0
+          for (let b = CLAUSE_BOUNDARY.exec(unconsumed); b; b = CLAUSE_BOUNDARY.exec(unconsumed)) {
+            lastBoundaryEnd = b.index + b[0].length
+          }
+          if (lastBoundaryEnd >= EARLY_SPEECH_MIN_EMIT) {
+            const clause = unconsumed.slice(0, lastBoundaryEnd).trim()
+            if (clause) segments.push(clause)
+            partialConsumed += lastBoundaryEnd
+          }
+        }
+      } else {
+        // No open block — any partial bookkeeping belongs to a block that
+        // has since closed.
+        partialConsumed = 0
+      }
+
       return {
         ...overlay,
         text,
@@ -84,12 +130,13 @@ export function applyOverlay(overlay: LiveOverlay, event: TurnStreamEvent): Live
           ? { voiceSegments: [...overlay.voiceSegments, ...segments] }
           : {}),
         voiceScanIndex: scanIndex,
+        voicePartialConsumed: partialConsumed,
       }
     }
     case 'reasoning_delta':
       return { ...overlay, reasoning: overlay.reasoning + event.delta }
     case 'model_call_completed':
-      return { ...overlay, text: '', reasoning: '', voiceScanIndex: 0 }
+      return { ...overlay, text: '', reasoning: '', voiceScanIndex: 0, voicePartialConsumed: 0 }
     case 'tool_progress': {
       const progress = event.progress
       if (

apps/x/packages/core/src/voice/voice.ts

@@ -32,46 +32,57 @@ export async function getVoiceConfig(): Promise<VoiceConfig> {
     };
 }
 
-export async function synthesizeSpeech(text: string): Promise<{ audioBase64: string; mimeType: string }> {
+async function resolveTtsEndpoint(streaming: boolean): Promise<{ url: string; headers: Record<string, string> }> {
     const config = await getVoiceConfig();
     const signedIn = await isSignedIn();
 
-    let url: string;
-    let headers: Record<string, string>;
-
     if (signedIn) {
         const voiceId = config.elevenlabs?.voiceId || 's3TPKV1kjDlVtZbl4Ksh';
         const accessToken = await getAccessToken();
-        url = `${API_URL}/v1/voice/text-to-speech/${voiceId}`;
-        headers = {
-            'Authorization': `Bearer ${accessToken}`,
-            'Content-Type': 'application/json',
+        // The proxy has no dedicated /stream route — the same endpoint is
+        // used and the body is consumed progressively; if the proxy buffers,
+        // streaming degrades to today's full-body latency, never worse.
+        return {
+            url: `${API_URL}/v1/voice/text-to-speech/${voiceId}`,
+            headers: {
+                'Authorization': `Bearer ${accessToken}`,
+                'Content-Type': 'application/json',
+            },
         };
-        console.log('[voice] synthesizing speech via Rowboat proxy, text length:', text.length, 'voiceId:', voiceId);
-    } else {
-        if (!config.elevenlabs) {
-            throw new Error(`ElevenLabs not configured. Create ${path.join(WorkDir, 'config', 'elevenlabs.json')} with { "apiKey": "<your-key>" }`);
-        }
-        const voiceId = config.elevenlabs.voiceId || 's3TPKV1kjDlVtZbl4Ksh';
-        url = `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`;
-        headers = {
+    }
+
+    if (!config.elevenlabs) {
+        throw new Error(`ElevenLabs not configured. Create ${path.join(WorkDir, 'config', 'elevenlabs.json')} with { "apiKey": "<your-key>" }`);
+    }
+    const voiceId = config.elevenlabs.voiceId || 's3TPKV1kjDlVtZbl4Ksh';
+    return {
+        url: `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}${streaming ? '/stream' : ''}`,
+        headers: {
             'xi-api-key': config.elevenlabs.apiKey,
             'Content-Type': 'application/json',
-        };
-        console.log('[voice] synthesizing speech via ElevenLabs, text length:', text.length, 'voiceId:', voiceId);
-    }
+        },
+    };
+}
+
+function ttsRequestBody(text: string): string {
+    return JSON.stringify({
+        text,
+        model_id: 'eleven_flash_v2_5',
+        voice_settings: {
+            stability: 0.5,
+            similarity_boost: 0.75,
+        },
+    });
+}
+
+export async function synthesizeSpeech(text: string): Promise<{ audioBase64: string; mimeType: string }> {
+    const { url, headers } = await resolveTtsEndpoint(false);
+    console.log('[voice] synthesizing speech, text length:', text.length);
 
     const response = await fetch(url, {
         method: 'POST',
         headers,
-        body: JSON.stringify({
-            text,
-            model_id: 'eleven_flash_v2_5',
-            voice_settings: {
-                stability: 0.5,
-                similarity_boost: 0.75,
-            },
-        }),
+        body: ttsRequestBody(text),
     });
 
     if (!response.ok) {
@@ -85,3 +96,42 @@ export async function synthesizeSpeech(text: string): Promise<{ audioBase64: str
     console.log('[voice] synthesized audio, base64 length:', audioBase64.length);
     return { audioBase64, mimeType: 'audio/mpeg' };
 }
+
+/**
+ * Streaming synthesis: invokes `onChunk` with MP3 bytes as they arrive so
+ * playback can start on the first chunk. Resolves when the stream ends;
+ * rejects on HTTP/stream errors. Abort via the provided signal.
+ */
+export async function synthesizeSpeechStream(
+    text: string,
+    onChunk: (chunk: Buffer) => void,
+    signal?: AbortSignal,
+): Promise<void> {
+    const { url, headers } = await resolveTtsEndpoint(true);
+    console.log('[voice] streaming speech synthesis, text length:', text.length);
+
+    const response = await fetch(url, {
+        method: 'POST',
+        headers,
+        body: ttsRequestBody(text),
+        signal: signal ?? null,
+    });
+
+    if (!response.ok) {
+        const errText = await response.text().catch(() => 'Unknown error');
+        console.error('[voice] TTS stream API error:', response.status, errText);
+        throw new Error(`TTS API error ${response.status}: ${errText}`);
+    }
+    if (!response.body) {
+        throw new Error('TTS API returned no body');
+    }
+
+    const reader = response.body.getReader();
+    for (;;) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        if (value && value.byteLength > 0) {
+            onChunk(Buffer.from(value));
+        }
+    }
+}

apps/x/packages/shared/src/ipc.ts

@@ -1362,6 +1362,34 @@ const ipcSchemas = {
       mimeType: z.string(),
     }),
   },
+  // Streaming TTS: main starts the synthesis and pushes audio chunks over
+  // 'voice:tts-chunk' as they arrive, so playback can begin on the first
+  // chunk instead of after the full body (~0.5-1s earlier first-audio).
+  'voice:synthesizeStreamStart': {
+    req: z.object({
+      requestId: z.string(),
+      text: z.string(),
+    }),
+    res: z.object({
+      ok: z.boolean(),
+      error: z.string().optional(),
+    }),
+  },
+  'voice:synthesizeStreamCancel': {
+    req: z.object({ requestId: z.string() }),
+    res: z.object({}),
+  },
+  // Push channel: main → renderer with streaming TTS audio. `done: true`
+  // (possibly with a final chunk) ends the stream; `error` aborts it.
+  'voice:tts-chunk': {
+    req: z.object({
+      requestId: z.string(),
+      chunkBase64: z.string().optional(),
+      done: z.boolean(),
+      error: z.string().optional(),
+    }),
+    res: z.null(),
+  },
   // Ensures the OS-level microphone permission is settled before capturing.
   // On first-ever use (macOS) the permission is 'not-determined'; resolving
   // the native prompt up front prevents the in-flight getUserMedia from