diff --git a/apps/x/apps/renderer/src/App.tsx b/apps/x/apps/renderer/src/App.tsx index bfe71c85..7ccb56e3 100644 --- a/apps/x/apps/renderer/src/App.tsx +++ b/apps/x/apps/renderer/src/App.tsx @@ -1040,8 +1040,9 @@ function App() { const editorRefsByTabId = useRef>(new Map()) const [pendingPaletteSubmit, setPendingPaletteSubmit] = useState<{ text: string; mention: CommandPaletteMention | null } | null>(null) - const handleSubmitRecording = useCallback(() => { - const text = voice.submit() + const handleSubmitRecording = useCallback(async () => { + if (!isRecordingRef.current) return + const text = await voice.submit() setIsRecording(false) isRecordingRef.current = false if (text) { @@ -6377,7 +6378,7 @@ function App() { onWorkDirChange={(v) => setTabWorkDir(tab.id, v)} isRecording={isActive && isRecording} recordingText={isActive ? voice.interimText : undefined} - recordingState={isActive ? (voice.state === 'connecting' ? 'connecting' : 'listening') : undefined} + recordingState={isActive ? (voice.state === 'submitting' ? 'stopping' : voice.state === 'connecting' ? 'connecting' : 'listening') : undefined} audioLevelsRef={voice.audioLevelsRef} onStartRecording={isActive ? handleStartRecording : undefined} onSubmitRecording={isActive ? handleSubmitRecording : undefined} @@ -6486,7 +6487,7 @@ function App() { collapsedLeftPaddingPx={collapsedLeftPaddingPx} isRecording={isRecording} recordingText={voice.interimText} - recordingState={voice.state === 'connecting' ? 'connecting' : 'listening'} + recordingState={voice.state === 'submitting' ? 'stopping' : voice.state === 'connecting' ? 'connecting' : 'listening'} audioLevelsRef={voice.audioLevelsRef} onStartRecording={handleStartRecording} onSubmitRecording={handleSubmitRecording} diff --git a/apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx b/apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx index f3609212..182ed20a 100644 --- a/apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx +++ b/apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx @@ -223,11 +223,11 @@ interface ChatInputInnerProps { onDraftChange?: (text: string) => void isRecording?: boolean recordingText?: string - recordingState?: 'connecting' | 'listening' + recordingState?: 'connecting' | 'listening' | 'stopping' /** Live mic amplitude history (RMS per frame) driving the recording waveform. */ audioLevelsRef?: React.MutableRefObject onStartRecording?: () => void - onSubmitRecording?: () => void + onSubmitRecording?: () => void | Promise onCancelRecording?: () => void voiceAvailable?: boolean ttsAvailable?: boolean @@ -262,6 +262,7 @@ function ChatInputInner({ onDraftChange, isRecording, recordingText, + recordingState, audioLevelsRef, onStartRecording, onSubmitRecording, @@ -829,23 +830,33 @@ function ChatInputInner({ > - {/* Audio-reactive waveform only — the transcribed words are intentionally - not shown while recording; they're still captured and submitted. */} -
+
+
+ {recordingText?.trim() || (recordingState === 'stopping' ? 'Finalizing...' : 'Listening...')} +
) : ( @@ -1477,10 +1488,10 @@ export interface ChatInputWithMentionsProps { onDraftChange?: (text: string) => void isRecording?: boolean recordingText?: string - recordingState?: 'connecting' | 'listening' + recordingState?: 'connecting' | 'listening' | 'stopping' audioLevelsRef?: React.MutableRefObject onStartRecording?: () => void - onSubmitRecording?: () => void + onSubmitRecording?: () => void | Promise onCancelRecording?: () => void voiceAvailable?: boolean ttsAvailable?: boolean diff --git a/apps/x/apps/renderer/src/components/chat-sidebar.tsx b/apps/x/apps/renderer/src/components/chat-sidebar.tsx index a463b080..a9d6d9f7 100644 --- a/apps/x/apps/renderer/src/components/chat-sidebar.tsx +++ b/apps/x/apps/renderer/src/components/chat-sidebar.tsx @@ -177,10 +177,10 @@ interface ChatSidebarProps { // Voice / TTS props isRecording?: boolean recordingText?: string - recordingState?: 'connecting' | 'listening' + recordingState?: 'connecting' | 'listening' | 'stopping' audioLevelsRef?: React.MutableRefObject onStartRecording?: () => void - onSubmitRecording?: () => void + onSubmitRecording?: () => void | Promise onCancelRecording?: () => void voiceAvailable?: boolean ttsAvailable?: boolean diff --git a/apps/x/apps/renderer/src/hooks/useMeetingTranscription.ts b/apps/x/apps/renderer/src/hooks/useMeetingTranscription.ts index 96f42412..75b4d91f 100644 --- a/apps/x/apps/renderer/src/hooks/useMeetingTranscription.ts +++ b/apps/x/apps/renderer/src/hooks/useMeetingTranscription.ts @@ -1,6 +1,7 @@ import { useCallback, useRef, useState } from 'react'; import { toast } from 'sonner'; import { buildDeepgramListenUrl } from '@/lib/deepgram-listen-url'; +import { finalizeDeepgramStream } from '@/lib/deepgram-finalize'; import { useRowboatAccount } from '@/hooks/useRowboatAccount'; export type MeetingTranscriptionState = 'idle' | 'connecting' | 'recording' | 'stopping'; @@ -196,6 +197,25 @@ export function useMeetingTranscription(onAutoStop?: () => void) { }, 1000); }, [writeTranscriptToFile]); + const stopInputCapture = useCallback(() => { + if (processorRef.current) { + processorRef.current.disconnect(); + processorRef.current = null; + } + if (audioCtxRef.current) { + audioCtxRef.current.close(); + audioCtxRef.current = null; + } + if (micStreamRef.current) { + micStreamRef.current.getTracks().forEach(t => t.stop()); + micStreamRef.current = null; + } + if (systemStreamRef.current) { + systemStreamRef.current.getTracks().forEach(t => t.stop()); + systemStreamRef.current = null; + } + }, []); + const cleanup = useCallback(() => { if (writeTimerRef.current) { clearTimeout(writeTimerRef.current); @@ -213,28 +233,13 @@ export function useMeetingTranscription(onAutoStop?: () => void) { clearInterval(trackPollingRef.current); trackPollingRef.current = null; } - if (processorRef.current) { - processorRef.current.disconnect(); - processorRef.current = null; - } - if (audioCtxRef.current) { - audioCtxRef.current.close(); - audioCtxRef.current = null; - } - if (micStreamRef.current) { - micStreamRef.current.getTracks().forEach(t => t.stop()); - micStreamRef.current = null; - } - if (systemStreamRef.current) { - systemStreamRef.current.getTracks().forEach(t => t.stop()); - systemStreamRef.current = null; - } + stopInputCapture(); if (wsRef.current) { wsRef.current.onclose = null; wsRef.current.close(); wsRef.current = null; } - }, []); + }, [stopInputCapture]); const start = useCallback(async (calendarEvent?: CalendarEventMeta): Promise => { if (state !== 'idle') return null; @@ -551,12 +556,14 @@ export function useMeetingTranscription(onAutoStop?: () => void) { if (state !== 'recording') return; setState('stopping'); + stopInputCapture(); + await finalizeDeepgramStream(wsRef.current, 2200); cleanup(); - interimRef.current = new Map(); await writeTranscriptToFile(); + interimRef.current = new Map(); setState('idle'); - }, [state, cleanup, writeTranscriptToFile]); + }, [state, cleanup, stopInputCapture, writeTranscriptToFile]); return { state, start, stop }; } diff --git a/apps/x/apps/renderer/src/hooks/useVoiceMode.ts b/apps/x/apps/renderer/src/hooks/useVoiceMode.ts index 8263e45d..a94a7eb8 100644 --- a/apps/x/apps/renderer/src/hooks/useVoiceMode.ts +++ b/apps/x/apps/renderer/src/hooks/useVoiceMode.ts @@ -1,10 +1,11 @@ import { useCallback, useRef, useState } from 'react'; import { buildDeepgramListenUrl } from '@/lib/deepgram-listen-url'; +import { finalizeDeepgramStream } from '@/lib/deepgram-finalize'; import { useRowboatAccount } from '@/hooks/useRowboatAccount'; import posthog from 'posthog-js'; import * as analytics from '@/lib/analytics'; -export type VoiceState = 'idle' | 'connecting' | 'listening'; +export type VoiceState = 'idle' | 'connecting' | 'listening' | 'submitting'; const DEEPGRAM_PARAMS = new URLSearchParams({ model: 'nova-3', @@ -129,8 +130,45 @@ export function useVoiceMode() { }; }, [refreshAuth]); - // Stop audio capture and close WS - const stopAudioCapture = useCallback(() => { + const waitForWsOpen = useCallback(async (timeoutMs = 1500): Promise => { + const ws = wsRef.current; + if (!ws) return false; + if (ws.readyState === WebSocket.OPEN) return true; + if (ws.readyState !== WebSocket.CONNECTING) return false; + + return new Promise((resolve) => { + let done = false; + let timeout: ReturnType; + const finish = (ok: boolean) => { + if (done) return; + done = true; + clearTimeout(timeout); + ws.removeEventListener('open', onOpen); + ws.removeEventListener('error', onError); + ws.removeEventListener('close', onClose); + resolve(ok); + }; + const onOpen = () => finish(true); + const onError = () => finish(false); + const onClose = () => finish(false); + timeout = setTimeout(() => finish(false), timeoutMs); + ws.addEventListener('open', onOpen); + ws.addEventListener('error', onError); + ws.addEventListener('close', onClose); + }); + }, []); + + const flushBufferedAudio = useCallback(() => { + const ws = wsRef.current; + if (!ws || ws.readyState !== WebSocket.OPEN) return; + const buffered = audioBufferRef.current; + audioBufferRef.current = []; + for (const chunk of buffered) { + ws.send(chunk); + } + }, []); + + const stopInputCapture = useCallback(() => { if (processorRef.current) { processorRef.current.disconnect(); processorRef.current = null; @@ -143,6 +181,11 @@ export function useVoiceMode() { mediaStreamRef.current.getTracks().forEach(t => t.stop()); mediaStreamRef.current = null; } + }, []); + + // Stop audio capture and close WS + const stopAudioCapture = useCallback(() => { + stopInputCapture(); if (wsRef.current) { wsRef.current.onclose = null; wsRef.current.close(); @@ -155,7 +198,7 @@ export function useVoiceMode() { transcriptBufferRef.current = ''; interimRef.current = ''; setState('idle'); - }, []); + }, [stopInputCapture]); const start = useCallback(async () => { if (state !== 'idle') return; @@ -250,15 +293,25 @@ export function useVoiceMode() { }, [state, connectWs, stopAudioCapture]); /** Stop recording and return the full transcript (finalized + any current interim) */ - const submit = useCallback((): string => { + const submit = useCallback(async (): Promise => { + setState('submitting'); + stopInputCapture(); + + if (wsRef.current?.readyState === WebSocket.CONNECTING) { + await waitForWsOpen(); + } + flushBufferedAudio(); + await finalizeDeepgramStream(wsRef.current); + let text = transcriptBufferRef.current; if (interimRef.current) { text += (text ? ' ' : '') + interimRef.current; } text = text.trim(); + stopAudioCapture(); return text; - }, [stopAudioCapture]); + }, [flushBufferedAudio, stopAudioCapture, stopInputCapture, waitForWsOpen]); /** Cancel recording without returning transcript */ const cancel = useCallback(() => { diff --git a/apps/x/apps/renderer/src/lib/deepgram-finalize.ts b/apps/x/apps/renderer/src/lib/deepgram-finalize.ts new file mode 100644 index 00000000..91c3e37f --- /dev/null +++ b/apps/x/apps/renderer/src/lib/deepgram-finalize.ts @@ -0,0 +1,33 @@ +export async function finalizeDeepgramStream(ws: WebSocket | null, timeoutMs = 1800): Promise { + if (!ws || ws.readyState !== WebSocket.OPEN) return; + + await new Promise((resolve) => { + let done = false; + let timeout: ReturnType; + const finish = () => { + if (done) return; + done = true; + clearTimeout(timeout); + ws.removeEventListener('message', onMessage); + resolve(); + }; + timeout = setTimeout(finish, timeoutMs); + const onMessage = (event: MessageEvent) => { + try { + const data = JSON.parse(event.data); + if (data?.is_final || data?.speech_final || data?.type === 'UtteranceEnd') { + finish(); + } + } catch { + // Ignore non-JSON control frames. + } + }; + + ws.addEventListener('message', onMessage); + try { + ws.send(JSON.stringify({ type: 'Finalize' })); + } catch { + finish(); + } + }); +} diff --git a/apps/x/packages/core/src/knowledge/sync_gmail.ts b/apps/x/packages/core/src/knowledge/sync_gmail.ts index e4d478e9..6b4018ec 100644 --- a/apps/x/packages/core/src/knowledge/sync_gmail.ts +++ b/apps/x/packages/core/src/knowledge/sync_gmail.ts @@ -248,6 +248,7 @@ function notifyNewEmails(threads: SyncedThread[]): void { const now = Date.now(); for (const { threadId } of threads) { const snapshot = readCachedSnapshot(threadId)?.snapshot; + if (snapshot?.importance !== 'important') continue; if (snapshot && isEmailTooOldToNotify(snapshotDateMs(snapshot), now)) continue; const subject = snapshot?.subject?.trim() || '(no subject)'; const from = snapshot?.from?.trim();