mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-07-12 21:02:17 +02:00
hands free videochat
This commit is contained in:
Arjun
parent
6f901095ff
commit
ad24699661
5 changed files with 334 additions and 83 deletions
|
|
@ -1013,21 +1013,16 @@ function App() {
|
|||
voiceRef.current = voice
|
||||
|
||||
// Video chat mode: while on, the webcam runs and frames are attached to
|
||||
// every outgoing message so the assistant can see the user.
|
||||
// every outgoing message so the assistant can see the user. 'chat' keeps
|
||||
// the normal composer; 'call' is fully hands-free — continuous listening,
|
||||
// auto-submitted utterances, spoken responses (handler defined below, after
|
||||
// the voice/submit plumbing it drives).
|
||||
const video = useVideoMode()
|
||||
const [videoEnabled, setVideoEnabled] = useState(false)
|
||||
const videoEnabledRef = useRef(false)
|
||||
const handleToggleVideo = useCallback(async () => {
|
||||
if (videoEnabledRef.current) {
|
||||
video.stop()
|
||||
videoEnabledRef.current = false
|
||||
setVideoEnabled(false)
|
||||
} else {
|
||||
const ok = await video.start()
|
||||
videoEnabledRef.current = ok
|
||||
setVideoEnabled(ok)
|
||||
}
|
||||
}, [video])
|
||||
const [videoChatMode, setVideoChatMode] = useState<'off' | 'chat' | 'call'>('off')
|
||||
const videoChatModeRef = useRef<'off' | 'chat' | 'call'>('off')
|
||||
// TTS settings to restore when a hands-free call ends (a call forces
|
||||
// full-read-aloud TTS for its duration).
|
||||
const preCallTtsRef = useRef<{ enabled: boolean; mode: 'summary' | 'full' } | null>(null)
|
||||
|
||||
const handleToggleMeetingRef = useRef<(() => void) | undefined>(undefined)
|
||||
const meetingTranscription = useMeetingTranscription(() => {
|
||||
|
|
@ -1087,6 +1082,8 @@ function App() {
|
|||
}, [])
|
||||
|
||||
const handleStartRecording = useCallback(() => {
|
||||
// Hands-free call mode owns the mic — ignore push-to-talk while it's on.
|
||||
if (videoChatModeRef.current === 'call') return
|
||||
setIsRecording(true)
|
||||
isRecordingRef.current = true
|
||||
voice.start()
|
||||
|
|
@ -1148,6 +1145,69 @@ function App() {
|
|||
isRecordingRef.current = false
|
||||
}, [voice])
|
||||
|
||||
// Video chat mode transitions. 'chat' just runs the camera; 'call' also
|
||||
// listens continuously (auto-submitting each utterance as a voice message)
|
||||
// and forces full read-aloud TTS so the assistant answers out loud.
|
||||
const handleVideoModeChange = useCallback(async (mode: 'off' | 'chat' | 'call') => {
|
||||
const prev = videoChatModeRef.current
|
||||
if (mode === prev) return
|
||||
|
||||
// Leaving a call: release the mic and restore the pre-call TTS settings.
|
||||
if (prev === 'call') {
|
||||
voiceRef.current.cancel()
|
||||
const saved = preCallTtsRef.current
|
||||
preCallTtsRef.current = null
|
||||
const restoreEnabled = saved?.enabled ?? false
|
||||
setTtsEnabled(restoreEnabled)
|
||||
ttsEnabledRef.current = restoreEnabled
|
||||
if (saved) {
|
||||
setTtsMode(saved.mode)
|
||||
ttsModeRef.current = saved.mode
|
||||
}
|
||||
if (!restoreEnabled) ttsRef.current.cancel()
|
||||
}
|
||||
|
||||
if (mode === 'off') {
|
||||
video.stop()
|
||||
videoChatModeRef.current = 'off'
|
||||
setVideoChatMode('off')
|
||||
return
|
||||
}
|
||||
|
||||
if (prev === 'off') {
|
||||
const ok = await video.start()
|
||||
if (!ok) return // camera denied/unavailable — stay off
|
||||
}
|
||||
|
||||
if (mode === 'call') {
|
||||
// A manual push-to-talk recording can't coexist with the call's mic.
|
||||
if (isRecordingRef.current) {
|
||||
voiceRef.current.cancel()
|
||||
setIsRecording(false)
|
||||
isRecordingRef.current = false
|
||||
}
|
||||
preCallTtsRef.current = { enabled: ttsEnabledRef.current, mode: ttsModeRef.current }
|
||||
setTtsEnabled(true)
|
||||
ttsEnabledRef.current = true
|
||||
setTtsMode('full')
|
||||
ttsModeRef.current = 'full'
|
||||
void voiceRef.current.startContinuous((text) => {
|
||||
pendingVoiceInputRef.current = true
|
||||
handlePromptSubmitRef.current?.({ text, files: [] })
|
||||
})
|
||||
}
|
||||
|
||||
videoChatModeRef.current = mode
|
||||
setVideoChatMode(mode)
|
||||
}, [video])
|
||||
|
||||
// During a call, mute the mic while the assistant is thinking or speaking
|
||||
// so its own TTS (or a half-turn) never gets transcribed back at it.
|
||||
useEffect(() => {
|
||||
if (videoChatMode !== 'call') return
|
||||
voiceRef.current.setPaused(activeIsProcessing || tts.state !== 'idle')
|
||||
}, [videoChatMode, activeIsProcessing, tts.state])
|
||||
|
||||
// Enter to submit voice input, Escape to cancel
|
||||
useEffect(() => {
|
||||
const handleKeyDown = (e: KeyboardEvent) => {
|
||||
|
|
@ -2629,7 +2689,7 @@ function App() {
|
|||
|
||||
// Video chat mode: drain the webcam frames buffered since the last send
|
||||
// so they ride along with this message as inline image parts.
|
||||
const videoFrames = videoEnabledRef.current ? video.collectFrames() : []
|
||||
const videoFrames = videoChatModeRef.current !== 'off' ? video.collectFrames() : []
|
||||
|
||||
const userMessageId = `user-${Date.now()}`
|
||||
const displayAttachments: ChatMessage['attachments'] = hasAttachments || videoFrames.length > 0
|
||||
|
|
@ -2700,7 +2760,7 @@ function App() {
|
|||
...(ttsEnabledRef.current ? { voiceOutput: ttsModeRef.current } : {}),
|
||||
...(searchEnabled ? { searchEnabled: true } : {}),
|
||||
...(codeMode ? { codeMode } : {}),
|
||||
...(videoEnabledRef.current ? { videoMode: true } : {}),
|
||||
...(videoChatModeRef.current !== 'off' ? { videoMode: true } : {}),
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -6387,8 +6447,9 @@ function App() {
|
|||
onTtsModeChange={isActive ? handleTtsModeChange : undefined}
|
||||
ttsAvatarEnabled={ttsAvatarEnabled}
|
||||
onToggleTtsAvatar={isActive ? handleToggleTtsAvatar : undefined}
|
||||
videoEnabled={videoEnabled}
|
||||
onToggleVideo={isActive ? handleToggleVideo : undefined}
|
||||
videoChatMode={videoChatMode}
|
||||
onVideoModeChange={isActive ? handleVideoModeChange : undefined}
|
||||
videoCallAvailable={voiceAvailable && ttsAvailable}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
|
|
@ -6502,14 +6563,28 @@ function App() {
|
|||
onTtsModeChange={handleTtsModeChange}
|
||||
ttsAvatarEnabled={ttsAvatarEnabled}
|
||||
onToggleTtsAvatar={handleToggleTtsAvatar}
|
||||
videoEnabled={videoEnabled}
|
||||
onToggleVideo={handleToggleVideo}
|
||||
videoChatMode={videoChatMode}
|
||||
onVideoModeChange={handleVideoModeChange}
|
||||
videoCallAvailable={voiceAvailable && ttsAvailable}
|
||||
onComposioConnected={handleComposioConnected}
|
||||
/>
|
||||
)}
|
||||
{/* Webcam PiP preview while video chat mode is on */}
|
||||
{videoEnabled && (
|
||||
<VideoPreviewOverlay streamRef={video.streamRef} onTurnOff={handleToggleVideo} />
|
||||
{videoChatMode !== 'off' && (
|
||||
<VideoPreviewOverlay
|
||||
streamRef={video.streamRef}
|
||||
onTurnOff={() => handleVideoModeChange('off')}
|
||||
callStatus={
|
||||
videoChatMode === 'call'
|
||||
? tts.state !== 'idle'
|
||||
? 'speaking'
|
||||
: activeIsProcessing
|
||||
? 'thinking'
|
||||
: 'listening'
|
||||
: undefined
|
||||
}
|
||||
interimText={videoChatMode === 'call' ? voice.interimText : undefined}
|
||||
/>
|
||||
)}
|
||||
{/* Talking head hovers over the active view while avatar voice mode is
|
||||
on (hidden during the tour, which shows its own mascot) */}
|
||||
|
|
|
|||
|
|
@ -239,9 +239,12 @@ interface ChatInputInnerProps {
|
|||
onTtsModeChange?: (mode: 'summary' | 'full') => void
|
||||
ttsAvatarEnabled?: boolean
|
||||
onToggleTtsAvatar?: () => void
|
||||
/** Video chat mode: live webcam frames are attached to each message. */
|
||||
videoEnabled?: boolean
|
||||
onToggleVideo?: () => void
|
||||
/** Video chat mode: 'chat' attaches webcam frames to messages; 'call' is
|
||||
* fully hands-free — continuous listening, spoken responses. */
|
||||
videoChatMode?: 'off' | 'chat' | 'call'
|
||||
onVideoModeChange?: (mode: 'off' | 'chat' | 'call') => void
|
||||
/** Hands-free call needs both voice input (STT) and voice output (TTS). */
|
||||
videoCallAvailable?: boolean
|
||||
/** Fired when the user picks a different model in the dropdown (only when no run exists yet). */
|
||||
onSelectedModelChange?: (model: SelectedModel | null) => void
|
||||
/** Work directory for this chat (per-chat). Null when none is set. */
|
||||
|
|
@ -282,8 +285,9 @@ function ChatInputInner({
|
|||
onTtsModeChange,
|
||||
ttsAvatarEnabled,
|
||||
onToggleTtsAvatar,
|
||||
videoEnabled,
|
||||
onToggleVideo,
|
||||
videoChatMode = 'off',
|
||||
onVideoModeChange,
|
||||
videoCallAvailable,
|
||||
onSelectedModelChange,
|
||||
workDir = null,
|
||||
onWorkDirChange,
|
||||
|
|
@ -1356,32 +1360,61 @@ function ChatInputInner({
|
|||
</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
{onToggleVideo && (
|
||||
<Tooltip delayDuration={CHAT_INPUT_TOOLTIP_DELAY_MS}>
|
||||
<TooltipTrigger asChild>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggleVideo}
|
||||
className={cn(
|
||||
'relative flex h-7 w-7 shrink-0 items-center justify-center rounded-full transition-colors',
|
||||
videoEnabled
|
||||
? 'text-red-500 hover:bg-muted'
|
||||
: 'text-muted-foreground hover:bg-muted hover:text-foreground'
|
||||
)}
|
||||
aria-label={videoEnabled ? 'Turn off video chat' : 'Turn on video chat'}
|
||||
>
|
||||
<Video className="h-4 w-4" />
|
||||
{!videoEnabled && (
|
||||
<span className="absolute inset-0 flex items-center justify-center pointer-events-none">
|
||||
<span className="block h-[1.5px] w-5 -rotate-45 rounded-full bg-muted-foreground" />
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="top">
|
||||
{videoEnabled ? 'Video chat on — frames are shared with the assistant' : 'Video chat off'}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
{onVideoModeChange && (
|
||||
<div className="flex shrink-0 items-center">
|
||||
<Tooltip delayDuration={CHAT_INPUT_TOOLTIP_DELAY_MS}>
|
||||
<TooltipTrigger asChild>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onVideoModeChange(videoChatMode === 'off' ? 'chat' : 'off')}
|
||||
className={cn(
|
||||
'relative flex h-7 w-7 shrink-0 items-center justify-center rounded-full transition-colors',
|
||||
videoChatMode !== 'off'
|
||||
? 'text-red-500 hover:bg-muted'
|
||||
: 'text-muted-foreground hover:bg-muted hover:text-foreground'
|
||||
)}
|
||||
aria-label={videoChatMode === 'off' ? 'Turn on video chat' : 'Turn off video chat'}
|
||||
>
|
||||
<Video className="h-4 w-4" />
|
||||
{videoChatMode === 'off' && (
|
||||
<span className="absolute inset-0 flex items-center justify-center pointer-events-none">
|
||||
<span className="block h-[1.5px] w-5 -rotate-45 rounded-full bg-muted-foreground" />
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="top">
|
||||
{videoChatMode === 'off'
|
||||
? 'Video chat off'
|
||||
: videoChatMode === 'chat'
|
||||
? 'Video chat on — camera frames are shared with the assistant'
|
||||
: 'Video call on — speak freely, the assistant answers out loud'}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
{videoChatMode !== 'off' && (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<button
|
||||
type="button"
|
||||
className="flex h-7 w-4 shrink-0 items-center justify-center text-muted-foreground transition-colors hover:text-foreground"
|
||||
>
|
||||
<ChevronDown className="h-3 w-3" />
|
||||
</button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end">
|
||||
<DropdownMenuRadioGroup
|
||||
value={videoChatMode}
|
||||
onValueChange={(v) => onVideoModeChange(v as 'chat' | 'call')}
|
||||
>
|
||||
<DropdownMenuRadioItem value="chat">Video + chat</DropdownMenuRadioItem>
|
||||
<DropdownMenuRadioItem value="call" disabled={!videoCallAvailable}>
|
||||
Video call (hands-free)
|
||||
</DropdownMenuRadioItem>
|
||||
</DropdownMenuRadioGroup>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{voiceAvailable && onStartRecording && (
|
||||
<button
|
||||
|
|
@ -1559,8 +1592,9 @@ export interface ChatInputWithMentionsProps {
|
|||
onTtsModeChange?: (mode: 'summary' | 'full') => void
|
||||
ttsAvatarEnabled?: boolean
|
||||
onToggleTtsAvatar?: () => void
|
||||
videoEnabled?: boolean
|
||||
onToggleVideo?: () => void
|
||||
videoChatMode?: 'off' | 'chat' | 'call'
|
||||
onVideoModeChange?: (mode: 'off' | 'chat' | 'call') => void
|
||||
videoCallAvailable?: boolean
|
||||
onSelectedModelChange?: (model: SelectedModel | null) => void
|
||||
workDir?: string | null
|
||||
onWorkDirChange?: (value: string | null) => void
|
||||
|
|
@ -1597,8 +1631,9 @@ export function ChatInputWithMentions({
|
|||
onTtsModeChange,
|
||||
ttsAvatarEnabled,
|
||||
onToggleTtsAvatar,
|
||||
videoEnabled,
|
||||
onToggleVideo,
|
||||
videoChatMode,
|
||||
onVideoModeChange,
|
||||
videoCallAvailable,
|
||||
onSelectedModelChange,
|
||||
workDir,
|
||||
onWorkDirChange,
|
||||
|
|
@ -1632,8 +1667,9 @@ export function ChatInputWithMentions({
|
|||
onTtsModeChange={onTtsModeChange}
|
||||
ttsAvatarEnabled={ttsAvatarEnabled}
|
||||
onToggleTtsAvatar={onToggleTtsAvatar}
|
||||
videoEnabled={videoEnabled}
|
||||
onToggleVideo={onToggleVideo}
|
||||
videoChatMode={videoChatMode}
|
||||
onVideoModeChange={onVideoModeChange}
|
||||
videoCallAvailable={videoCallAvailable}
|
||||
onSelectedModelChange={onSelectedModelChange}
|
||||
workDir={workDir}
|
||||
onWorkDirChange={onWorkDirChange}
|
||||
|
|
|
|||
|
|
@ -194,8 +194,9 @@ interface ChatSidebarProps {
|
|||
onTtsModeChange?: (mode: 'summary' | 'full') => void
|
||||
ttsAvatarEnabled?: boolean
|
||||
onToggleTtsAvatar?: () => void
|
||||
videoEnabled?: boolean
|
||||
onToggleVideo?: () => void
|
||||
videoChatMode?: 'off' | 'chat' | 'call'
|
||||
onVideoModeChange?: (mode: 'off' | 'chat' | 'call') => void
|
||||
videoCallAvailable?: boolean
|
||||
onComposioConnected?: (toolkitSlug: string) => void
|
||||
}
|
||||
|
||||
|
|
@ -262,8 +263,9 @@ export function ChatSidebar({
|
|||
onTtsModeChange,
|
||||
ttsAvatarEnabled,
|
||||
onToggleTtsAvatar,
|
||||
videoEnabled,
|
||||
onToggleVideo,
|
||||
videoChatMode,
|
||||
onVideoModeChange,
|
||||
videoCallAvailable,
|
||||
onComposioConnected,
|
||||
}: ChatSidebarProps) {
|
||||
const { state: sidebarState } = useSidebar()
|
||||
|
|
@ -840,8 +842,9 @@ export function ChatSidebar({
|
|||
onTtsModeChange={isActive ? onTtsModeChange : undefined}
|
||||
ttsAvatarEnabled={ttsAvatarEnabled}
|
||||
onToggleTtsAvatar={isActive ? onToggleTtsAvatar : undefined}
|
||||
videoEnabled={videoEnabled}
|
||||
onToggleVideo={isActive ? onToggleVideo : undefined}
|
||||
videoChatMode={videoChatMode}
|
||||
onVideoModeChange={isActive ? onVideoModeChange : undefined}
|
||||
videoCallAvailable={videoCallAvailable}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,16 @@ interface VideoPreviewOverlayProps {
|
|||
/** Live camera stream from useVideoMode — attached to the preview element. */
|
||||
streamRef: React.MutableRefObject<MediaStream | null>
|
||||
onTurnOff: () => void
|
||||
/** Hands-free call mode: current phase of the conversation loop. */
|
||||
callStatus?: 'listening' | 'thinking' | 'speaking'
|
||||
/** Hands-free call mode: live transcript of the in-progress utterance. */
|
||||
interimText?: string
|
||||
}
|
||||
|
||||
const CALL_STATUS_DISPLAY: Record<NonNullable<VideoPreviewOverlayProps['callStatus']>, { label: string; dotClass: string }> = {
|
||||
listening: { label: 'Listening', dotClass: 'bg-green-500 animate-pulse' },
|
||||
thinking: { label: 'Thinking…', dotClass: 'bg-amber-400' },
|
||||
speaking: { label: 'Speaking', dotClass: 'bg-sky-400 animate-pulse' },
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -12,7 +22,7 @@ interface VideoPreviewOverlayProps {
|
|||
* on. Mirrored like a selfie camera so the user's movements feel natural.
|
||||
* Sits above the composer dock, mirroring the talking-head overlay's corner.
|
||||
*/
|
||||
export function VideoPreviewOverlay({ streamRef, onTurnOff }: VideoPreviewOverlayProps) {
|
||||
export function VideoPreviewOverlay({ streamRef, onTurnOff, callStatus, interimText }: VideoPreviewOverlayProps) {
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
|
|
@ -52,9 +62,25 @@ export function VideoPreviewOverlay({ streamRef, onTurnOff }: VideoPreviewOverla
|
|||
style={{ transform: 'scaleX(-1)' }}
|
||||
/>
|
||||
<span className="pointer-events-none absolute bottom-1.5 left-1.5 flex items-center gap-1 rounded-full bg-black/60 px-1.5 py-0.5 text-[10px] font-medium text-white">
|
||||
<span className="block h-1.5 w-1.5 animate-pulse rounded-full bg-red-500" />
|
||||
Video on
|
||||
{callStatus ? (
|
||||
<>
|
||||
<span className={`block h-1.5 w-1.5 rounded-full ${CALL_STATUS_DISPLAY[callStatus].dotClass}`} />
|
||||
{CALL_STATUS_DISPLAY[callStatus].label}
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<span className="block h-1.5 w-1.5 animate-pulse rounded-full bg-red-500" />
|
||||
Video on
|
||||
</>
|
||||
)}
|
||||
</span>
|
||||
{callStatus && interimText && (
|
||||
<div className="pointer-events-none absolute inset-x-0 -bottom-1 translate-y-full pt-1">
|
||||
<div className="max-h-16 overflow-hidden rounded-lg bg-black/60 px-2 py-1 text-[11px] leading-snug text-white/90">
|
||||
{interimText}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,25 @@ const DEEPGRAM_PARAMS = new URLSearchParams({
|
|||
endpointing: '100',
|
||||
no_delay: 'true',
|
||||
});
|
||||
const DEEPGRAM_LISTEN_URL = `wss://api.deepgram.com/v1/listen?${DEEPGRAM_PARAMS.toString()}`;
|
||||
// Hands-free (continuous) mode uses a much longer endpoint than push-to-talk:
|
||||
// speech_final fires only after this much silence, so a thinking pause doesn't
|
||||
// cut the user off mid-sentence during a call.
|
||||
const CONTINUOUS_ENDPOINTING_MS = 1800;
|
||||
// While the mic is paused (assistant speaking), keep the idle Deepgram socket
|
||||
// alive — it closes after ~10s without audio otherwise.
|
||||
const KEEPALIVE_INTERVAL_MS = 5000;
|
||||
|
||||
function deepgramParams(continuous: boolean): URLSearchParams {
|
||||
if (!continuous) return DEEPGRAM_PARAMS;
|
||||
const params = new URLSearchParams(DEEPGRAM_PARAMS);
|
||||
params.set('endpointing', String(CONTINUOUS_ENDPOINTING_MS));
|
||||
// Second end-of-speech signal: speech_final can be missed (it often rides
|
||||
// on a result with an empty transcript, or never fires when background
|
||||
// noise keeps the endpointer engaged). UtteranceEnd is word-timing based
|
||||
// and arrives as its own message type, so we listen for both.
|
||||
params.set('utterance_end_ms', '2000');
|
||||
return params;
|
||||
}
|
||||
|
||||
// Cap on retained per-frame amplitude samples (~64ms/frame ⇒ ~5 min of history).
|
||||
// The waveform only ever displays the most recent window, so older samples are dropped.
|
||||
|
|
@ -53,6 +71,11 @@ export function useVoiceMode() {
|
|||
const audioLevelsRef = useRef<number[]>([]);
|
||||
// Running peak amplitude for the waveform auto-gain (see PEAK_DECAY/MIN_PEAK).
|
||||
const audioPeakRef = useRef(0);
|
||||
// Hands-free mode: invoked with each completed utterance (speech_final).
|
||||
const continuousCbRef = useRef<((text: string) => void) | null>(null);
|
||||
// While true (assistant is speaking), mic audio is dropped instead of streamed.
|
||||
const pausedRef = useRef(false);
|
||||
const keepAliveTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
|
||||
// Refresh cached auth details (called on warmup, not on mic click)
|
||||
const refreshAuth = useCallback(async () => {
|
||||
|
|
@ -71,9 +94,22 @@ export function useVoiceMode() {
|
|||
}
|
||||
}, [refreshRowboatAccount]);
|
||||
|
||||
// Hands-free mode: flush the accumulated utterance to the callback.
|
||||
// Called on either end-of-speech signal (speech_final or UtteranceEnd);
|
||||
// both may fire for the same utterance — the second finds an empty
|
||||
// buffer and is a no-op.
|
||||
const fireContinuousUtterance = useCallback(() => {
|
||||
if (!continuousCbRef.current || pausedRef.current) return;
|
||||
const utterance = transcriptBufferRef.current.trim();
|
||||
transcriptBufferRef.current = '';
|
||||
interimRef.current = '';
|
||||
setInterimText('');
|
||||
if (utterance) continuousCbRef.current(utterance);
|
||||
}, []);
|
||||
|
||||
// Create and connect a Deepgram WebSocket using cached auth.
|
||||
// Starts the connection and returns immediately (does not wait for open).
|
||||
const connectWs = useCallback(async () => {
|
||||
const connectWs = useCallback(async (continuous = false) => {
|
||||
if (wsRef.current && (wsRef.current.readyState === WebSocket.OPEN || wsRef.current.readyState === WebSocket.CONNECTING)) return;
|
||||
|
||||
// Refresh auth if we don't have it cached yet
|
||||
|
|
@ -82,12 +118,13 @@ export function useVoiceMode() {
|
|||
}
|
||||
if (!cachedAuth) return;
|
||||
|
||||
const params = deepgramParams(continuous);
|
||||
let ws: WebSocket;
|
||||
if (cachedAuth.type === 'rowboat') {
|
||||
const listenUrl = buildDeepgramListenUrl(cachedAuth.url, DEEPGRAM_PARAMS);
|
||||
const listenUrl = buildDeepgramListenUrl(cachedAuth.url, params);
|
||||
ws = new WebSocket(listenUrl, ['bearer', cachedAuth.token]);
|
||||
} else {
|
||||
ws = new WebSocket(DEEPGRAM_LISTEN_URL, ['token', cachedAuth.apiKey]);
|
||||
ws = new WebSocket(`wss://api.deepgram.com/v1/listen?${params.toString()}`, ['token', cachedAuth.apiKey]);
|
||||
}
|
||||
wsRef.current = ws;
|
||||
|
||||
|
|
@ -103,16 +140,35 @@ export function useVoiceMode() {
|
|||
|
||||
ws.onmessage = (event) => {
|
||||
const data = JSON.parse(event.data);
|
||||
if (!data.channel?.alternatives?.[0]) return;
|
||||
|
||||
// Hands-free mode: word-timing based end-of-speech marker.
|
||||
if (data.type === 'UtteranceEnd') {
|
||||
fireContinuousUtterance();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!data.channel?.alternatives?.[0]) return;
|
||||
const transcript = data.channel.alternatives[0].transcript;
|
||||
if (!transcript) return;
|
||||
|
||||
if (data.is_final) {
|
||||
transcriptBufferRef.current += (transcriptBufferRef.current ? ' ' : '') + transcript;
|
||||
interimRef.current = '';
|
||||
setInterimText(transcriptBufferRef.current);
|
||||
} else {
|
||||
// NOTE: the endpoint marker (speech_final) usually arrives on a
|
||||
// result whose transcript is EMPTY — the silence after the user
|
||||
// stops talking. Empty finals must still reach the speech_final
|
||||
// check below or hands-free utterances never complete.
|
||||
if (transcript) {
|
||||
transcriptBufferRef.current += (transcriptBufferRef.current ? ' ' : '') + transcript;
|
||||
interimRef.current = '';
|
||||
}
|
||||
// Hands-free mode: an endpoint completes the utterance — hand
|
||||
// it off and reset for the next one.
|
||||
if (continuousCbRef.current && data.speech_final) {
|
||||
fireContinuousUtterance();
|
||||
return;
|
||||
}
|
||||
if (transcript) {
|
||||
setInterimText(transcriptBufferRef.current);
|
||||
}
|
||||
} else if (transcript) {
|
||||
interimRef.current = transcript;
|
||||
setInterimText(transcriptBufferRef.current + (transcriptBufferRef.current ? ' ' : '') + transcript);
|
||||
}
|
||||
|
|
@ -127,8 +183,17 @@ export function useVoiceMode() {
|
|||
ws.onclose = () => {
|
||||
console.log('[voice] WebSocket closed');
|
||||
wsRef.current = null;
|
||||
// A hands-free call is long-lived — if the socket drops while the
|
||||
// call is still on, reconnect instead of silently going deaf.
|
||||
if (continuousCbRef.current) {
|
||||
setTimeout(() => {
|
||||
if (continuousCbRef.current && !wsRef.current) {
|
||||
void connectWs(true);
|
||||
}
|
||||
}, 1000);
|
||||
}
|
||||
};
|
||||
}, [refreshAuth]);
|
||||
}, [refreshAuth, fireContinuousUtterance]);
|
||||
|
||||
const waitForWsOpen = useCallback(async (timeoutMs = 1500): Promise<boolean> => {
|
||||
const ws = wsRef.current;
|
||||
|
|
@ -191,6 +256,12 @@ export function useVoiceMode() {
|
|||
wsRef.current.close();
|
||||
wsRef.current = null;
|
||||
}
|
||||
continuousCbRef.current = null;
|
||||
pausedRef.current = false;
|
||||
if (keepAliveTimerRef.current) {
|
||||
clearInterval(keepAliveTimerRef.current);
|
||||
keepAliveTimerRef.current = null;
|
||||
}
|
||||
audioBufferRef.current = [];
|
||||
audioLevelsRef.current = [];
|
||||
audioPeakRef.current = 0;
|
||||
|
|
@ -200,7 +271,7 @@ export function useVoiceMode() {
|
|||
setState('idle');
|
||||
}, [stopInputCapture]);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
const start = useCallback(async (continuous = false) => {
|
||||
if (state !== 'idle') return;
|
||||
|
||||
transcriptBufferRef.current = '';
|
||||
|
|
@ -235,7 +306,7 @@ export function useVoiceMode() {
|
|||
console.error('Microphone access denied:', err);
|
||||
return null;
|
||||
}),
|
||||
connectWs(),
|
||||
connectWs(continuous),
|
||||
]);
|
||||
|
||||
if (!stream) {
|
||||
|
|
@ -259,6 +330,9 @@ export function useVoiceMode() {
|
|||
processorRef.current = processor;
|
||||
|
||||
processor.onaudioprocess = (e) => {
|
||||
// Paused (assistant is speaking in a call): drop mic audio so the
|
||||
// assistant's own TTS never gets transcribed back at it.
|
||||
if (pausedRef.current) return;
|
||||
const float32 = e.inputBuffer.getChannelData(0);
|
||||
const int16 = new Int16Array(float32.length);
|
||||
let sumSquares = 0;
|
||||
|
|
@ -283,8 +357,13 @@ export function useVoiceMode() {
|
|||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send(buffer);
|
||||
} else {
|
||||
// WebSocket still connecting — buffer the audio
|
||||
// WebSocket still connecting (or reconnecting mid-call) —
|
||||
// buffer the audio, bounded so an unreachable server during a
|
||||
// long call can't grow it without limit (~30s at 64ms/chunk).
|
||||
audioBufferRef.current.push(buffer);
|
||||
if (audioBufferRef.current.length > 500) {
|
||||
audioBufferRef.current.shift();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -318,10 +397,42 @@ export function useVoiceMode() {
|
|||
stopAudioCapture();
|
||||
}, [stopAudioCapture]);
|
||||
|
||||
/**
|
||||
* Hands-free (call) mode: listen continuously and invoke `onUtterance`
|
||||
* with each completed utterance. Runs until cancel()/stop.
|
||||
*/
|
||||
const startContinuous = useCallback(async (onUtterance: (text: string) => void) => {
|
||||
continuousCbRef.current = onUtterance;
|
||||
await start(true);
|
||||
}, [start]);
|
||||
|
||||
/**
|
||||
* Mute/unmute the continuous stream (used while the assistant is
|
||||
* thinking/speaking). Keeps the Deepgram socket alive with KeepAlives and
|
||||
* discards any half-heard utterance from before the pause.
|
||||
*/
|
||||
const setPaused = useCallback((paused: boolean) => {
|
||||
if (pausedRef.current === paused) return;
|
||||
pausedRef.current = paused;
|
||||
if (paused) {
|
||||
transcriptBufferRef.current = '';
|
||||
interimRef.current = '';
|
||||
setInterimText('');
|
||||
keepAliveTimerRef.current = setInterval(() => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send(JSON.stringify({ type: 'KeepAlive' }));
|
||||
}
|
||||
}, KEEPALIVE_INTERVAL_MS);
|
||||
} else if (keepAliveTimerRef.current) {
|
||||
clearInterval(keepAliveTimerRef.current);
|
||||
keepAliveTimerRef.current = null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
/** Pre-cache auth details so mic click skips IPC round-trips */
|
||||
const warmup = useCallback(() => {
|
||||
refreshAuth().catch(() => {});
|
||||
}, [refreshAuth]);
|
||||
|
||||
return { state, interimText, audioLevelsRef, start, submit, cancel, warmup };
|
||||
return { state, interimText, audioLevelsRef, start, submit, cancel, warmup, startContinuous, setPaused };
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue