mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-07-12 21:02:17 +02:00
Merge pull request #672 from rowboatlabs/call_mic_mute
Add in-call mic mute: pauses all input (mic audio + frame capture)
This commit is contained in:
arkml
GitHub
commit
36c0cca3f9
7 changed files with 147 additions and 23 deletions
|
|
@ -44,9 +44,19 @@ starts anyway as a voice call, with a toast linking to System Settings.
|
|||
Practice/coaching is always an explicit choice — expanding to full screen
|
||||
never turns the coach on.
|
||||
|
||||
In-call controls (identical bar on both surfaces): camera toggle (silhouette
|
||||
avatar while off, no webcam frames captured), screen share toggle, mascot ⇄
|
||||
"R" letter avatar, end call. While the assistant is thinking or speaking, a
|
||||
In-call controls (identical bar on both surfaces): mic mute, camera toggle
|
||||
(silhouette avatar while off, no webcam frames captured), screen share
|
||||
toggle, mascot ⇄ "R" letter avatar, end call. **Mute is a full input
|
||||
pause**, not just audio — mic audio stops reaching Deepgram
|
||||
(`useVoiceMode.setPaused`, OR'd with the automatic thinking/speaking pause)
|
||||
AND camera/screen frame capture stops (`useVideoMode.setCapturePaused`;
|
||||
`collectFrames()` returns nothing while muted, so typed messages carry no
|
||||
frames either), letting the user talk to someone in the room without the
|
||||
assistant listening in. Devices stay acquired for instant unmute (camera
|
||||
light and macOS share indicator stay on — the pill's share badge switches to
|
||||
"Sharing paused"), the status chip shows "Muted" instead of "Listening",
|
||||
and assistant output is unaffected (in-flight speech keeps playing; Stop
|
||||
handles that). Mute resets to off at call start/end. While the assistant is thinking or speaking, a
|
||||
red **Stop** button appears on the mascot tile — it silences TTS instantly,
|
||||
skips queued voice segments, and aborts the run if it's still generating
|
||||
(stopping a run from anywhere, including the composer, also silences TTS). Captions of the in-progress utterance and the
|
||||
|
|
|
|||
|
|
@ -395,6 +395,7 @@ let lastVideoPopoutState: {
|
|||
ttsState: 'idle' | 'synthesizing' | 'speaking';
|
||||
status: 'listening' | 'thinking' | 'speaking' | null;
|
||||
cameraOn: boolean;
|
||||
micMuted: boolean;
|
||||
screenSharing: boolean;
|
||||
interimText: string | null;
|
||||
} | null = null;
|
||||
|
|
|
|||
|
|
@ -1061,6 +1061,11 @@ function App() {
|
|||
const inCallRef = useRef(false)
|
||||
// User explicitly shrank the full-screen call to the floating pill.
|
||||
const [callMinimized, setCallMinimized] = useState(false)
|
||||
// In-call mute: a full input pause, not just audio — mic audio stops
|
||||
// reaching Deepgram AND camera/screen frame capture stops, so nothing said
|
||||
// or shown while muted ever reaches the assistant. Output is untouched
|
||||
// (in-flight speech keeps playing; the Stop control handles that).
|
||||
const [micMuted, setMicMuted] = useState(false)
|
||||
// Practice preset: adds the coaching persona to the system prompt.
|
||||
const [practiceMode, setPracticeMode] = useState(false)
|
||||
const practiceModeRef = useRef(false)
|
||||
|
|
@ -1200,6 +1205,7 @@ function App() {
|
|||
|
||||
setPracticeMode(preset === 'practice')
|
||||
practiceModeRef.current = preset === 'practice'
|
||||
setMicMuted(false)
|
||||
// Pill-first presets start minimized; face-to-face presets start expanded.
|
||||
setCallMinimized(preset === 'voice' || preset === 'share')
|
||||
inCallRef.current = true
|
||||
|
|
@ -1217,17 +1223,26 @@ function App() {
|
|||
video.stop()
|
||||
setPracticeMode(false)
|
||||
practiceModeRef.current = false
|
||||
setMicMuted(false)
|
||||
setCallMinimized(false)
|
||||
inCallRef.current = false
|
||||
setInCall(false)
|
||||
}, [video])
|
||||
|
||||
// During a call, mute the mic while the assistant is thinking or speaking
|
||||
// so its own TTS (or a half-turn) never gets transcribed back at it.
|
||||
// so its own TTS (or a half-turn) never gets transcribed back at it — and
|
||||
// whenever the user muted themselves.
|
||||
useEffect(() => {
|
||||
if (!inCall) return
|
||||
voiceRef.current.setPaused(activeIsProcessing || tts.state !== 'idle')
|
||||
}, [inCall, activeIsProcessing, tts.state])
|
||||
voiceRef.current.setPaused(micMuted || activeIsProcessing || tts.state !== 'idle')
|
||||
}, [inCall, micMuted, activeIsProcessing, tts.state])
|
||||
|
||||
// The user-mute half that lives in the video pipeline: stop sampling
|
||||
// camera/screen frames while muted (see useVideoMode.setCapturePaused).
|
||||
const setCapturePaused = video.setCapturePaused
|
||||
useEffect(() => {
|
||||
setCapturePaused(micMuted)
|
||||
}, [micMuted, setCapturePaused])
|
||||
|
||||
// Screen sharing: frames of the shared screen ride along with each message
|
||||
// next to the webcam frames. The surface change (full screen → pill) falls
|
||||
|
|
@ -1248,6 +1263,14 @@ function App() {
|
|||
void video.setCameraEnabled(!video.cameraOn)
|
||||
}, [video])
|
||||
|
||||
// Zoom-style mute button, except it pauses ALL input (mic + frames) so the
|
||||
// user can talk to someone in the room without the assistant listening in.
|
||||
// Devices stay acquired (camera light and share indicator stay on) so
|
||||
// unmuting is instant.
|
||||
const handleToggleMic = useCallback(() => {
|
||||
setMicMuted((m) => !m)
|
||||
}, [])
|
||||
|
||||
// Minimizing the full-screen call drops you back to working — and the pill
|
||||
// exists to work *together*, so sharing starts automatically (the symmetric
|
||||
// twin of expand, which stops it). If capture fails (permission), the call
|
||||
|
|
@ -1327,11 +1350,12 @@ function App() {
|
|||
ttsState: tts.state,
|
||||
status: videoCallStatus,
|
||||
cameraOn: video.cameraOn,
|
||||
micMuted,
|
||||
screenSharing: video.screenState === 'live',
|
||||
interimText: voice.interimText || null,
|
||||
})
|
||||
.catch(() => {})
|
||||
}, [inCall, tts.state, videoCallStatus, video.cameraOn, video.screenState, voice.interimText])
|
||||
}, [inCall, tts.state, videoCallStatus, video.cameraOn, micMuted, video.screenState, voice.interimText])
|
||||
|
||||
// Execute popout control-bar actions (the popout window has no access to
|
||||
// the call's mic/camera/capture — they live here). 'expand' goes full
|
||||
|
|
@ -1339,7 +1363,8 @@ function App() {
|
|||
// process already refocused the app window.
|
||||
useEffect(() => {
|
||||
return window.ipc.on('video:popout-action', ({ action }) => {
|
||||
if (action === 'toggle-camera') handleToggleCamera()
|
||||
if (action === 'toggle-mic') handleToggleMic()
|
||||
else if (action === 'toggle-camera') handleToggleCamera()
|
||||
else if (action === 'toggle-share') void handleToggleScreenShare()
|
||||
else if (action === 'stop-speaking') handleInterruptAssistant()
|
||||
else if (action === 'end-call') endCall()
|
||||
|
|
@ -1348,7 +1373,7 @@ function App() {
|
|||
setCallMinimized(false)
|
||||
}
|
||||
})
|
||||
}, [handleToggleCamera, handleToggleScreenShare, handleInterruptAssistant, endCall, video])
|
||||
}, [handleToggleMic, handleToggleCamera, handleToggleScreenShare, handleInterruptAssistant, endCall, video])
|
||||
|
||||
// Enter to submit voice input, Escape to cancel
|
||||
useEffect(() => {
|
||||
|
|
@ -6835,6 +6860,8 @@ function App() {
|
|||
onToggleScreenShare={handleToggleScreenShare}
|
||||
cameraOn={video.cameraOn}
|
||||
onToggleCamera={handleToggleCamera}
|
||||
micMuted={micMuted}
|
||||
onToggleMic={handleToggleMic}
|
||||
practiceMode={practiceMode}
|
||||
onMinimize={() => void handleMinimizeCall()}
|
||||
onInterrupt={handleInterruptAssistant}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { useEffect, useRef, useState } from 'react'
|
||||
import { Minimize2, MonitorUp, PhoneOff, Presentation, Square, User, Video, VideoOff } from 'lucide-react'
|
||||
import { Mic, MicOff, Minimize2, MonitorUp, PhoneOff, Presentation, Square, User, Video, VideoOff } from 'lucide-react'
|
||||
|
||||
import { MascotFaceIcon, TalkingHead } from '@/components/talking-head'
|
||||
import type { TTSState } from '@/hooks/useVoiceTTS'
|
||||
|
|
@ -12,6 +12,9 @@ interface VideoCallViewProps {
|
|||
streamRef: React.MutableRefObject<MediaStream | null>
|
||||
cameraOn: boolean
|
||||
onToggleCamera: () => void
|
||||
/** User mute = full input pause: no mic audio AND no frame capture. */
|
||||
micMuted: boolean
|
||||
onToggleMic: () => void
|
||||
/** Starting a share collapses this view into the floating popout (the
|
||||
* surface is derived from devices — see App.tsx). */
|
||||
onToggleScreenShare: () => void
|
||||
|
|
@ -51,6 +54,8 @@ export function VideoCallView({
|
|||
streamRef,
|
||||
cameraOn,
|
||||
onToggleCamera,
|
||||
micMuted,
|
||||
onToggleMic,
|
||||
onToggleScreenShare,
|
||||
practiceMode,
|
||||
onMinimize,
|
||||
|
|
@ -128,6 +133,12 @@ export function VideoCallView({
|
|||
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
|
||||
You
|
||||
</span>
|
||||
{micMuted && (
|
||||
<span className="absolute bottom-3 right-3 flex items-center gap-1.5 rounded-md bg-red-600/90 px-2 py-0.5 text-sm font-medium text-white">
|
||||
<MicOff className="h-3.5 w-3.5" />
|
||||
Muted — nothing is heard or captured
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Assistant */}
|
||||
|
|
@ -185,9 +196,34 @@ export function VideoCallView({
|
|||
{/* Control bar */}
|
||||
<div className="flex items-center justify-center gap-4 pb-5">
|
||||
<span className="flex items-center gap-2 rounded-full bg-neutral-800 px-3 py-1.5 text-xs font-medium text-white/90">
|
||||
<span className={cn('block h-2 w-2 rounded-full', STATUS_DISPLAY[status].dotClass)} />
|
||||
{STATUS_DISPLAY[status].label}
|
||||
{/* Muted overrides "Listening" — the green pulse would be a lie.
|
||||
Thinking/speaking still show: output continues while muted. */}
|
||||
{micMuted && status === 'listening' ? (
|
||||
<>
|
||||
<span className="block h-2 w-2 rounded-full bg-red-500" />
|
||||
Muted
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<span className={cn('block h-2 w-2 rounded-full', STATUS_DISPLAY[status].dotClass)} />
|
||||
{STATUS_DISPLAY[status].label}
|
||||
</>
|
||||
)}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggleMic}
|
||||
className={cn(
|
||||
'flex h-10 w-10 items-center justify-center rounded-full transition-colors',
|
||||
micMuted
|
||||
? 'bg-red-600 text-white hover:bg-red-500'
|
||||
: 'bg-neutral-800 text-white/90 hover:bg-neutral-700'
|
||||
)}
|
||||
aria-label={micMuted ? 'Unmute' : 'Mute (pauses mic and frame capture)'}
|
||||
title={micMuted ? 'Unmute' : 'Mute — pauses your mic and all frame capture'}
|
||||
>
|
||||
{micMuted ? <MicOff className="h-5 w-5" /> : <Mic className="h-5 w-5" />}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggleCamera}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { Maximize2, MonitorUp, PhoneOff, Square, User, Video, VideoOff } from 'lucide-react'
|
||||
import { Maximize2, Mic, MicOff, MonitorUp, PhoneOff, Square, User, Video, VideoOff } from 'lucide-react'
|
||||
|
||||
import { TalkingHead } from '@/components/talking-head'
|
||||
|
||||
|
|
@ -7,6 +7,8 @@ type PopoutState = {
|
|||
ttsState: 'idle' | 'synthesizing' | 'speaking'
|
||||
status: 'listening' | 'thinking' | 'speaking' | null
|
||||
cameraOn: boolean
|
||||
/** User mute = full input pause: no mic audio AND no frame capture. */
|
||||
micMuted: boolean
|
||||
screenSharing: boolean
|
||||
interimText: string | null
|
||||
}
|
||||
|
|
@ -34,7 +36,7 @@ export function VideoPopout() {
|
|||
// Camera defaults OFF: guessing "on" would flash the user's video for a
|
||||
// beat before the real state arrives — which reads as a bug. The true
|
||||
// state is fetched immediately below.
|
||||
const [state, setState] = useState<PopoutState>({ ttsState: 'idle', status: null, cameraOn: false, screenSharing: false, interimText: null })
|
||||
const [state, setState] = useState<PopoutState>({ ttsState: 'idle', status: null, cameraOn: false, micMuted: false, screenSharing: false, interimText: null })
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
|
|
@ -80,7 +82,7 @@ export function VideoPopout() {
|
|||
// so the mascot still animates while the assistant speaks in the main window.
|
||||
const getLevel = useCallback(() => 0.45 + 0.35 * Math.sin(performance.now() / 90), [])
|
||||
|
||||
const sendAction = useCallback((action: 'toggle-camera' | 'toggle-share' | 'stop-speaking' | 'end-call' | 'expand') => {
|
||||
const sendAction = useCallback((action: 'toggle-mic' | 'toggle-camera' | 'toggle-share' | 'stop-speaking' | 'end-call' | 'expand') => {
|
||||
void window.ipc.invoke('video:popoutAction', { action }).catch(() => {})
|
||||
}, [])
|
||||
|
||||
|
|
@ -112,11 +114,18 @@ export function VideoPopout() {
|
|||
You
|
||||
</span>
|
||||
{/* Persistent consent badge — the user must always be able to see
|
||||
at a glance that their screen is going out. */}
|
||||
at a glance that their screen is going out. Muted pauses frame
|
||||
capture while keeping the share stream open, so say so. */}
|
||||
{state.screenSharing && (
|
||||
<span className="absolute left-1.5 top-1.5 flex items-center gap-1 rounded-full bg-sky-600/90 px-1.5 py-0.5 text-[10px] font-medium text-white">
|
||||
<span className="block h-1.5 w-1.5 animate-pulse rounded-full bg-white" />
|
||||
Sharing screen
|
||||
<span className={`block h-1.5 w-1.5 rounded-full bg-white ${state.micMuted ? '' : 'animate-pulse'}`} />
|
||||
{state.micMuted ? 'Sharing paused' : 'Sharing screen'}
|
||||
</span>
|
||||
)}
|
||||
{state.micMuted && (
|
||||
<span className="absolute bottom-1 right-1.5 flex items-center gap-1 rounded bg-red-600/90 px-1.5 py-0.5 text-[10px] font-medium text-white">
|
||||
<MicOff className="h-2.5 w-2.5" />
|
||||
Muted
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
|
@ -127,8 +136,18 @@ export function VideoPopout() {
|
|||
</span>
|
||||
{statusDisplay && (
|
||||
<span className="absolute right-1.5 top-1.5 flex items-center gap-1 rounded-full bg-black/50 px-1.5 py-0.5 text-[10px] font-medium text-white">
|
||||
<span className={`block h-1.5 w-1.5 rounded-full ${statusDisplay.dotClass}`} />
|
||||
{statusDisplay.label}
|
||||
{/* Muted overrides "Listening" — the green pulse would be a lie. */}
|
||||
{state.micMuted && state.status === 'listening' ? (
|
||||
<>
|
||||
<span className="block h-1.5 w-1.5 rounded-full bg-red-500" />
|
||||
Muted
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<span className={`block h-1.5 w-1.5 rounded-full ${statusDisplay.dotClass}`} />
|
||||
{statusDisplay.label}
|
||||
</>
|
||||
)}
|
||||
</span>
|
||||
)}
|
||||
{(state.status === 'speaking' || state.status === 'thinking') && (
|
||||
|
|
@ -157,6 +176,19 @@ export function VideoPopout() {
|
|||
|
||||
{/* Control bar — actions execute in the main app window */}
|
||||
<div className="flex h-7 shrink-0 items-center justify-center gap-2" style={noDragRegion}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => sendAction('toggle-mic')}
|
||||
className={`flex h-6 w-6 items-center justify-center rounded-full transition-colors ${
|
||||
state.micMuted
|
||||
? 'bg-red-600 text-white hover:bg-red-500'
|
||||
: 'bg-neutral-700 text-white/90 hover:bg-neutral-600'
|
||||
}`}
|
||||
aria-label={state.micMuted ? 'Unmute' : 'Mute (pauses mic and frame capture)'}
|
||||
title={state.micMuted ? 'Unmute' : 'Mute — pauses your mic and all frame capture'}
|
||||
>
|
||||
{state.micMuted ? <MicOff className="h-3.5 w-3.5" /> : <Mic className="h-3.5 w-3.5" />}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => sendAction('toggle-camera')}
|
||||
|
|
|
|||
|
|
@ -154,6 +154,10 @@ export function useVideoMode() {
|
|||
// Camera can be turned off mid-session (Meet-style) while the mode — and
|
||||
// any screen share — keeps running. Resets to on for the next session.
|
||||
const [cameraOn, setCameraOn] = useState(true);
|
||||
// In-call mute pauses capture entirely: nothing lands in the ring buffers
|
||||
// and collectFrames() returns nothing, so a muted stretch can never ride
|
||||
// along with a later message. Streams stay open for instant resume.
|
||||
const capturePausedRef = useRef(false);
|
||||
const cameraPipeRef = useRef<CapturePipe>(emptyPipe());
|
||||
const screenPipeRef = useRef<CapturePipe>(emptyPipe());
|
||||
// Stable stream refs for preview components (<video srcObject>).
|
||||
|
|
@ -165,13 +169,19 @@ export function useVideoMode() {
|
|||
screenStateRef.current = screenState;
|
||||
|
||||
const captureCameraFrame = useCallback(() => {
|
||||
if (capturePausedRef.current) return;
|
||||
capturePipeFrame(cameraPipeRef.current, CAMERA_FRAME_WIDTH, CAMERA_JPEG_QUALITY);
|
||||
}, []);
|
||||
|
||||
const captureScreenFrame = useCallback(() => {
|
||||
if (capturePausedRef.current) return;
|
||||
capturePipeFrame(screenPipeRef.current, SCREEN_FRAME_WIDTH, SCREEN_JPEG_QUALITY);
|
||||
}, []);
|
||||
|
||||
const setCapturePaused = useCallback((paused: boolean) => {
|
||||
capturePausedRef.current = paused;
|
||||
}, []);
|
||||
|
||||
const stopScreenShare = useCallback(() => {
|
||||
teardownPipe(screenPipeRef.current);
|
||||
screenStreamRef.current = null;
|
||||
|
|
@ -183,6 +193,7 @@ export function useVideoMode() {
|
|||
streamRef.current = null;
|
||||
setState('idle');
|
||||
setCameraOn(true);
|
||||
capturePausedRef.current = false;
|
||||
stopScreenShare();
|
||||
}, [stopScreenShare]);
|
||||
|
||||
|
|
@ -295,6 +306,9 @@ export function useVideoMode() {
|
|||
*/
|
||||
const collectFrames = useCallback((): CapturedVideoFrame[] => {
|
||||
if (stateRef.current !== 'live') return [];
|
||||
// Muted: no frames at all — not even pre-mute buffered ones — so a
|
||||
// typed message during a mute carries nothing captured around it.
|
||||
if (capturePausedRef.current) return [];
|
||||
// Grab a frame right now so the message always includes the moment of send.
|
||||
captureCameraFrame();
|
||||
const frames = drainPipe(cameraPipeRef.current, MAX_CAMERA_FRAMES_PER_MESSAGE, 'camera');
|
||||
|
|
@ -308,5 +322,5 @@ export function useVideoMode() {
|
|||
// Release the camera/screen if the component unmounts with video mode on.
|
||||
useEffect(() => stop, [stop]);
|
||||
|
||||
return { state, screenState, cameraOn, streamRef, screenStreamRef, start, stop, startScreenShare, stopScreenShare, setCameraEnabled, collectFrames };
|
||||
return { state, screenState, cameraOn, streamRef, screenStreamRef, start, stop, startScreenShare, stopScreenShare, setCameraEnabled, setCapturePaused, collectFrames };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1465,6 +1465,8 @@ const ipcSchemas = {
|
|||
ttsState: z.enum(['idle', 'synthesizing', 'speaking']),
|
||||
status: z.enum(['listening', 'thinking', 'speaking']).nullable(),
|
||||
cameraOn: z.boolean(),
|
||||
// User mute: mic audio and frame capture are both paused.
|
||||
micMuted: z.boolean(),
|
||||
screenSharing: z.boolean(),
|
||||
// Live transcript of the in-progress utterance.
|
||||
interimText: z.string().nullable(),
|
||||
|
|
@ -1483,6 +1485,7 @@ const ipcSchemas = {
|
|||
ttsState: z.enum(['idle', 'synthesizing', 'speaking']),
|
||||
status: z.enum(['listening', 'thinking', 'speaking']).nullable(),
|
||||
cameraOn: z.boolean(),
|
||||
micMuted: z.boolean(),
|
||||
screenSharing: z.boolean(),
|
||||
interimText: z.string().nullable(),
|
||||
})
|
||||
|
|
@ -1494,7 +1497,7 @@ const ipcSchemas = {
|
|||
// main app window (handled in the main process).
|
||||
'video:popoutAction': {
|
||||
req: z.object({
|
||||
action: z.enum(['toggle-camera', 'toggle-share', 'stop-speaking', 'end-call', 'expand']),
|
||||
action: z.enum(['toggle-mic', 'toggle-camera', 'toggle-share', 'stop-speaking', 'end-call', 'expand']),
|
||||
}),
|
||||
res: z.object({}),
|
||||
},
|
||||
|
|
@ -1504,6 +1507,7 @@ const ipcSchemas = {
|
|||
ttsState: z.enum(['idle', 'synthesizing', 'speaking']),
|
||||
status: z.enum(['listening', 'thinking', 'speaking']).nullable(),
|
||||
cameraOn: z.boolean(),
|
||||
micMuted: z.boolean(),
|
||||
screenSharing: z.boolean(),
|
||||
interimText: z.string().nullable(),
|
||||
}),
|
||||
|
|
@ -1512,7 +1516,7 @@ const ipcSchemas = {
|
|||
// Push channel: main → app window with a popout control-bar action.
|
||||
'video:popout-action': {
|
||||
req: z.object({
|
||||
action: z.enum(['toggle-camera', 'toggle-share', 'stop-speaking', 'end-call', 'expand']),
|
||||
action: z.enum(['toggle-mic', 'toggle-camera', 'toggle-share', 'stop-speaking', 'end-call', 'expand']),
|
||||
}),
|
||||
res: z.null(),
|
||||
},
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue