Add in-call mic mute: pauses all input (mic audio + frame capture)

A mute button on both call surfaces (full-screen call and floating
popout) that pauses everything going to the assistant while keeping the
call alive — for stepping away to talk to someone in the room without
ending and restarting the call.

- Mic audio stops reaching Deepgram (user mute OR'd into the existing
  thinking/speaking setPaused; KeepAlives keep the socket warm so
  unmute is instant)
- Camera/screen frames stop being sampled (new setCapturePaused in
  useVideoMode); collectFrames() returns nothing while muted, so typed
  messages during a mute carry no frames either
- Devices stay acquired for instant resume; mute resets at call
  start/end; assistant output is unaffected (Stop handles that)
- Honest UI: "Muted" status chip replaces the green "Listening" pulse,
  muted badge on the user tile, pill's share badge flips to "Sharing
  paused"; new toggle-mic popout action + micMuted in popout state

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Arjun 2026-07-06 00:28:34 +05:30
parent 1c380c33ed
commit adb8b16a3c
7 changed files with 147 additions and 23 deletions

View file

@ -44,9 +44,19 @@ starts anyway as a voice call, with a toast linking to System Settings.
Practice/coaching is always an explicit choice — expanding to full screen
never turns the coach on.
In-call controls (identical bar on both surfaces): camera toggle (silhouette
avatar while off, no webcam frames captured), screen share toggle, mascot ⇄
"R" letter avatar, end call. While the assistant is thinking or speaking, a
In-call controls (identical bar on both surfaces): mic mute, camera toggle
(silhouette avatar while off, no webcam frames captured), screen share
toggle, mascot ⇄ "R" letter avatar, end call. **Mute is a full input
pause**, not just audio — mic audio stops reaching Deepgram
(`useVoiceMode.setPaused`, OR'd with the automatic thinking/speaking pause)
AND camera/screen frame capture stops (`useVideoMode.setCapturePaused`;
`collectFrames()` returns nothing while muted, so typed messages carry no
frames either), letting the user talk to someone in the room without the
assistant listening in. Devices stay acquired for instant unmute (camera
light and macOS share indicator stay on — the pill's share badge switches to
"Sharing paused"), the status chip shows "Muted" instead of "Listening",
and assistant output is unaffected (in-flight speech keeps playing; Stop
handles that). Mute resets to off at call start/end. While the assistant is thinking or speaking, a
red **Stop** button appears on the mascot tile — it silences TTS instantly,
skips queued voice segments, and aborts the run if it's still generating
(stopping a run from anywhere, including the composer, also silences TTS). Captions of the in-progress utterance and the

View file

@ -395,6 +395,7 @@ let lastVideoPopoutState: {
ttsState: 'idle' | 'synthesizing' | 'speaking';
status: 'listening' | 'thinking' | 'speaking' | null;
cameraOn: boolean;
micMuted: boolean;
screenSharing: boolean;
interimText: string | null;
} | null = null;

View file

@ -1061,6 +1061,11 @@ function App() {
const inCallRef = useRef(false)
// User explicitly shrank the full-screen call to the floating pill.
const [callMinimized, setCallMinimized] = useState(false)
// In-call mute: a full input pause, not just audio — mic audio stops
// reaching Deepgram AND camera/screen frame capture stops, so nothing said
// or shown while muted ever reaches the assistant. Output is untouched
// (in-flight speech keeps playing; the Stop control handles that).
const [micMuted, setMicMuted] = useState(false)
// Practice preset: adds the coaching persona to the system prompt.
const [practiceMode, setPracticeMode] = useState(false)
const practiceModeRef = useRef(false)
@ -1200,6 +1205,7 @@ function App() {
setPracticeMode(preset === 'practice')
practiceModeRef.current = preset === 'practice'
setMicMuted(false)
// Pill-first presets start minimized; face-to-face presets start expanded.
setCallMinimized(preset === 'voice' || preset === 'share')
inCallRef.current = true
@ -1217,17 +1223,26 @@ function App() {
video.stop()
setPracticeMode(false)
practiceModeRef.current = false
setMicMuted(false)
setCallMinimized(false)
inCallRef.current = false
setInCall(false)
}, [video])
// During a call, mute the mic while the assistant is thinking or speaking
// so its own TTS (or a half-turn) never gets transcribed back at it.
// so its own TTS (or a half-turn) never gets transcribed back at it — and
// whenever the user muted themselves.
useEffect(() => {
if (!inCall) return
voiceRef.current.setPaused(activeIsProcessing || tts.state !== 'idle')
}, [inCall, activeIsProcessing, tts.state])
voiceRef.current.setPaused(micMuted || activeIsProcessing || tts.state !== 'idle')
}, [inCall, micMuted, activeIsProcessing, tts.state])
// The user-mute half that lives in the video pipeline: stop sampling
// camera/screen frames while muted (see useVideoMode.setCapturePaused).
const setCapturePaused = video.setCapturePaused
useEffect(() => {
setCapturePaused(micMuted)
}, [micMuted, setCapturePaused])
// Screen sharing: frames of the shared screen ride along with each message
// next to the webcam frames. The surface change (full screen → pill) falls
@ -1248,6 +1263,14 @@ function App() {
void video.setCameraEnabled(!video.cameraOn)
}, [video])
// Zoom-style mute button, except it pauses ALL input (mic + frames) so the
// user can talk to someone in the room without the assistant listening in.
// Devices stay acquired (camera light and share indicator stay on) so
// unmuting is instant.
const handleToggleMic = useCallback(() => {
setMicMuted((m) => !m)
}, [])
// Minimizing the full-screen call drops you back to working — and the pill
// exists to work *together*, so sharing starts automatically (the symmetric
// twin of expand, which stops it). If capture fails (permission), the call
@ -1327,11 +1350,12 @@ function App() {
ttsState: tts.state,
status: videoCallStatus,
cameraOn: video.cameraOn,
micMuted,
screenSharing: video.screenState === 'live',
interimText: voice.interimText || null,
})
.catch(() => {})
}, [inCall, tts.state, videoCallStatus, video.cameraOn, video.screenState, voice.interimText])
}, [inCall, tts.state, videoCallStatus, video.cameraOn, micMuted, video.screenState, voice.interimText])
// Execute popout control-bar actions (the popout window has no access to
// the call's mic/camera/capture — they live here). 'expand' goes full
@ -1339,7 +1363,8 @@ function App() {
// process already refocused the app window.
useEffect(() => {
return window.ipc.on('video:popout-action', ({ action }) => {
if (action === 'toggle-camera') handleToggleCamera()
if (action === 'toggle-mic') handleToggleMic()
else if (action === 'toggle-camera') handleToggleCamera()
else if (action === 'toggle-share') void handleToggleScreenShare()
else if (action === 'stop-speaking') handleInterruptAssistant()
else if (action === 'end-call') endCall()
@ -1348,7 +1373,7 @@ function App() {
setCallMinimized(false)
}
})
}, [handleToggleCamera, handleToggleScreenShare, handleInterruptAssistant, endCall, video])
}, [handleToggleMic, handleToggleCamera, handleToggleScreenShare, handleInterruptAssistant, endCall, video])
// Enter to submit voice input, Escape to cancel
useEffect(() => {
@ -6835,6 +6860,8 @@ function App() {
onToggleScreenShare={handleToggleScreenShare}
cameraOn={video.cameraOn}
onToggleCamera={handleToggleCamera}
micMuted={micMuted}
onToggleMic={handleToggleMic}
practiceMode={practiceMode}
onMinimize={() => void handleMinimizeCall()}
onInterrupt={handleInterruptAssistant}

View file

@ -1,5 +1,5 @@
import { useEffect, useRef, useState } from 'react'
import { Minimize2, MonitorUp, PhoneOff, Presentation, Square, User, Video, VideoOff } from 'lucide-react'
import { Mic, MicOff, Minimize2, MonitorUp, PhoneOff, Presentation, Square, User, Video, VideoOff } from 'lucide-react'
import { MascotFaceIcon, TalkingHead } from '@/components/talking-head'
import type { TTSState } from '@/hooks/useVoiceTTS'
@ -12,6 +12,9 @@ interface VideoCallViewProps {
streamRef: React.MutableRefObject<MediaStream | null>
cameraOn: boolean
onToggleCamera: () => void
/** User mute = full input pause: no mic audio AND no frame capture. */
micMuted: boolean
onToggleMic: () => void
/** Starting a share collapses this view into the floating popout (the
* surface is derived from devices see App.tsx). */
onToggleScreenShare: () => void
@ -51,6 +54,8 @@ export function VideoCallView({
streamRef,
cameraOn,
onToggleCamera,
micMuted,
onToggleMic,
onToggleScreenShare,
practiceMode,
onMinimize,
@ -128,6 +133,12 @@ export function VideoCallView({
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
You
</span>
{micMuted && (
<span className="absolute bottom-3 right-3 flex items-center gap-1.5 rounded-md bg-red-600/90 px-2 py-0.5 text-sm font-medium text-white">
<MicOff className="h-3.5 w-3.5" />
Muted nothing is heard or captured
</span>
)}
</div>
{/* Assistant */}
@ -185,9 +196,34 @@ export function VideoCallView({
{/* Control bar */}
<div className="flex items-center justify-center gap-4 pb-5">
<span className="flex items-center gap-2 rounded-full bg-neutral-800 px-3 py-1.5 text-xs font-medium text-white/90">
<span className={cn('block h-2 w-2 rounded-full', STATUS_DISPLAY[status].dotClass)} />
{STATUS_DISPLAY[status].label}
{/* Muted overrides "Listening" the green pulse would be a lie.
Thinking/speaking still show: output continues while muted. */}
{micMuted && status === 'listening' ? (
<>
<span className="block h-2 w-2 rounded-full bg-red-500" />
Muted
</>
) : (
<>
<span className={cn('block h-2 w-2 rounded-full', STATUS_DISPLAY[status].dotClass)} />
{STATUS_DISPLAY[status].label}
</>
)}
</span>
<button
type="button"
onClick={onToggleMic}
className={cn(
'flex h-10 w-10 items-center justify-center rounded-full transition-colors',
micMuted
? 'bg-red-600 text-white hover:bg-red-500'
: 'bg-neutral-800 text-white/90 hover:bg-neutral-700'
)}
aria-label={micMuted ? 'Unmute' : 'Mute (pauses mic and frame capture)'}
title={micMuted ? 'Unmute' : 'Mute — pauses your mic and all frame capture'}
>
{micMuted ? <MicOff className="h-5 w-5" /> : <Mic className="h-5 w-5" />}
</button>
<button
type="button"
onClick={onToggleCamera}

View file

@ -1,5 +1,5 @@
import { useCallback, useEffect, useRef, useState } from 'react'
import { Maximize2, MonitorUp, PhoneOff, Square, User, Video, VideoOff } from 'lucide-react'
import { Maximize2, Mic, MicOff, MonitorUp, PhoneOff, Square, User, Video, VideoOff } from 'lucide-react'
import { TalkingHead } from '@/components/talking-head'
@ -7,6 +7,8 @@ type PopoutState = {
ttsState: 'idle' | 'synthesizing' | 'speaking'
status: 'listening' | 'thinking' | 'speaking' | null
cameraOn: boolean
/** User mute = full input pause: no mic audio AND no frame capture. */
micMuted: boolean
screenSharing: boolean
interimText: string | null
}
@ -34,7 +36,7 @@ export function VideoPopout() {
// Camera defaults OFF: guessing "on" would flash the user's video for a
// beat before the real state arrives — which reads as a bug. The true
// state is fetched immediately below.
const [state, setState] = useState<PopoutState>({ ttsState: 'idle', status: null, cameraOn: false, screenSharing: false, interimText: null })
const [state, setState] = useState<PopoutState>({ ttsState: 'idle', status: null, cameraOn: false, micMuted: false, screenSharing: false, interimText: null })
const videoRef = useRef<HTMLVideoElement | null>(null)
useEffect(() => {
@ -80,7 +82,7 @@ export function VideoPopout() {
// so the mascot still animates while the assistant speaks in the main window.
const getLevel = useCallback(() => 0.45 + 0.35 * Math.sin(performance.now() / 90), [])
const sendAction = useCallback((action: 'toggle-camera' | 'toggle-share' | 'stop-speaking' | 'end-call' | 'expand') => {
const sendAction = useCallback((action: 'toggle-mic' | 'toggle-camera' | 'toggle-share' | 'stop-speaking' | 'end-call' | 'expand') => {
void window.ipc.invoke('video:popoutAction', { action }).catch(() => {})
}, [])
@ -112,11 +114,18 @@ export function VideoPopout() {
You
</span>
{/* Persistent consent badge the user must always be able to see
at a glance that their screen is going out. */}
at a glance that their screen is going out. Muted pauses frame
capture while keeping the share stream open, so say so. */}
{state.screenSharing && (
<span className="absolute left-1.5 top-1.5 flex items-center gap-1 rounded-full bg-sky-600/90 px-1.5 py-0.5 text-[10px] font-medium text-white">
<span className="block h-1.5 w-1.5 animate-pulse rounded-full bg-white" />
Sharing screen
<span className={`block h-1.5 w-1.5 rounded-full bg-white ${state.micMuted ? '' : 'animate-pulse'}`} />
{state.micMuted ? 'Sharing paused' : 'Sharing screen'}
</span>
)}
{state.micMuted && (
<span className="absolute bottom-1 right-1.5 flex items-center gap-1 rounded bg-red-600/90 px-1.5 py-0.5 text-[10px] font-medium text-white">
<MicOff className="h-2.5 w-2.5" />
Muted
</span>
)}
</div>
@ -127,8 +136,18 @@ export function VideoPopout() {
</span>
{statusDisplay && (
<span className="absolute right-1.5 top-1.5 flex items-center gap-1 rounded-full bg-black/50 px-1.5 py-0.5 text-[10px] font-medium text-white">
<span className={`block h-1.5 w-1.5 rounded-full ${statusDisplay.dotClass}`} />
{statusDisplay.label}
{/* Muted overrides "Listening" — the green pulse would be a lie. */}
{state.micMuted && state.status === 'listening' ? (
<>
<span className="block h-1.5 w-1.5 rounded-full bg-red-500" />
Muted
</>
) : (
<>
<span className={`block h-1.5 w-1.5 rounded-full ${statusDisplay.dotClass}`} />
{statusDisplay.label}
</>
)}
</span>
)}
{(state.status === 'speaking' || state.status === 'thinking') && (
@ -157,6 +176,19 @@ export function VideoPopout() {
{/* Control bar — actions execute in the main app window */}
<div className="flex h-7 shrink-0 items-center justify-center gap-2" style={noDragRegion}>
<button
type="button"
onClick={() => sendAction('toggle-mic')}
className={`flex h-6 w-6 items-center justify-center rounded-full transition-colors ${
state.micMuted
? 'bg-red-600 text-white hover:bg-red-500'
: 'bg-neutral-700 text-white/90 hover:bg-neutral-600'
}`}
aria-label={state.micMuted ? 'Unmute' : 'Mute (pauses mic and frame capture)'}
title={state.micMuted ? 'Unmute' : 'Mute — pauses your mic and all frame capture'}
>
{state.micMuted ? <MicOff className="h-3.5 w-3.5" /> : <Mic className="h-3.5 w-3.5" />}
</button>
<button
type="button"
onClick={() => sendAction('toggle-camera')}

View file

@ -154,6 +154,10 @@ export function useVideoMode() {
// Camera can be turned off mid-session (Meet-style) while the mode — and
// any screen share — keeps running. Resets to on for the next session.
const [cameraOn, setCameraOn] = useState(true);
// In-call mute pauses capture entirely: nothing lands in the ring buffers
// and collectFrames() returns nothing, so a muted stretch can never ride
// along with a later message. Streams stay open for instant resume.
const capturePausedRef = useRef(false);
const cameraPipeRef = useRef<CapturePipe>(emptyPipe());
const screenPipeRef = useRef<CapturePipe>(emptyPipe());
// Stable stream refs for preview components (<video srcObject>).
@ -165,13 +169,19 @@ export function useVideoMode() {
screenStateRef.current = screenState;
const captureCameraFrame = useCallback(() => {
if (capturePausedRef.current) return;
capturePipeFrame(cameraPipeRef.current, CAMERA_FRAME_WIDTH, CAMERA_JPEG_QUALITY);
}, []);
const captureScreenFrame = useCallback(() => {
if (capturePausedRef.current) return;
capturePipeFrame(screenPipeRef.current, SCREEN_FRAME_WIDTH, SCREEN_JPEG_QUALITY);
}, []);
const setCapturePaused = useCallback((paused: boolean) => {
capturePausedRef.current = paused;
}, []);
const stopScreenShare = useCallback(() => {
teardownPipe(screenPipeRef.current);
screenStreamRef.current = null;
@ -183,6 +193,7 @@ export function useVideoMode() {
streamRef.current = null;
setState('idle');
setCameraOn(true);
capturePausedRef.current = false;
stopScreenShare();
}, [stopScreenShare]);
@ -295,6 +306,9 @@ export function useVideoMode() {
*/
const collectFrames = useCallback((): CapturedVideoFrame[] => {
if (stateRef.current !== 'live') return [];
// Muted: no frames at all — not even pre-mute buffered ones — so a
// typed message during a mute carries nothing captured around it.
if (capturePausedRef.current) return [];
// Grab a frame right now so the message always includes the moment of send.
captureCameraFrame();
const frames = drainPipe(cameraPipeRef.current, MAX_CAMERA_FRAMES_PER_MESSAGE, 'camera');
@ -308,5 +322,5 @@ export function useVideoMode() {
// Release the camera/screen if the component unmounts with video mode on.
useEffect(() => stop, [stop]);
return { state, screenState, cameraOn, streamRef, screenStreamRef, start, stop, startScreenShare, stopScreenShare, setCameraEnabled, collectFrames };
return { state, screenState, cameraOn, streamRef, screenStreamRef, start, stop, startScreenShare, stopScreenShare, setCameraEnabled, setCapturePaused, collectFrames };
}

View file

@ -1465,6 +1465,8 @@ const ipcSchemas = {
ttsState: z.enum(['idle', 'synthesizing', 'speaking']),
status: z.enum(['listening', 'thinking', 'speaking']).nullable(),
cameraOn: z.boolean(),
// User mute: mic audio and frame capture are both paused.
micMuted: z.boolean(),
screenSharing: z.boolean(),
// Live transcript of the in-progress utterance.
interimText: z.string().nullable(),
@ -1483,6 +1485,7 @@ const ipcSchemas = {
ttsState: z.enum(['idle', 'synthesizing', 'speaking']),
status: z.enum(['listening', 'thinking', 'speaking']).nullable(),
cameraOn: z.boolean(),
micMuted: z.boolean(),
screenSharing: z.boolean(),
interimText: z.string().nullable(),
})
@ -1494,7 +1497,7 @@ const ipcSchemas = {
// main app window (handled in the main process).
'video:popoutAction': {
req: z.object({
action: z.enum(['toggle-camera', 'toggle-share', 'stop-speaking', 'end-call', 'expand']),
action: z.enum(['toggle-mic', 'toggle-camera', 'toggle-share', 'stop-speaking', 'end-call', 'expand']),
}),
res: z.object({}),
},
@ -1504,6 +1507,7 @@ const ipcSchemas = {
ttsState: z.enum(['idle', 'synthesizing', 'speaking']),
status: z.enum(['listening', 'thinking', 'speaking']).nullable(),
cameraOn: z.boolean(),
micMuted: z.boolean(),
screenSharing: z.boolean(),
interimText: z.string().nullable(),
}),
@ -1512,7 +1516,7 @@ const ipcSchemas = {
// Push channel: main → app window with a popout control-bar action.
'video:popout-action': {
req: z.object({
action: z.enum(['toggle-camera', 'toggle-share', 'stop-speaking', 'end-call', 'expand']),
action: z.enum(['toggle-mic', 'toggle-camera', 'toggle-share', 'stop-speaking', 'end-call', 'expand']),
}),
res: z.null(),
},