apunkt/rowboat
1
0
Fork 0
mirror of https://github.com/rowboatlabs/rowboat.git synced 2026-07-15 21:11:08 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Add in-call mic mute: pauses all input (mic audio + frame capture)

Browse source 
A mute button on both call surfaces (full-screen call and floating
popout) that pauses everything going to the assistant while keeping the
call alive — for stepping away to talk to someone in the room without
ending and restarting the call.

- Mic audio stops reaching Deepgram (user mute OR'd into the existing
  thinking/speaking setPaused; KeepAlives keep the socket warm so
  unmute is instant)
- Camera/screen frames stop being sampled (new setCapturePaused in
  useVideoMode); collectFrames() returns nothing while muted, so typed
  messages during a mute carry no frames either
- Devices stay acquired for instant resume; mute resets at call
  start/end; assistant output is unaffected (Stop handles that)
- Honest UI: "Muted" status chip replaces the green "Listening" pulse,
  muted badge on the user tile, pill's share badge flips to "Sharing
  paused"; new toggle-mic popout action + micMuted in popout state

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Arjun 2026-07-06 00:28:34 +05:30
parent 1c380c33ed
commit adb8b16a3c
7 changed files with 147 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

1
apps/x/apps/main/src/ipc.ts
View file

@ -395,6 +395,7 @@ let lastVideoPopoutState: {
ttsState: 'idle' | 'synthesizing' | 'speaking';
status: 'listening' | 'thinking' | 'speaking' | null;
cameraOn: boolean;
micMuted: boolean;
screenSharing: boolean;
interimText: string | null;
} | null = null;

39
apps/x/apps/renderer/src/App.tsx
View file

@ -1061,6 +1061,11 @@ function App() {
const inCallRef = useRef(false)
// User explicitly shrank the full-screen call to the floating pill.
const [callMinimized, setCallMinimized] = useState(false)
// In-call mute: a full input pause, not just audio — mic audio stops
// reaching Deepgram AND camera/screen frame capture stops, so nothing said
// or shown while muted ever reaches the assistant. Output is untouched
// (in-flight speech keeps playing; the Stop control handles that).
const [micMuted, setMicMuted] = useState(false)
// Practice preset: adds the coaching persona to the system prompt.
const [practiceMode, setPracticeMode] = useState(false)
const practiceModeRef = useRef(false)
@ -1200,6 +1205,7 @@ function App() {
setPracticeMode(preset === 'practice')
practiceModeRef.current = preset === 'practice'
setMicMuted(false)
// Pill-first presets start minimized; face-to-face presets start expanded.
setCallMinimized(preset === 'voice' || preset === 'share')
inCallRef.current = true
@ -1217,17 +1223,26 @@ function App() {
video.stop()
setPracticeMode(false)
practiceModeRef.current = false
setMicMuted(false)
setCallMinimized(false)
inCallRef.current = false
setInCall(false)
}, [video])
// During a call, mute the mic while the assistant is thinking or speaking
// so its own TTS (or a half-turn) never gets transcribed back at it.
// so its own TTS (or a half-turn) never gets transcribed back at it — and
// whenever the user muted themselves.
useEffect(() => {
if (!inCall) return
voiceRef.current.setPaused(activeIsProcessing || tts.state !== 'idle')
}, [inCall, activeIsProcessing, tts.state])
voiceRef.current.setPaused(micMuted || activeIsProcessing || tts.state !== 'idle')
}, [inCall, micMuted, activeIsProcessing, tts.state])
// The user-mute half that lives in the video pipeline: stop sampling
// camera/screen frames while muted (see useVideoMode.setCapturePaused).
const setCapturePaused = video.setCapturePaused
useEffect(() => {
setCapturePaused(micMuted)
}, [micMuted, setCapturePaused])
// Screen sharing: frames of the shared screen ride along with each message
// next to the webcam frames. The surface change (full screen → pill) falls
@ -1248,6 +1263,14 @@ function App() {
void video.setCameraEnabled(!video.cameraOn)
}, [video])
// Zoom-style mute button, except it pauses ALL input (mic + frames) so the
// user can talk to someone in the room without the assistant listening in.
// Devices stay acquired (camera light and share indicator stay on) so
// unmuting is instant.
const handleToggleMic = useCallback(() => {
setMicMuted((m) => !m)
}, [])
// Minimizing the full-screen call drops you back to working — and the pill
// exists to work *together*, so sharing starts automatically (the symmetric
// twin of expand, which stops it). If capture fails (permission), the call
@ -1327,11 +1350,12 @@ function App() {
ttsState: tts.state,
status: videoCallStatus,
cameraOn: video.cameraOn,
micMuted,
screenSharing: video.screenState === 'live',
interimText: voice.interimText || null,
})
.catch(() => {})
}, [inCall, tts.state, videoCallStatus, video.cameraOn, video.screenState, voice.interimText])
}, [inCall, tts.state, videoCallStatus, video.cameraOn, micMuted, video.screenState, voice.interimText])
// Execute popout control-bar actions (the popout window has no access to
// the call's mic/camera/capture — they live here). 'expand' goes full
@ -1339,7 +1363,8 @@ function App() {
// process already refocused the app window.
useEffect(() => {
return window.ipc.on('video:popout-action', ({ action }) => {
if (action === 'toggle-camera') handleToggleCamera()
if (action === 'toggle-mic') handleToggleMic()
else if (action === 'toggle-camera') handleToggleCamera()
else if (action === 'toggle-share') void handleToggleScreenShare()
else if (action === 'stop-speaking') handleInterruptAssistant()
else if (action === 'end-call') endCall()
@ -1348,7 +1373,7 @@ function App() {
setCallMinimized(false)
}
})
}, [handleToggleCamera, handleToggleScreenShare, handleInterruptAssistant, endCall, video])
}, [handleToggleMic, handleToggleCamera, handleToggleScreenShare, handleInterruptAssistant, endCall, video])
// Enter to submit voice input, Escape to cancel
useEffect(() => {
@ -6835,6 +6860,8 @@ function App() {
onToggleScreenShare={handleToggleScreenShare}
cameraOn={video.cameraOn}
onToggleCamera={handleToggleCamera}
micMuted={micMuted}
onToggleMic={handleToggleMic}
practiceMode={practiceMode}
onMinimize={() => void handleMinimizeCall()}
onInterrupt={handleInterruptAssistant}

42
apps/x/apps/renderer/src/components/video-call-view.tsx
View file

@ -1,5 +1,5 @@
import { useEffect, useRef, useState } from 'react'
import { Minimize2, MonitorUp, PhoneOff, Presentation, Square, User, Video, VideoOff } from 'lucide-react'
import { Mic, MicOff, Minimize2, MonitorUp, PhoneOff, Presentation, Square, User, Video, VideoOff } from 'lucide-react'
import { MascotFaceIcon, TalkingHead } from '@/components/talking-head'
import type { TTSState } from '@/hooks/useVoiceTTS'
@ -12,6 +12,9 @@ interface VideoCallViewProps {
streamRef: React.MutableRefObject<MediaStream | null>
cameraOn: boolean
onToggleCamera: () => void
/** User mute = full input pause: no mic audio AND no frame capture. */
micMuted: boolean
onToggleMic: () => void
/** Starting a share collapses this view into the floating popout (the
* surface is derived from devices — see App.tsx). */
onToggleScreenShare: () => void
@ -51,6 +54,8 @@ export function VideoCallView({
streamRef,
cameraOn,
onToggleCamera,
micMuted,
onToggleMic,
onToggleScreenShare,
practiceMode,
onMinimize,
@ -128,6 +133,12 @@ export function VideoCallView({
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
You
</span>
{micMuted && (
<span className="absolute bottom-3 right-3 flex items-center gap-1.5 rounded-md bg-red-600/90 px-2 py-0.5 text-sm font-medium text-white">
<MicOff className="h-3.5 w-3.5" />
Muted — nothing is heard or captured
</span>
)}
</div>
{/* Assistant */}
@ -185,9 +196,34 @@ export function VideoCallView({
{/* Control bar */}
<div className="flex items-center justify-center gap-4 pb-5">
<span className="flex items-center gap-2 rounded-full bg-neutral-800 px-3 py-1.5 text-xs font-medium text-white/90">
<span className={cn('block h-2 w-2 rounded-full', STATUS_DISPLAY[status].dotClass)} />
{STATUS_DISPLAY[status].label}
{/* Muted overrides "Listening" — the green pulse would be a lie.
Thinking/speaking still show: output continues while muted. */}
{micMuted && status === 'listening' ? (
<>
<span className="block h-2 w-2 rounded-full bg-red-500" />
Muted
</>
) : (
<>
<span className={cn('block h-2 w-2 rounded-full', STATUS_DISPLAY[status].dotClass)} />
{STATUS_DISPLAY[status].label}
</>
)}
</span>
<button
type="button"
onClick={onToggleMic}
className={cn(
'flex h-10 w-10 items-center justify-center rounded-full transition-colors',
micMuted
? 'bg-red-600 text-white hover:bg-red-500'
: 'bg-neutral-800 text-white/90 hover:bg-neutral-700'
)}
aria-label={micMuted ? 'Unmute' : 'Mute (pauses mic and frame capture)'}
title={micMuted ? 'Unmute' : 'Mute — pauses your mic and all frame capture'}
>
{micMuted ? <MicOff className="h-5 w-5" /> : <Mic className="h-5 w-5" />}
</button>
<button
type="button"
onClick={onToggleCamera}

48
apps/x/apps/renderer/src/components/video-popout.tsx
View file

@ -1,5 +1,5 @@
import { useCallback, useEffect, useRef, useState } from 'react'
import { Maximize2, MonitorUp, PhoneOff, Square, User, Video, VideoOff } from 'lucide-react'
import { Maximize2, Mic, MicOff, MonitorUp, PhoneOff, Square, User, Video, VideoOff } from 'lucide-react'
import { TalkingHead } from '@/components/talking-head'
@ -7,6 +7,8 @@ type PopoutState = {
ttsState: 'idle' | 'synthesizing' | 'speaking'
status: 'listening' | 'thinking' | 'speaking' | null
cameraOn: boolean
/** User mute = full input pause: no mic audio AND no frame capture. */
micMuted: boolean
screenSharing: boolean
interimText: string | null
}
@ -34,7 +36,7 @@ export function VideoPopout() {
// Camera defaults OFF: guessing "on" would flash the user's video for a
// beat before the real state arrives — which reads as a bug. The true
// state is fetched immediately below.
const [state, setState] = useState<PopoutState>({ ttsState: 'idle', status: null, cameraOn: false, screenSharing: false, interimText: null })
const [state, setState] = useState<PopoutState>({ ttsState: 'idle', status: null, cameraOn: false, micMuted: false, screenSharing: false, interimText: null })
const videoRef = useRef<HTMLVideoElement | null>(null)
useEffect(() => {
@ -80,7 +82,7 @@ export function VideoPopout() {
// so the mascot still animates while the assistant speaks in the main window.
const getLevel = useCallback(() => 0.45 + 0.35 * Math.sin(performance.now() / 90), [])
const sendAction = useCallback((action: 'toggle-camera' | 'toggle-share' | 'stop-speaking' | 'end-call' | 'expand') => {
const sendAction = useCallback((action: 'toggle-mic' | 'toggle-camera' | 'toggle-share' | 'stop-speaking' | 'end-call' | 'expand') => {
void window.ipc.invoke('video:popoutAction', { action }).catch(() => {})
}, [])
@ -112,11 +114,18 @@ export function VideoPopout() {
You
</span>
{/* Persistent consent badge — the user must always be able to see
at a glance that their screen is going out. */}
at a glance that their screen is going out. Muted pauses frame
capture while keeping the share stream open, so say so. */}
{state.screenSharing && (
<span className="absolute left-1.5 top-1.5 flex items-center gap-1 rounded-full bg-sky-600/90 px-1.5 py-0.5 text-[10px] font-medium text-white">
<span className="block h-1.5 w-1.5 animate-pulse rounded-full bg-white" />
Sharing screen
<span className={`block h-1.5 w-1.5 rounded-full bg-white ${state.micMuted ? '' : 'animate-pulse'}`} />
{state.micMuted ? 'Sharing paused' : 'Sharing screen'}
</span>
)}
{state.micMuted && (
<span className="absolute bottom-1 right-1.5 flex items-center gap-1 rounded bg-red-600/90 px-1.5 py-0.5 text-[10px] font-medium text-white">
<MicOff className="h-2.5 w-2.5" />
Muted
</span>
)}
</div>
@ -127,8 +136,18 @@ export function VideoPopout() {
</span>
{statusDisplay && (
<span className="absolute right-1.5 top-1.5 flex items-center gap-1 rounded-full bg-black/50 px-1.5 py-0.5 text-[10px] font-medium text-white">
<span className={`block h-1.5 w-1.5 rounded-full ${statusDisplay.dotClass}`} />
{statusDisplay.label}
{/* Muted overrides "Listening" — the green pulse would be a lie. */}
{state.micMuted && state.status === 'listening' ? (
<>
<span className="block h-1.5 w-1.5 rounded-full bg-red-500" />
Muted
</>
) : (
<>
<span className={`block h-1.5 w-1.5 rounded-full ${statusDisplay.dotClass}`} />
{statusDisplay.label}
</>
)}
</span>
)}
{(state.status === 'speaking' || state.status === 'thinking') && (
@ -157,6 +176,19 @@ export function VideoPopout() {
{/* Control bar — actions execute in the main app window */}
<div className="flex h-7 shrink-0 items-center justify-center gap-2" style={noDragRegion}>
<button
type="button"
onClick={() => sendAction('toggle-mic')}
className={`flex h-6 w-6 items-center justify-center rounded-full transition-colors ${
state.micMuted
? 'bg-red-600 text-white hover:bg-red-500'
: 'bg-neutral-700 text-white/90 hover:bg-neutral-600'
}`}
aria-label={state.micMuted ? 'Unmute' : 'Mute (pauses mic and frame capture)'}
title={state.micMuted ? 'Unmute' : 'Mute — pauses your mic and all frame capture'}
>
{state.micMuted ? <MicOff className="h-3.5 w-3.5" /> : <Mic className="h-3.5 w-3.5" />}
</button>
<button
type="button"
onClick={() => sendAction('toggle-camera')}

16
apps/x/apps/renderer/src/hooks/useVideoMode.ts
View file

@ -154,6 +154,10 @@ export function useVideoMode() {
// Camera can be turned off mid-session (Meet-style) while the mode — and
// any screen share — keeps running. Resets to on for the next session.
const [cameraOn, setCameraOn] = useState(true);
// In-call mute pauses capture entirely: nothing lands in the ring buffers
// and collectFrames() returns nothing, so a muted stretch can never ride
// along with a later message. Streams stay open for instant resume.
const capturePausedRef = useRef(false);
const cameraPipeRef = useRef<CapturePipe>(emptyPipe());
const screenPipeRef = useRef<CapturePipe>(emptyPipe());
// Stable stream refs for preview components (<video srcObject>).
@ -165,13 +169,19 @@ export function useVideoMode() {
screenStateRef.current = screenState;
const captureCameraFrame = useCallback(() => {
if (capturePausedRef.current) return;
capturePipeFrame(cameraPipeRef.current, CAMERA_FRAME_WIDTH, CAMERA_JPEG_QUALITY);
}, []);
const captureScreenFrame = useCallback(() => {
if (capturePausedRef.current) return;
capturePipeFrame(screenPipeRef.current, SCREEN_FRAME_WIDTH, SCREEN_JPEG_QUALITY);
}, []);
const setCapturePaused = useCallback((paused: boolean) => {
capturePausedRef.current = paused;
}, []);
const stopScreenShare = useCallback(() => {
teardownPipe(screenPipeRef.current);
screenStreamRef.current = null;
@ -183,6 +193,7 @@ export function useVideoMode() {
streamRef.current = null;
setState('idle');
setCameraOn(true);
capturePausedRef.current = false;
stopScreenShare();
}, [stopScreenShare]);
@ -295,6 +306,9 @@ export function useVideoMode() {
*/
const collectFrames = useCallback((): CapturedVideoFrame[] => {
if (stateRef.current !== 'live') return [];
// Muted: no frames at all — not even pre-mute buffered ones — so a
// typed message during a mute carries nothing captured around it.
if (capturePausedRef.current) return [];
// Grab a frame right now so the message always includes the moment of send.
captureCameraFrame();
const frames = drainPipe(cameraPipeRef.current, MAX_CAMERA_FRAMES_PER_MESSAGE, 'camera');
@ -308,5 +322,5 @@ export function useVideoMode() {
// Release the camera/screen if the component unmounts with video mode on.
useEffect(() => stop, [stop]);
return { state, screenState, cameraOn, streamRef, screenStreamRef, start, stop, startScreenShare, stopScreenShare, setCameraEnabled, collectFrames };
return { state, screenState, cameraOn, streamRef, screenStreamRef, start, stop, startScreenShare, stopScreenShare, setCameraEnabled, setCapturePaused, collectFrames };
}