popout and full screen work as expected

This commit is contained in:
Arjun 2026-07-04 12:06:32 +05:30
parent d2501c4f4d
commit e9ee8fd975
15 changed files with 563 additions and 696 deletions

View file

@ -390,6 +390,7 @@ let lastVideoPopoutState: {
ttsState: 'idle' | 'synthesizing' | 'speaking';
status: 'listening' | 'thinking' | 'speaking' | null;
cameraOn: boolean;
screenSharing: boolean;
interimText: string | null;
} | null = null;
@ -1816,19 +1817,20 @@ export function setupIpcHandlers() {
}
return {};
},
'video:focusMain': async () => {
'video:getPopoutState': async () => {
return { state: lastVideoPopoutState };
},
'video:popoutAction': async (_event, args) => {
// Relay a popout control-bar action to the app window, which owns the
// call (mic, camera, screen capture) and executes it there. 'expand'
// additionally brings the app window back to the foreground.
const main = findMainAppWindow();
if (main) {
if (args.action === 'expand' && main) {
if (main.isMinimized()) main.restore();
main.show();
main.focus();
}
return {};
},
'video:popoutAction': async (_event, args) => {
// Relay a popout control-bar action to the app window, which owns the
// call (mic, camera, screen capture) and executes it there.
findMainAppWindow()?.webContents.send('video:popout-action', args);
main?.webContents.send('video:popout-action', args);
return {};
},
// Live-note handlers

View file

@ -12,7 +12,7 @@ import { ChatSidebar } from './components/chat-sidebar';
import { useSessionChat } from '@/hooks/useSessionChat';
import { ChatHeader } from './components/chat-header';
import { ChatEmptyState } from './components/chat-empty-state';
import { ChatInputWithMentions, type PermissionMode, type StagedAttachment, type VideoChatMode } from './components/chat-input-with-mentions';
import { ChatInputWithMentions, type CallPreset, type PermissionMode, type StagedAttachment } from './components/chat-input-with-mentions';
import { ChatMessageAttachments } from '@/components/chat-message-attachments'
import { GraphView, type GraphEdge, type GraphNode } from '@/components/graph-view';
import { BasesView, type BaseConfig, DEFAULT_BASE_CONFIG } from '@/components/bases-view';
@ -120,8 +120,6 @@ import { toast } from "sonner"
import { useVoiceMode } from '@/hooks/useVoiceMode'
import { useVideoMode } from '@/hooks/useVideoMode'
import { useVoiceTTS } from '@/hooks/useVoiceTTS'
import { TalkingHeadOverlay } from '@/components/talking-head'
import { VideoPreviewOverlay } from '@/components/video-preview-overlay'
import { VideoCallView } from '@/components/video-call-view'
import { ProductTour, type TourNavTarget } from '@/components/product-tour'
import { useMeetingTranscription, type CalendarEventMeta } from '@/hooks/useMeetingTranscription'
@ -974,9 +972,9 @@ function App() {
const [ttsAvailable, setTtsAvailable] = useState(false)
const [ttsEnabled, setTtsEnabled] = useState(false)
const ttsEnabledRef = useRef(false)
const [ttsMode, setTtsMode] = useState<'summary' | 'full'>('summary')
// Read-aloud style: 'summary' for typed chat, forced to 'full' during a
// call and restored after. Context decides — the user never picks it.
const ttsModeRef = useRef<'summary' | 'full'>('summary')
const [ttsAvatarEnabled, setTtsAvatarEnabled] = useState(false)
const [tourActive, setTourActive] = useState(false)
const [isRecording, setIsRecording] = useState(false)
const voiceTextBufferRef = useRef('')
@ -1021,16 +1019,20 @@ function App() {
const voiceRef = useRef(voice)
voiceRef.current = voice
// Video chat mode: while on, the webcam runs and frames are attached to
// every outgoing message so the assistant can see the user. 'chat' keeps
// the normal composer; 'call' is fully hands-free — continuous listening,
// auto-submitted utterances, spoken responses (handler defined below, after
// the voice/submit plumbing it drives).
// Calls: one engine (hands-free voice loop + forced read-aloud TTS + frame
// capture), started via presets that only differ in device defaults. The
// presentation is DERIVED from devices, never picked: screen sharing →
// floating popout; camera on → full-screen call; camera off → popout
// (mascot pill). Handlers live below the voice/submit plumbing they drive.
const video = useVideoMode()
const [videoChatMode, setVideoChatMode] = useState<VideoChatMode>('off')
const videoChatModeRef = useRef<VideoChatMode>('off')
// TTS settings to restore when a hands-free call ends (a call forces
// full-read-aloud TTS for its duration).
const [inCall, setInCall] = useState(false)
const inCallRef = useRef(false)
// User explicitly shrank the full-screen call to the floating pill.
const [callMinimized, setCallMinimized] = useState(false)
// Practice preset: adds the coaching persona to the system prompt.
const [practiceMode, setPracticeMode] = useState(false)
const practiceModeRef = useRef(false)
// TTS settings to restore when a call ends (a call forces full read-aloud).
const preCallTtsRef = useRef<{ enabled: boolean; mode: 'summary' | 'full' } | null>(null)
const handleToggleMeetingRef = useRef<(() => void) | undefined>(undefined)
@ -1091,8 +1093,8 @@ function App() {
}, [])
const handleStartRecording = useCallback(() => {
// Hands-free call mode owns the mic — ignore push-to-talk while it's on.
if (videoChatModeRef.current === 'call' || videoChatModeRef.current === 'meeting') return
// A live call owns the mic — ignore push-to-talk while one is running.
if (inCallRef.current) return
setIsRecording(true)
isRecordingRef.current = true
voice.start()
@ -1123,143 +1125,128 @@ function App() {
ttsEnabledRef.current = next
if (!next) {
ttsRef.current.cancel()
setTtsAvatarEnabled(false)
}
return next
})
}, [])
// Talking-head mode implies voice output: enabling it turns TTS on,
// disabling it turns both off.
const handleToggleTtsAvatar = useCallback(() => {
setTtsAvatarEnabled(prev => {
const next = !prev
setTtsEnabled(next)
ttsEnabledRef.current = next
if (!next) {
ttsRef.current.cancel()
}
return next
})
}, [])
const handleTtsModeChange = useCallback((mode: 'summary' | 'full') => {
setTtsMode(mode)
ttsModeRef.current = mode
}, [])
const handleCancelRecording = useCallback(() => {
voice.cancel()
setIsRecording(false)
isRecordingRef.current = false
}, [voice])
// Video chat mode transitions. 'chat' just runs the camera; 'call' and
// 'meeting' also listen continuously (auto-submitting each utterance as a
// voice message) and force full read-aloud TTS so the assistant answers out
// loud. 'meeting' is the same pipeline presented as a full-screen call.
const handleVideoModeChange = useCallback(async (mode: VideoChatMode) => {
const prev = videoChatModeRef.current
if (mode === prev) return
const wasHandsFree = prev === 'call' || prev === 'meeting'
const isHandsFree = mode === 'call' || mode === 'meeting'
// Start a call. Presets only differ in device defaults — the engine
// (continuous listening, auto-submitted utterances, forced read-aloud TTS,
// frame capture) is identical for all of them. The default entry ('share',
// the call button's main click) is "work together": screen shared, camera
// off, floating pill — the user keeps working while the assistant watches
// along. 'video'/'practice' open face-to-face full screen instead.
const startCall = useCallback(async (preset: CallPreset) => {
if (inCallRef.current) return
const camera = preset === 'video' || preset === 'practice'
const ok = await video.start({ camera })
if (!ok) return // camera denied/unavailable — stay out of the call
if (preset === 'share') {
// If screen capture fails (usually the macOS Screen Recording
// permission), continue as a voice call — sharing is one tap away on
// the pill once permission is granted.
const shared = await video.startScreenShare()
if (!shared) {
toast("Couldn't share your screen", {
description: 'Grant Rowboat Screen Recording access, then tap the share button on the call.',
action: {
label: 'Open Settings',
onClick: () => void window.ipc.invoke('meeting:openScreenRecordingSettings', null).catch(() => {}),
},
})
}
}
// Leaving hands-free: release the mic and restore the pre-call TTS settings.
if (wasHandsFree && !isHandsFree) {
// A manual push-to-talk recording can't coexist with the call's mic.
if (isRecordingRef.current) {
voiceRef.current.cancel()
const saved = preCallTtsRef.current
preCallTtsRef.current = null
const restoreEnabled = saved?.enabled ?? false
setTtsEnabled(restoreEnabled)
ttsEnabledRef.current = restoreEnabled
if (saved) {
setTtsMode(saved.mode)
ttsModeRef.current = saved.mode
}
if (!restoreEnabled) ttsRef.current.cancel()
setIsRecording(false)
isRecordingRef.current = false
}
preCallTtsRef.current = { enabled: ttsEnabledRef.current, mode: ttsModeRef.current }
setTtsEnabled(true)
ttsEnabledRef.current = true
ttsModeRef.current = 'full'
void voiceRef.current.startContinuous((text) => {
pendingVoiceInputRef.current = true
handlePromptSubmitRef.current?.({ text, files: [] })
})
if (mode === 'off') {
video.stop()
videoChatModeRef.current = 'off'
setVideoChatMode('off')
return
setPracticeMode(preset === 'practice')
practiceModeRef.current = preset === 'practice'
// Pill-first presets start minimized; face-to-face presets start expanded.
setCallMinimized(preset === 'voice' || preset === 'share')
inCallRef.current = true
setInCall(true)
analytics.callStarted(preset)
}, [video])
const endCall = useCallback(() => {
if (!inCallRef.current) return
voiceRef.current.cancel()
const saved = preCallTtsRef.current
preCallTtsRef.current = null
const restoreEnabled = saved?.enabled ?? false
setTtsEnabled(restoreEnabled)
ttsEnabledRef.current = restoreEnabled
if (saved) {
ttsModeRef.current = saved.mode
}
if (prev === 'off') {
const ok = await video.start()
if (!ok) return // camera denied/unavailable — stay off
}
// Entering hands-free (call ↔ meeting switches keep the mic/TTS as-is).
if (isHandsFree && !wasHandsFree) {
// A manual push-to-talk recording can't coexist with the call's mic.
if (isRecordingRef.current) {
voiceRef.current.cancel()
setIsRecording(false)
isRecordingRef.current = false
}
preCallTtsRef.current = { enabled: ttsEnabledRef.current, mode: ttsModeRef.current }
setTtsEnabled(true)
ttsEnabledRef.current = true
setTtsMode('full')
ttsModeRef.current = 'full'
void voiceRef.current.startContinuous((text) => {
pendingVoiceInputRef.current = true
handlePromptSubmitRef.current?.({ text, files: [] })
})
}
videoChatModeRef.current = mode
setVideoChatMode(mode)
if (!restoreEnabled) ttsRef.current.cancel()
video.stop()
setPracticeMode(false)
practiceModeRef.current = false
setCallMinimized(false)
inCallRef.current = false
setInCall(false)
}, [video])
// During a call, mute the mic while the assistant is thinking or speaking
// so its own TTS (or a half-turn) never gets transcribed back at it.
useEffect(() => {
if (videoChatMode !== 'call' && videoChatMode !== 'meeting') return
if (!inCall) return
voiceRef.current.setPaused(activeIsProcessing || tts.state !== 'idle')
}, [videoChatMode, activeIsProcessing, tts.state])
}, [inCall, activeIsProcessing, tts.state])
// Presenting from the full-screen call collapses it into the floating
// popout (the user needs their screen — possibly Rowboat itself — free to
// navigate while sharing). When the share ends, return to full screen.
const returnToMeetingAfterShareRef = useRef(false)
// Screen sharing (any video mode): frames of the shared screen ride along
// with each message next to the webcam frames.
// Screen sharing: frames of the shared screen ride along with each message
// next to the webcam frames. The surface change (full screen → pill) falls
// out of the derivation below.
const handleToggleScreenShare = useCallback(async () => {
if (video.screenState === 'live') {
video.stopScreenShare()
} else {
const ok = await video.startScreenShare()
if (ok && videoChatModeRef.current === 'meeting') {
returnToMeetingAfterShareRef.current = true
void handleVideoModeChange('call')
}
await video.startScreenShare()
}
}, [video, handleVideoModeChange])
}, [video])
// Share ended (from the popout, the PiP, or the OS): restore the
// full-screen call if that's where the share was started from.
useEffect(() => {
if (video.screenState === 'live') return
if (!returnToMeetingAfterShareRef.current) return
returnToMeetingAfterShareRef.current = false
if (videoChatModeRef.current === 'call') {
void handleVideoModeChange('meeting')
}
}, [video.screenState, handleVideoModeChange])
// Meet-style camera mute: video mode (and any screen share) stays on, but
// no webcam frames are captured while the camera is off.
// Meet-style camera mute: the call (and any screen share) stays on, but no
// webcam frames are captured while the camera is off. Deliberately does NOT
// change the surface — turning your camera on from the pill puts your video
// IN the pill; expanding to full screen is its own explicit action.
const handleToggleCamera = useCallback(() => {
void video.setCameraEnabled(!video.cameraOn)
}, [video])
// Current phase of a hands-free call (null outside call/meeting modes).
// Minimizing the full-screen call drops you back to working — and the pill
// exists to work *together*, so sharing starts automatically (the symmetric
// twin of expand, which stops it). If capture fails (permission), the call
// still minimizes as a plain pill. `callMinimized` is also set so stopping
// the share from the pill keeps you in the pill rather than snapping back
// to full screen.
const handleMinimizeCall = useCallback(async () => {
setCallMinimized(true)
await video.startScreenShare()
}, [video])
// Current phase of the call (null when not in one).
const videoCallStatus: 'listening' | 'thinking' | 'speaking' | null =
videoChatMode === 'call' || videoChatMode === 'meeting'
inCall
? tts.state === 'speaking'
? 'speaking'
: tts.state === 'synthesizing' || activeIsProcessing
@ -1267,39 +1254,70 @@ function App() {
: 'listening'
: null
// Meet-style popout: the floating always-on-top mini-call appears the
// moment screen sharing starts and stays for the whole share — including
// over Rowboat itself (the shared screen may BE Rowboat), and across app
// switches. It disappears when sharing stops or video mode ends.
useEffect(() => {
const shouldShow = videoChatMode !== 'off' && video.screenState === 'live'
void window.ipc.invoke('video:setPopout', { show: shouldShow }).catch(() => {})
}, [videoChatMode, video.screenState])
// The call's surface follows one rule: full screen and screen sharing are
// mutually exclusive (a full-screen call covers the screen — sharing it
// would show the call itself). Sharing → floating pill, always. Not
// sharing → full screen unless the user shrank it (`callMinimized`).
// Expanding the pill auto-stops any share; presenting from full screen
// auto-collapses to the pill.
const callSurface: 'fullscreen' | 'popout' | null = !inCall
? null
: video.screenState === 'live' || callMinimized
? 'popout'
: 'fullscreen'
// Keep the popout's mascot/status/camera/caption mirror of the call fresh.
useEffect(() => {
void window.ipc.invoke('video:setPopout', { show: callSurface === 'popout' }).catch(() => {})
}, [callSurface])
// Consent surface for screen sharing: an unmissable toast the moment any
// share starts (auto-started calls included), with one-tap stop. The pill
// also carries a persistent "Sharing screen" badge, and macOS shows its
// purple recording indicator.
const prevScreenStateRef = useRef(video.screenState)
useEffect(() => {
const prev = prevScreenStateRef.current
prevScreenStateRef.current = video.screenState
if (video.screenState === 'live' && prev !== 'live') {
toast('Your screen is being shared', {
description: 'The assistant sees snapshots of it along with what you say.',
action: { label: 'Stop sharing', onClick: () => video.stopScreenShare() },
duration: 6000,
})
}
}, [video.screenState, video])
// Keep the popout's mascot/status/devices/caption mirror of the call fresh.
// The main process caches the latest state and replays it when the popout
// loads.
useEffect(() => {
if (videoChatMode === 'off') return
if (!inCall) return
void window.ipc
.invoke('video:popoutState', {
ttsState: tts.state,
status: videoCallStatus,
cameraOn: video.cameraOn,
interimText: videoCallStatus ? voice.interimText || null : null,
screenSharing: video.screenState === 'live',
interimText: voice.interimText || null,
})
.catch(() => {})
}, [videoChatMode, tts.state, videoCallStatus, video.cameraOn, voice.interimText])
}, [inCall, tts.state, videoCallStatus, video.cameraOn, video.screenState, voice.interimText])
// Execute popout control-bar actions (the popout window has no access to
// the call's mic/camera/capture — they live here).
// the call's mic/camera/capture — they live here). 'expand' goes full
// screen, which by the exclusivity rule stops any running share; the main
// process already refocused the app window.
useEffect(() => {
return window.ipc.on('video:popout-action', ({ action }) => {
if (action === 'toggle-camera') handleToggleCamera()
else if (action === 'stop-share') video.stopScreenShare()
else if (action === 'end-call') void handleVideoModeChange('off')
else if (action === 'toggle-share') void handleToggleScreenShare()
else if (action === 'end-call') endCall()
else if (action === 'expand') {
if (video.screenState === 'live') video.stopScreenShare()
setCallMinimized(false)
}
})
}, [handleToggleCamera, video, handleVideoModeChange])
}, [handleToggleCamera, handleToggleScreenShare, endCall, video])
// Enter to submit voice input, Escape to cancel
useEffect(() => {
@ -2783,7 +2801,7 @@ function App() {
// Video chat mode: drain the webcam frames buffered since the last send
// so they ride along with this message as inline image parts.
const videoFrames = videoChatModeRef.current !== 'off' ? video.collectFrames() : []
const videoFrames = inCallRef.current ? video.collectFrames() : []
const userMessageId = `user-${Date.now()}`
const displayAttachments: ChatMessage['attachments'] = hasAttachments || videoFrames.length > 0
@ -2854,7 +2872,8 @@ function App() {
...(ttsEnabledRef.current ? { voiceOutput: ttsModeRef.current } : {}),
...(searchEnabled ? { searchEnabled: true } : {}),
...(codeMode ? { codeMode } : {}),
...(videoChatModeRef.current !== 'off' ? { videoMode: true } : {}),
...(inCallRef.current && (video.cameraOn || video.screenState === 'live') ? { videoMode: true } : {}),
...(practiceModeRef.current ? { coachMode: true } : {}),
},
},
},
@ -6536,14 +6555,11 @@ function App() {
voiceAvailable={isActive && voiceAvailable}
ttsAvailable={isActive && ttsAvailable}
ttsEnabled={ttsEnabled}
ttsMode={ttsMode}
onToggleTts={isActive ? handleToggleTts : undefined}
onTtsModeChange={isActive ? handleTtsModeChange : undefined}
ttsAvatarEnabled={ttsAvatarEnabled}
onToggleTtsAvatar={isActive ? handleToggleTtsAvatar : undefined}
videoChatMode={videoChatMode}
onVideoModeChange={isActive ? handleVideoModeChange : undefined}
videoCallAvailable={voiceAvailable && ttsAvailable}
inCall={inCall}
onStartCall={isActive ? startCall : undefined}
onEndCall={isActive ? endCall : undefined}
callAvailable={voiceAvailable && ttsAvailable}
/>
</div>
)
@ -6652,69 +6668,32 @@ function App() {
voiceAvailable={voiceAvailable}
ttsAvailable={ttsAvailable}
ttsEnabled={ttsEnabled}
ttsMode={ttsMode}
onToggleTts={handleToggleTts}
onTtsModeChange={handleTtsModeChange}
ttsAvatarEnabled={ttsAvatarEnabled}
onToggleTtsAvatar={handleToggleTtsAvatar}
videoChatMode={videoChatMode}
onVideoModeChange={handleVideoModeChange}
videoCallAvailable={voiceAvailable && ttsAvailable}
inCall={inCall}
onStartCall={startCall}
onEndCall={endCall}
callAvailable={voiceAvailable && ttsAvailable}
onComposioConnected={handleComposioConnected}
/>
)}
{/* Webcam PiP preview while video chat mode is on. Hidden while
screen sharing (the floating popout is the call surface then)
and in the full-screen meeting view (which replaces it). */}
{(videoChatMode === 'chat' || videoChatMode === 'call') && video.screenState !== 'live' && (
<VideoPreviewOverlay
streamRef={video.streamRef}
onTurnOff={() => handleVideoModeChange('off')}
callStatus={
videoChatMode === 'call'
? tts.state !== 'idle'
? 'speaking'
: activeIsProcessing
? 'thinking'
: 'listening'
: undefined
}
interimText={videoChatMode === 'call' ? voice.interimText : undefined}
onToggleScreenShare={handleToggleScreenShare}
cameraOn={video.cameraOn}
onToggleCamera={handleToggleCamera}
/>
)}
{/* Full-screen Meet-style call: user tile + animated mascot tile */}
{videoChatMode === 'meeting' && (
{/* Full-screen call: user tile + animated mascot tile. Shown only
when the derived surface says so (camera on, no screen share,
not minimized) otherwise the call lives in the floating
popout window. */}
{callSurface === 'fullscreen' && (
<VideoCallView
streamRef={video.streamRef}
screenStreamRef={video.screenStreamRef}
isScreenSharing={video.screenState === 'live'}
onToggleScreenShare={handleToggleScreenShare}
cameraOn={video.cameraOn}
onToggleCamera={handleToggleCamera}
practiceMode={practiceMode}
onMinimize={() => void handleMinimizeCall()}
ttsState={tts.state}
getTtsLevel={tts.getLevel}
status={
tts.state === 'speaking'
? 'speaking'
: tts.state === 'synthesizing' || activeIsProcessing
? 'thinking'
: 'listening'
}
status={videoCallStatus ?? 'listening'}
interimText={voice.interimText}
assistantCaption={assistantCaption}
onLeave={() => handleVideoModeChange('off')}
/>
)}
{/* Talking head hovers over the active view while avatar voice mode is
on (hidden during the tour, which shows its own mascot) */}
{ttsAvatarEnabled && !tourActive && (
<TalkingHeadOverlay
ttsState={tts.state}
getLevel={tts.getLevel}
onDismiss={handleToggleTtsAvatar}
onLeave={endCall}
/>
)}
{/* Mascot-guided product tour */}

View file

@ -21,7 +21,10 @@ import {
Lock,
Mic,
MoreHorizontal,
Phone,
PhoneOff,
Plus,
Presentation,
ShieldCheck,
Square,
Terminal,
@ -51,7 +54,6 @@ import {
} from '@/lib/attachment-presentation'
import { getExtension, getFileDisplayName, getMimeFromExtension, isImageMime } from '@/lib/file-utils'
import { cn } from '@/lib/utils'
import { MascotFaceIcon } from '@/components/talking-head'
import {
type FileMention,
type PromptInputMessage,
@ -212,7 +214,17 @@ function compactWorkDirPath(path: string) {
return path.replace(/^\/Users\/[^/]+/, '~')
}
export type VideoChatMode = 'off' | 'chat' | 'call' | 'meeting'
// Call presets: front doors into the same call engine, differing only in
// starting devices. 'share' is the call button's main click — the "work
// together" default (screen shared, camera off, floating pill). The chevron
// menu holds the deviations.
export type CallPreset = 'voice' | 'video' | 'share' | 'practice'
const CALL_PRESET_MENU: Array<{ preset: CallPreset; label: string; description: string; Icon: typeof Phone }> = [
{ preset: 'voice', label: 'Voice call', description: 'Just talk — nothing is shared, the mascot hovers while you work', Icon: AudioLines },
{ preset: 'video', label: 'Video call', description: 'Camera on, face to face — it sees your expressions', Icon: Video },
{ preset: 'practice', label: 'Practice session', description: 'Rehearse a pitch or interview with live coaching', Icon: Presentation },
]
interface ChatInputInnerProps {
onSubmit: (message: PromptInputMessage, mentions?: FileMention[], attachments?: StagedAttachment[], searchEnabled?: boolean, codeMode?: 'claude' | 'codex', permissionMode?: PermissionMode) => void
@ -236,18 +248,14 @@ interface ChatInputInnerProps {
voiceAvailable?: boolean
ttsAvailable?: boolean
ttsEnabled?: boolean
ttsMode?: 'summary' | 'full'
onToggleTts?: () => void
onTtsModeChange?: (mode: 'summary' | 'full') => void
ttsAvatarEnabled?: boolean
onToggleTtsAvatar?: () => void
/** Video chat mode: 'chat' attaches webcam frames to messages; 'call' and
* 'meeting' are fully hands-free continuous listening, spoken responses
* ('meeting' additionally takes over the whole screen, Meet-style). */
videoChatMode?: VideoChatMode
onVideoModeChange?: (mode: VideoChatMode) => void
/** Hands-free call needs both voice input (STT) and voice output (TTS). */
videoCallAvailable?: boolean
/** A call is live (hands-free voice loop + spoken responses). */
inCall?: boolean
/** Start a call with the given preset's device defaults. */
onStartCall?: (preset: CallPreset) => void
onEndCall?: () => void
/** Calls need both voice input (STT) and voice output (TTS) configured. */
callAvailable?: boolean
/** Fired when the user picks a different model in the dropdown (only when no run exists yet). */
onSelectedModelChange?: (model: SelectedModel | null) => void
/** Work directory for this chat (per-chat). Null when none is set. */
@ -283,14 +291,11 @@ function ChatInputInner({
voiceAvailable,
ttsAvailable,
ttsEnabled,
ttsMode,
onToggleTts,
onTtsModeChange,
ttsAvatarEnabled,
onToggleTtsAvatar,
videoChatMode = 'off',
onVideoModeChange,
videoCallAvailable,
inCall,
onStartCall,
onEndCall,
callAvailable,
onSelectedModelChange,
workDir = null,
onWorkDirChange,
@ -1290,68 +1295,21 @@ function ChatInputInner({
</DropdownMenu>
) : null}
{onToggleTts && ttsAvailable && (
<div className="flex shrink-0 items-center">
<Tooltip delayDuration={CHAT_INPUT_TOOLTIP_DELAY_MS}>
<TooltipTrigger asChild>
<button
type="button"
onClick={onToggleTts}
className={cn(
'relative flex h-7 w-7 shrink-0 items-center justify-center rounded-full transition-colors',
ttsEnabled
? 'text-foreground hover:bg-muted'
: 'text-muted-foreground hover:bg-muted hover:text-foreground'
)}
aria-label={ttsEnabled ? 'Disable voice output' : 'Enable voice output'}
>
<Headphones className="h-4 w-4" />
{!ttsEnabled && (
<span className="absolute inset-0 flex items-center justify-center pointer-events-none">
<span className="block h-[1.5px] w-5 -rotate-45 rounded-full bg-muted-foreground" />
</span>
)}
</button>
</TooltipTrigger>
<TooltipContent side="top">
{ttsEnabled ? 'Voice output on' : 'Voice output off'}
</TooltipContent>
</Tooltip>
{ttsEnabled && onTtsModeChange && (
<DropdownMenu>
<DropdownMenuTrigger asChild>
<button
type="button"
className="flex h-7 w-4 shrink-0 items-center justify-center text-muted-foreground transition-colors hover:text-foreground"
>
<ChevronDown className="h-3 w-3" />
</button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end">
<DropdownMenuRadioGroup value={ttsMode ?? 'summary'} onValueChange={(v) => onTtsModeChange(v as 'summary' | 'full')}>
<DropdownMenuRadioItem value="summary">Speak summary</DropdownMenuRadioItem>
<DropdownMenuRadioItem value="full">Speak full response</DropdownMenuRadioItem>
</DropdownMenuRadioGroup>
</DropdownMenuContent>
</DropdownMenu>
)}
</div>
)}
{onToggleTtsAvatar && ttsAvailable && (
<Tooltip delayDuration={CHAT_INPUT_TOOLTIP_DELAY_MS}>
<TooltipTrigger asChild>
<button
type="button"
onClick={onToggleTtsAvatar}
onClick={onToggleTts}
className={cn(
'relative flex h-7 w-7 shrink-0 items-center justify-center rounded-full transition-colors',
ttsAvatarEnabled
ttsEnabled
? 'text-foreground hover:bg-muted'
: 'text-muted-foreground hover:bg-muted hover:text-foreground'
)}
aria-label={ttsAvatarEnabled ? 'Disable talking head' : 'Enable talking head'}
aria-label={ttsEnabled ? 'Disable read-aloud' : 'Enable read-aloud'}
>
<MascotFaceIcon />
{!ttsAvatarEnabled && (
<Headphones className="h-4 w-4" />
{!ttsEnabled && (
<span className="absolute inset-0 flex items-center justify-center pointer-events-none">
<span className="block h-[1.5px] w-5 -rotate-45 rounded-full bg-muted-foreground" />
</span>
@ -1359,64 +1317,70 @@ function ChatInputInner({
</button>
</TooltipTrigger>
<TooltipContent side="top">
{ttsAvatarEnabled ? 'Talking head on' : 'Talking head off'}
{ttsEnabled ? 'Read responses aloud: on' : 'Read responses aloud: off'}
</TooltipContent>
</Tooltip>
)}
{onVideoModeChange && (
{onStartCall && (
<div className="flex shrink-0 items-center">
<Tooltip delayDuration={CHAT_INPUT_TOOLTIP_DELAY_MS}>
<TooltipTrigger asChild>
<button
type="button"
onClick={() => onVideoModeChange(videoChatMode === 'off' ? 'chat' : 'off')}
onClick={() => {
if (inCall) {
onEndCall?.()
} else if (callAvailable) {
onStartCall('share')
}
}}
className={cn(
'relative flex h-7 w-7 shrink-0 items-center justify-center rounded-full transition-colors',
videoChatMode !== 'off'
? 'text-red-500 hover:bg-muted'
: 'text-muted-foreground hover:bg-muted hover:text-foreground'
'flex h-7 w-7 shrink-0 items-center justify-center rounded-full transition-colors',
inCall
? 'bg-red-600 text-white hover:bg-red-500'
: callAvailable
? 'text-muted-foreground hover:bg-muted hover:text-foreground'
: 'cursor-default text-muted-foreground/40'
)}
aria-label={videoChatMode === 'off' ? 'Turn on video chat' : 'Turn off video chat'}
aria-label={inCall ? 'End call' : 'Start a call'}
>
<Video className="h-4 w-4" />
{videoChatMode === 'off' && (
<span className="absolute inset-0 flex items-center justify-center pointer-events-none">
<span className="block h-[1.5px] w-5 -rotate-45 rounded-full bg-muted-foreground" />
</span>
)}
{inCall ? <PhoneOff className="h-4 w-4" /> : <Phone className="h-4 w-4" />}
</button>
</TooltipTrigger>
<TooltipContent side="top">
{videoChatMode === 'off'
? 'Video chat off'
: videoChatMode === 'chat'
? 'Video chat on — camera frames are shared with the assistant'
: 'Video call on — speak freely, the assistant answers out loud'}
{inCall
? 'End call'
: callAvailable
? 'Start a call — it sees your screen while you talk it through'
: 'Calls need voice input and output configured'}
</TooltipContent>
</Tooltip>
{videoChatMode !== 'off' && (
{!inCall && (
<DropdownMenu>
<DropdownMenuTrigger asChild>
<button
type="button"
className="flex h-7 w-4 shrink-0 items-center justify-center text-muted-foreground transition-colors hover:text-foreground"
aria-label="Call options"
>
<ChevronDown className="h-3 w-3" />
</button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end">
<DropdownMenuRadioGroup
value={videoChatMode}
onValueChange={(v) => onVideoModeChange(v as VideoChatMode)}
>
<DropdownMenuRadioItem value="chat">Video + chat</DropdownMenuRadioItem>
<DropdownMenuRadioItem value="call" disabled={!videoCallAvailable}>
Video call (hands-free)
</DropdownMenuRadioItem>
<DropdownMenuRadioItem value="meeting" disabled={!videoCallAvailable}>
Video call (full screen)
</DropdownMenuRadioItem>
</DropdownMenuRadioGroup>
<DropdownMenuContent align="end" className="w-72">
{CALL_PRESET_MENU.map(({ preset, label, description, Icon }) => (
<DropdownMenuItem
key={preset}
disabled={!callAvailable}
onSelect={() => onStartCall(preset)}
className="items-start gap-3 py-2"
>
<Icon className="mt-0.5 h-4 w-4 shrink-0 text-muted-foreground" />
<span className="min-w-0">
<span className="block text-sm font-medium leading-tight">{label}</span>
<span className="block pt-0.5 text-xs leading-tight text-muted-foreground">{description}</span>
</span>
</DropdownMenuItem>
))}
</DropdownMenuContent>
</DropdownMenu>
)}
@ -1593,14 +1557,11 @@ export interface ChatInputWithMentionsProps {
voiceAvailable?: boolean
ttsAvailable?: boolean
ttsEnabled?: boolean
ttsMode?: 'summary' | 'full'
onToggleTts?: () => void
onTtsModeChange?: (mode: 'summary' | 'full') => void
ttsAvatarEnabled?: boolean
onToggleTtsAvatar?: () => void
videoChatMode?: VideoChatMode
onVideoModeChange?: (mode: VideoChatMode) => void
videoCallAvailable?: boolean
inCall?: boolean
onStartCall?: (preset: CallPreset) => void
onEndCall?: () => void
callAvailable?: boolean
onSelectedModelChange?: (model: SelectedModel | null) => void
workDir?: string | null
onWorkDirChange?: (value: string | null) => void
@ -1632,14 +1593,11 @@ export function ChatInputWithMentions({
voiceAvailable,
ttsAvailable,
ttsEnabled,
ttsMode,
onToggleTts,
onTtsModeChange,
ttsAvatarEnabled,
onToggleTtsAvatar,
videoChatMode,
onVideoModeChange,
videoCallAvailable,
inCall,
onStartCall,
onEndCall,
callAvailable,
onSelectedModelChange,
workDir,
onWorkDirChange,
@ -1668,14 +1626,11 @@ export function ChatInputWithMentions({
voiceAvailable={voiceAvailable}
ttsAvailable={ttsAvailable}
ttsEnabled={ttsEnabled}
ttsMode={ttsMode}
onToggleTts={onToggleTts}
onTtsModeChange={onTtsModeChange}
ttsAvatarEnabled={ttsAvatarEnabled}
onToggleTtsAvatar={onToggleTtsAvatar}
videoChatMode={videoChatMode}
onVideoModeChange={onVideoModeChange}
videoCallAvailable={videoCallAvailable}
inCall={inCall}
onStartCall={onStartCall}
onEndCall={onEndCall}
callAvailable={callAvailable}
onSelectedModelChange={onSelectedModelChange}
workDir={workDir}
onWorkDirChange={onWorkDirChange}

View file

@ -37,7 +37,7 @@ import { MarkdownPreOverride } from '@/components/ai-elements/markdown-code-over
import { defaultRemarkPlugins } from 'streamdown'
import remarkBreaks from 'remark-breaks'
import { type ChatTab } from '@/components/tab-bar'
import { ChatInputWithMentions, type PermissionMode, type StagedAttachment, type SelectedModel, type VideoChatMode } from '@/components/chat-input-with-mentions'
import { ChatInputWithMentions, type CallPreset, type PermissionMode, type StagedAttachment, type SelectedModel } from '@/components/chat-input-with-mentions'
import { ChatMessageAttachments } from '@/components/chat-message-attachments'
import { useSidebar } from '@/components/ui/sidebar'
import { wikiLabel } from '@/lib/wiki-links'
@ -189,14 +189,11 @@ interface ChatSidebarProps {
voiceAvailable?: boolean
ttsAvailable?: boolean
ttsEnabled?: boolean
ttsMode?: 'summary' | 'full'
onToggleTts?: () => void
onTtsModeChange?: (mode: 'summary' | 'full') => void
ttsAvatarEnabled?: boolean
onToggleTtsAvatar?: () => void
videoChatMode?: VideoChatMode
onVideoModeChange?: (mode: VideoChatMode) => void
videoCallAvailable?: boolean
inCall?: boolean
onStartCall?: (preset: CallPreset) => void
onEndCall?: () => void
callAvailable?: boolean
onComposioConnected?: (toolkitSlug: string) => void
}
@ -258,14 +255,11 @@ export function ChatSidebar({
voiceAvailable,
ttsAvailable,
ttsEnabled,
ttsMode,
onToggleTts,
onTtsModeChange,
ttsAvatarEnabled,
onToggleTtsAvatar,
videoChatMode,
onVideoModeChange,
videoCallAvailable,
inCall,
onStartCall,
onEndCall,
callAvailable,
onComposioConnected,
}: ChatSidebarProps) {
const { state: sidebarState } = useSidebar()
@ -837,14 +831,11 @@ export function ChatSidebar({
voiceAvailable={isActive && voiceAvailable}
ttsAvailable={isActive && ttsAvailable}
ttsEnabled={ttsEnabled}
ttsMode={ttsMode}
onToggleTts={isActive ? onToggleTts : undefined}
onTtsModeChange={isActive ? onTtsModeChange : undefined}
ttsAvatarEnabled={ttsAvatarEnabled}
onToggleTtsAvatar={isActive ? onToggleTtsAvatar : undefined}
videoChatMode={videoChatMode}
onVideoModeChange={isActive ? onVideoModeChange : undefined}
videoCallAvailable={videoCallAvailable}
inCall={inCall}
onStartCall={isActive ? onStartCall : undefined}
onEndCall={isActive ? onEndCall : undefined}
callAvailable={callAvailable}
/>
</div>
)

View file

@ -1,5 +1,5 @@
import { useEffect, useRef, useState } from 'react'
import { MonitorUp, PhoneOff, User, Video, VideoOff } from 'lucide-react'
import { Minimize2, MonitorUp, PhoneOff, Presentation, User, Video, VideoOff } from 'lucide-react'
import { MascotFaceIcon, TalkingHead } from '@/components/talking-head'
import type { TTSState } from '@/hooks/useVoiceTTS'
@ -10,12 +10,15 @@ export type VideoCallStatus = 'listening' | 'thinking' | 'speaking'
interface VideoCallViewProps {
/** Live camera stream from useVideoMode — attached to the user's tile. */
streamRef: React.MutableRefObject<MediaStream | null>
/** Live screen-share stream — shown as the presentation tile when sharing. */
screenStreamRef: React.MutableRefObject<MediaStream | null>
isScreenSharing: boolean
onToggleScreenShare: () => void
cameraOn: boolean
onToggleCamera: () => void
/** Starting a share collapses this view into the floating popout (the
* surface is derived from devices see App.tsx). */
onToggleScreenShare: () => void
/** Practice preset: the assistant is coaching this session. */
practiceMode?: boolean
/** Shrink to the floating pill without touching any devices. */
onMinimize: () => void
ttsState: TTSState
/** Live TTS output level — drives the mascot's mouth animation. */
getTtsLevel: () => number
@ -33,53 +36,22 @@ const STATUS_DISPLAY: Record<VideoCallStatus, { label: string; dotClass: string
speaking: { label: 'Speaking', dotClass: 'bg-sky-400 animate-pulse' },
}
/** Attach a MediaStream ref to a <video> element for the lifetime of the mount. */
function StreamVideo({
streamRef,
mirrored,
className,
}: {
streamRef: React.MutableRefObject<MediaStream | null>
mirrored?: boolean
className?: string
}) {
const videoRef = useRef<HTMLVideoElement | null>(null)
useEffect(() => {
const videoEl = videoRef.current
if (!videoEl) return
videoEl.srcObject = streamRef.current
videoEl.play().catch(() => {})
return () => {
videoEl.srcObject = null
}
}, [streamRef])
return (
<video
ref={videoRef}
muted
playsInline
className={className}
style={mirrored ? { transform: 'scaleX(-1)' } : undefined}
/>
)
}
/**
* Full-screen hands-free call: a Meet-style layout with the user's webcam on
* one side and the mascot as the other participant. While presenting, the
* shared screen becomes the big tile and the participants shrink into a side
* rail. The mascot animates with the assistant's speech; dismissing it swaps
* in a Meet-style letter avatar ("R"). Live captions run along the bottom.
* Full-screen call surface: a Meet-style two-tile layout with the user's
* webcam on one side and the mascot as the other participant. Shown only
* while the camera is on with no screen share (the derived-surface rule in
* App.tsx) sharing or muting the camera moves the call into the floating
* popout. The mascot animates with the assistant's speech; dismissing it
* swaps in a Meet-style letter avatar ("R"). Live captions run along the
* bottom.
*/
export function VideoCallView({
streamRef,
screenStreamRef,
isScreenSharing,
onToggleScreenShare,
cameraOn,
onToggleCamera,
onToggleScreenShare,
practiceMode,
onMinimize,
ttsState,
getTtsLevel,
status,
@ -87,8 +59,20 @@ export function VideoCallView({
assistantCaption,
onLeave,
}: VideoCallViewProps) {
const videoRef = useRef<HTMLVideoElement | null>(null)
const [mascotVisible, setMascotVisible] = useState(true)
useEffect(() => {
if (!cameraOn) return
const videoEl = videoRef.current
if (!videoEl) return
videoEl.srcObject = streamRef.current
videoEl.play().catch(() => {})
return () => {
videoEl.srcObject = null
}
}, [streamRef, cameraOn])
const userSpeaking = status === 'listening' && Boolean(interimText)
const assistantSpeaking = ttsState === 'speaking'
@ -98,89 +82,80 @@ export function VideoCallView({
? { who: 'You', text: interimText }
: null
const userTile = (
<div
className={cn(
'relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
userSpeaking && 'ring-2 ring-green-500/80',
isScreenSharing && 'aspect-video w-full'
)}
>
{cameraOn ? (
<StreamVideo streamRef={streamRef} mirrored className="h-full w-full object-cover" />
) : (
<span
className={cn(
'flex items-center justify-center rounded-full bg-neutral-700 text-neutral-400',
isScreenSharing ? 'h-16 w-16' : 'h-40 w-40'
)}
aria-label="Camera off"
>
<User className={isScreenSharing ? 'h-8 w-8' : 'h-20 w-20'} />
</span>
)}
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
You
</span>
</div>
)
const assistantTile = (
<div
className={cn(
'group relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
assistantSpeaking && 'ring-2 ring-sky-400/80',
isScreenSharing && 'aspect-video w-full'
)}
>
{mascotVisible ? (
<TalkingHead ttsState={ttsState} getLevel={getTtsLevel} size={isScreenSharing ? 96 : 220} />
) : (
<span
className={cn(
'flex items-center justify-center rounded-full bg-sky-600 font-medium text-white',
isScreenSharing ? 'h-16 w-16 text-3xl' : 'h-40 w-40 text-7xl'
)}
aria-label="Rowboat"
>
R
</span>
)}
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
Rowboat
</span>
<button
type="button"
onClick={() => setMascotVisible((v) => !v)}
className="absolute right-3 top-3 rounded-md bg-black/50 px-2 py-1 text-xs text-white/80 opacity-0 transition-opacity hover:text-white group-hover:opacity-100"
>
{mascotVisible ? 'Hide mascot' : 'Show mascot'}
</button>
</div>
)
return (
<div className="fixed inset-0 z-[100] flex flex-col bg-neutral-950">
{/* Participant tiles — Meet-style presentation layout while sharing */}
{isScreenSharing ? (
<div className="flex min-h-0 flex-1 gap-3 p-4 pb-2">
<div className="relative flex flex-1 items-center justify-center overflow-hidden rounded-2xl bg-neutral-900">
<StreamVideo streamRef={screenStreamRef} className="h-full w-full object-contain" />
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
Your screen
</span>
</div>
<div className="flex w-52 shrink-0 flex-col gap-3">
{userTile}
{assistantTile}
</div>
</div>
) : (
<div className="grid min-h-0 flex-1 grid-cols-1 gap-3 p-4 pb-2 md:grid-cols-2">
{userTile}
{assistantTile}
</div>
{practiceMode && (
<span className="absolute left-4 top-4 z-10 flex items-center gap-1.5 rounded-full bg-violet-600/90 px-3 py-1 text-xs font-medium text-white">
<Presentation className="h-3.5 w-3.5" />
Practice session
</span>
)}
<button
type="button"
onClick={onMinimize}
className="absolute right-4 top-4 z-10 flex h-8 w-8 items-center justify-center rounded-full bg-neutral-800 text-white/80 transition-colors hover:bg-neutral-700 hover:text-white"
aria-label="Minimize call (shares your screen)"
title="Minimize — shares your screen so it can help you work"
>
<Minimize2 className="h-4 w-4" />
</button>
{/* Participant tiles */}
<div className="grid min-h-0 flex-1 grid-cols-1 gap-3 p-4 pb-2 md:grid-cols-2">
{/* User */}
<div
className={cn(
'relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
userSpeaking && 'ring-2 ring-green-500/80'
)}
>
{cameraOn ? (
<video
ref={videoRef}
muted
playsInline
className="h-full w-full object-cover"
style={{ transform: 'scaleX(-1)' }}
/>
) : (
<span className="flex h-40 w-40 items-center justify-center rounded-full bg-neutral-700 text-neutral-400" aria-label="Camera off">
<User className="h-20 w-20" />
</span>
)}
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
You
</span>
</div>
{/* Assistant */}
<div
className={cn(
'group relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
assistantSpeaking && 'ring-2 ring-sky-400/80'
)}
>
{mascotVisible ? (
<TalkingHead ttsState={ttsState} getLevel={getTtsLevel} size={220} />
) : (
<span
className="flex h-40 w-40 items-center justify-center rounded-full bg-sky-600 text-7xl font-medium text-white"
aria-label="Rowboat"
>
R
</span>
)}
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
Rowboat
</span>
<button
type="button"
onClick={() => setMascotVisible((v) => !v)}
className="absolute right-3 top-3 rounded-md bg-black/50 px-2 py-1 text-xs text-white/80 opacity-0 transition-opacity hover:text-white group-hover:opacity-100"
>
{mascotVisible ? 'Hide mascot' : 'Show mascot'}
</button>
</div>
</div>
{/* Captions */}
<div className="flex h-14 items-center justify-center px-6">
@ -215,14 +190,9 @@ export function VideoCallView({
<button
type="button"
onClick={onToggleScreenShare}
className={cn(
'flex h-10 w-10 items-center justify-center rounded-full transition-colors',
isScreenSharing
? 'bg-sky-600 text-white hover:bg-sky-500'
: 'bg-neutral-800 text-white/90 hover:bg-neutral-700'
)}
aria-label={isScreenSharing ? 'Stop presenting' : 'Present your screen'}
title={isScreenSharing ? 'Stop presenting' : 'Present your screen'}
className="flex h-10 w-10 items-center justify-center rounded-full bg-neutral-800 text-white/90 transition-colors hover:bg-neutral-700"
aria-label="Present your screen"
title="Present your screen"
>
<MonitorUp className="h-5 w-5" />
</button>
@ -243,7 +213,7 @@ export function VideoCallView({
type="button"
onClick={onLeave}
className="flex h-10 w-14 items-center justify-center rounded-full bg-red-600 text-white transition-colors hover:bg-red-500"
aria-label="Leave call"
aria-label="End call"
>
<PhoneOff className="h-5 w-5" />
</button>

View file

@ -7,6 +7,7 @@ type PopoutState = {
ttsState: 'idle' | 'synthesizing' | 'speaking'
status: 'listening' | 'thinking' | 'speaking' | null
cameraOn: boolean
screenSharing: boolean
interimText: string | null
}
@ -23,18 +24,30 @@ const noDragRegion = { WebkitAppRegion: 'no-drag' } as React.CSSProperties
* Content of the always-on-top popout window shown for the whole duration of
* a screen share (Meet-style floating mini-call) it floats over every app,
* including Rowboat itself, and is the call's control surface while sharing:
* camera toggle, stop-share, end-call. Rendered in its own BrowserWindow
* camera toggle, share toggle, end-call. Rendered in its own BrowserWindow
* (see `video:setPopout` in the main process); call state streams in over
* the `video:popout-state` push channel and control actions round-trip back
* through `video:popoutAction`. Captures its own webcam feed MediaStreams
* can't cross windows.
*/
export function VideoPopout() {
const [state, setState] = useState<PopoutState>({ ttsState: 'idle', status: null, cameraOn: true, interimText: null })
// Camera defaults OFF: guessing "on" would flash the user's video for a
// beat before the real state arrives — which reads as a bug. The true
// state is fetched immediately below.
const [state, setState] = useState<PopoutState>({ ttsState: 'idle', status: null, cameraOn: false, screenSharing: false, interimText: null })
const videoRef = useRef<HTMLVideoElement | null>(null)
useEffect(() => {
return window.ipc.on('video:popout-state', (next) => setState(next))
const cleanup = window.ipc.on('video:popout-state', (next) => setState(next))
// The main process replays the cached state on did-finish-load, but that
// can race this listener's registration — fetch it explicitly too.
window.ipc
.invoke('video:getPopoutState', null)
.then(({ state: cached }) => {
if (cached) setState(cached)
})
.catch(() => {})
return cleanup
}, [])
// Own camera feed, following the main window's camera-on/off state.
@ -67,7 +80,7 @@ export function VideoPopout() {
// so the mascot still animates while the assistant speaks in the main window.
const getLevel = useCallback(() => 0.45 + 0.35 * Math.sin(performance.now() / 90), [])
const sendAction = useCallback((action: 'toggle-camera' | 'stop-share' | 'end-call') => {
const sendAction = useCallback((action: 'toggle-camera' | 'toggle-share' | 'end-call' | 'expand') => {
void window.ipc.invoke('video:popoutAction', { action }).catch(() => {})
}, [])
@ -98,6 +111,14 @@ export function VideoPopout() {
<span className="absolute bottom-1 left-1.5 rounded bg-black/50 px-1 py-px text-[10px] text-white">
You
</span>
{/* Persistent consent badge the user must always be able to see
at a glance that their screen is going out. */}
{state.screenSharing && (
<span className="absolute left-1.5 top-1.5 flex items-center gap-1 rounded-full bg-sky-600/90 px-1.5 py-0.5 text-[10px] font-medium text-white">
<span className="block h-1.5 w-1.5 animate-pulse rounded-full bg-white" />
Sharing screen
</span>
)}
</div>
<div className="relative flex flex-1 items-center justify-center overflow-hidden rounded-lg bg-neutral-800">
<TalkingHead ttsState={state.ttsState} getLevel={getLevel} size={84} />
@ -138,10 +159,14 @@ export function VideoPopout() {
</button>
<button
type="button"
onClick={() => sendAction('stop-share')}
className="flex h-6 w-6 items-center justify-center rounded-full bg-sky-600 text-white transition-colors hover:bg-sky-500"
aria-label="Stop sharing screen"
title="Stop sharing screen"
onClick={() => sendAction('toggle-share')}
className={`flex h-6 w-6 items-center justify-center rounded-full transition-colors ${
state.screenSharing
? 'bg-sky-600 text-white hover:bg-sky-500'
: 'bg-neutral-700 text-white/90 hover:bg-neutral-600'
}`}
aria-label={state.screenSharing ? 'Stop sharing screen' : 'Share your screen'}
title={state.screenSharing ? 'Stop sharing screen' : 'Share your screen'}
>
<MonitorUp className="h-3.5 w-3.5" />
</button>
@ -149,17 +174,17 @@ export function VideoPopout() {
type="button"
onClick={() => sendAction('end-call')}
className="flex h-6 w-8 items-center justify-center rounded-full bg-red-600 text-white transition-colors hover:bg-red-500"
aria-label="End video mode"
title="End video mode"
aria-label="End call"
title="End call"
>
<PhoneOff className="h-3.5 w-3.5" />
</button>
<button
type="button"
onClick={() => void window.ipc.invoke('video:focusMain', null)}
onClick={() => sendAction('expand')}
className="flex h-6 w-6 items-center justify-center rounded-full bg-neutral-700 text-white/90 transition-colors hover:bg-neutral-600"
aria-label="Back to Rowboat"
title="Back to Rowboat"
aria-label="Expand to full screen (stops screen sharing)"
title="Expand to full screen (stops sharing)"
>
<Maximize2 className="h-3.5 w-3.5" />
</button>

View file

@ -1,129 +0,0 @@
import { useEffect, useRef } from 'react'
import { MonitorUp, User, VideoOff, X } from 'lucide-react'
interface VideoPreviewOverlayProps {
/** Live camera stream from useVideoMode — attached to the preview element. */
streamRef: React.MutableRefObject<MediaStream | null>
onTurnOff: () => void
/** Hands-free call mode: current phase of the conversation loop. */
callStatus?: 'listening' | 'thinking' | 'speaking'
/** Hands-free call mode: live transcript of the in-progress utterance. */
interimText?: string
isScreenSharing?: boolean
onToggleScreenShare?: () => void
cameraOn?: boolean
onToggleCamera?: () => void
}
const CALL_STATUS_DISPLAY: Record<NonNullable<VideoPreviewOverlayProps['callStatus']>, { label: string; dotClass: string }> = {
listening: { label: 'Listening', dotClass: 'bg-green-500 animate-pulse' },
thinking: { label: 'Thinking…', dotClass: 'bg-amber-400' },
speaking: { label: 'Speaking', dotClass: 'bg-sky-400 animate-pulse' },
}
/**
* Floating picture-in-picture webcam preview shown while video chat mode is
* on. Mirrored like a selfie camera so the user's movements feel natural.
* Sits above the composer dock, mirroring the talking-head overlay's corner.
*/
export function VideoPreviewOverlay({ streamRef, onTurnOff, callStatus, interimText, isScreenSharing, onToggleScreenShare, cameraOn = true, onToggleCamera }: VideoPreviewOverlayProps) {
const videoRef = useRef<HTMLVideoElement | null>(null)
useEffect(() => {
if (!cameraOn) return
const videoEl = videoRef.current
if (!videoEl) return
videoEl.srcObject = streamRef.current
videoEl.play().catch(() => {})
return () => {
videoEl.srcObject = null
}
}, [streamRef, cameraOn])
return (
<div
className="group fixed bottom-28 left-8 z-50"
style={{ animation: 'video-preview-pop 0.35s cubic-bezier(0.34, 1.56, 0.64, 1)' }}
>
<style>{`
@keyframes video-preview-pop {
0% { opacity: 0; scale: 0.4; }
100% { opacity: 1; scale: 1; }
}
`}</style>
<button
type="button"
onClick={onTurnOff}
className="absolute -right-1.5 -top-1.5 z-10 flex h-5 w-5 items-center justify-center rounded-full border border-border bg-background text-muted-foreground opacity-0 shadow-sm transition-opacity hover:text-foreground group-hover:opacity-100"
aria-label="Turn off video chat"
>
<X className="h-3 w-3" />
</button>
{onToggleScreenShare && (
<button
type="button"
onClick={onToggleScreenShare}
className={`absolute -left-1.5 -top-1.5 z-10 flex h-5 w-5 items-center justify-center rounded-full border shadow-sm transition-opacity ${
isScreenSharing
? 'border-sky-500 bg-sky-600 text-white opacity-100'
: 'border-border bg-background text-muted-foreground opacity-0 hover:text-foreground group-hover:opacity-100'
}`}
aria-label={isScreenSharing ? 'Stop sharing screen' : 'Share your screen'}
title={isScreenSharing ? 'Stop sharing screen' : 'Share your screen'}
>
<MonitorUp className="h-3 w-3" />
</button>
)}
{cameraOn ? (
<video
ref={videoRef}
muted
playsInline
className="h-32 w-auto rounded-xl border border-border/70 bg-black shadow-lg"
style={{ transform: 'scaleX(-1)' }}
/>
) : (
<div className="flex h-32 w-48 items-center justify-center rounded-xl border border-border/70 bg-black shadow-lg">
<span className="flex h-14 w-14 items-center justify-center rounded-full bg-neutral-700 text-neutral-400">
<User className="h-7 w-7" />
</span>
</div>
)}
{onToggleCamera && (
<button
type="button"
onClick={onToggleCamera}
className={`absolute -left-1.5 top-4 z-10 flex h-5 w-5 items-center justify-center rounded-full border shadow-sm transition-opacity ${
cameraOn
? 'border-border bg-background text-muted-foreground opacity-0 hover:text-foreground group-hover:opacity-100'
: 'border-red-500 bg-red-600 text-white opacity-100'
}`}
aria-label={cameraOn ? 'Turn off camera' : 'Turn on camera'}
title={cameraOn ? 'Turn off camera' : 'Turn on camera'}
>
<VideoOff className="h-3 w-3" />
</button>
)}
<span className="pointer-events-none absolute bottom-1.5 left-1.5 flex items-center gap-1 rounded-full bg-black/60 px-1.5 py-0.5 text-[10px] font-medium text-white">
{callStatus ? (
<>
<span className={`block h-1.5 w-1.5 rounded-full ${CALL_STATUS_DISPLAY[callStatus].dotClass}`} />
{CALL_STATUS_DISPLAY[callStatus].label}
</>
) : (
<>
<span className="block h-1.5 w-1.5 animate-pulse rounded-full bg-red-500" />
Video on
</>
)}
</span>
{callStatus && interimText && (
<div className="pointer-events-none absolute inset-x-0 -bottom-1 translate-y-full pt-1">
<div className="max-h-16 overflow-hidden rounded-lg bg-black/60 px-2 py-1 text-[11px] leading-snug text-white/90">
{interimText}
</div>
</div>
)}
</div>
)
}

View file

@ -234,15 +234,22 @@ export function useVideoMode() {
return true;
}, [acquireCamera]);
const start = useCallback(async (): Promise<boolean> => {
/**
* Start video mode. `camera: false` starts a camera-less session (voice
* call / screen-share-only) the mode is live so frames can flow from
* other sources, and the camera can be enabled later via setCameraEnabled.
*/
const start = useCallback(async ({ camera = true }: { camera?: boolean } = {}): Promise<boolean> => {
if (stateRef.current !== 'idle') return true;
setState('starting');
const ok = await acquireCamera();
if (!ok) {
setState('idle');
return false;
if (camera) {
const ok = await acquireCamera();
if (!ok) {
setState('idle');
return false;
}
}
setCameraOn(true);
setCameraOn(camera);
setState('live');
return true;
}, [acquireCamera]);

View file

@ -65,6 +65,10 @@ export function voiceInputStarted() {
posthog.capture('voice_input_started')
}
export function callStarted(preset: 'voice' | 'video' | 'share' | 'practice') {
posthog.capture('call_started', { preset })
}
export function searchExecuted(types: string[]) {
posthog.capture('search_executed', { types })
}